From fc9a0d3f1296c3f7aeb96e6c97f985fea165fef9 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 23 Sep 2024 18:50:51 -0400 Subject: [PATCH 01/62] start by refactoring block index into a read-through cache --- go.mod | 58 ++++++- go.sum | 97 +++++++++++ lib/blockchain.go | 327 +++++++++++++++++++++++-------------- lib/db_utils.go | 7 + lib/peer.go | 7 +- lib/pos_blockchain.go | 64 ++++---- lib/pos_blockchain_test.go | 90 +++++----- lib/pos_consensus.go | 27 ++- lib/pos_consensus_test.go | 8 +- lib/postgres.go | 13 +- lib/server.go | 15 +- lib/state_change_syncer.go | 2 +- lib/txindex.go | 10 +- 13 files changed, 497 insertions(+), 228 deletions(-) diff --git a/go.mod b/go.mod index 31047fa1b..a64a34f67 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/bxcodec/faker v2.0.1+incompatible github.com/cloudflare/circl v1.1.0 github.com/davecgh/go-spew v1.1.1 - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 - github.com/decred/dcrd/lru v1.1.1 + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 + github.com/decred/dcrd/lru v1.1.3 github.com/deso-protocol/go-deadlock v1.0.0 github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/dgraph-io/badger/v3 v3.2103.5 @@ -36,8 +36,8 @@ require ( github.com/stretchr/testify v1.8.0 github.com/tyler-smith/go-bip39 v1.0.2 github.com/unrolled/secure v1.0.8 - golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 + golang.org/x/crypto v0.27.0 + golang.org/x/sync v0.8.0 gopkg.in/DataDog/dd-trace-go.v1 v1.29.0 ) @@ -48,13 +48,46 @@ require ( ) require ( + decred.org/cspp/v2 v2.2.0 // indirect github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect github.com/bwesterb/go-ristretto v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/dchest/siphash v1.2.3 // indirect + github.com/decred/base58 v1.0.5 // indirect + github.com/decred/dcrd v1.9.4 // indirect + github.com/decred/dcrd/addrmgr/v2 v2.0.4 // indirect + github.com/decred/dcrd/bech32 v1.1.4 // indirect + github.com/decred/dcrd/blockchain/stake/v5 v5.0.1 // indirect + github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1 // indirect + github.com/decred/dcrd/certgen v1.2.0 // indirect + github.com/decred/dcrd/chaincfg/chainhash v1.0.4 // indirect + github.com/decred/dcrd/chaincfg/v3 v3.2.1 // indirect + github.com/decred/dcrd/connmgr/v3 v3.1.2 // indirect + github.com/decred/dcrd/container/apbf v1.0.1 // indirect + github.com/decred/dcrd/container/lru v1.0.0 // indirect + github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect + github.com/decred/dcrd/crypto/rand v1.0.0 // indirect + github.com/decred/dcrd/crypto/ripemd160 v1.0.2 // indirect + github.com/decred/dcrd/database/v3 v3.0.2 // indirect + github.com/decred/dcrd/dcrec v1.0.1 // indirect + github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3 // indirect + github.com/decred/dcrd/dcrjson/v4 v4.1.0 // indirect + github.com/decred/dcrd/dcrutil/v4 v4.0.2 // indirect + github.com/decred/dcrd/gcs/v4 v4.1.0 // indirect + github.com/decred/dcrd/math/uint256 v1.0.2 // indirect + github.com/decred/dcrd/mixing v0.4.1 // indirect + github.com/decred/dcrd/peer/v3 v3.1.2 // indirect + github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0 // indirect + github.com/decred/dcrd/txscript/v4 v4.1.1 // indirect + github.com/decred/dcrd/wire v1.7.0 // indirect + github.com/decred/go-socks v1.1.0 // indirect + github.com/decred/slog v1.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect @@ -63,15 +96,20 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.8 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jessevdk/go-flags v1.6.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jrick/bitset v1.0.0 // indirect + github.com/jrick/logrotate v1.1.2 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.12.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.9 // indirect @@ -90,6 +128,7 @@ require ( github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tinylib/msgp v1.1.2 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df // indirect @@ -99,11 +138,11 @@ require ( github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gonum.org/v1/gonum v0.6.1 // indirect google.golang.org/protobuf v1.28.1 // indirect @@ -113,5 +152,6 @@ require ( gopkg.in/kyokomi/emoji.v1 v1.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.3.0 // indirect mellium.im/sasl v0.2.1 // indirect ) diff --git a/go.sum b/go.sum index 3698c73f3..821ddcfdd 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +decred.org/cspp/v2 v2.2.0 h1:VSOUC1w0Wo+QOGS0r1XO6TLnO16X67KuvpDmRRYyr08= +decred.org/cspp/v2 v2.2.0/go.mod h1:9nO3bfvCheOPIFZw5f6sRQ42CjBFB5RKSaJ9Iq6G4MA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -39,6 +41,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= +github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -94,6 +98,8 @@ github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GK github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a h1:clYxJ3Os0EQUKDDVU8M0oipllX0EkuFNBfhVQuIfyF0= +github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a/go.mod h1:z/9Ck1EDixEbBbZ2KH2qNHekEmDLTOZ+FyoIPWWSVOI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -111,14 +117,78 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= +github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/decred/base58 v1.0.5 h1:hwcieUM3pfPnE/6p3J100zoRfGkQxBulZHo7GZfOqic= +github.com/decred/base58 v1.0.5/go.mod h1:s/8lukEHFA6bUQQb/v3rjUySJ2hu+RioCzLukAVkrfw= +github.com/decred/dcrd v1.9.4 h1:DgeZdf8TTtGuZGVs5DnAL0ghbhnrPYfENneqUqEaq8o= +github.com/decred/dcrd v1.9.4/go.mod h1:7I6LovGdFimw3LTPCtOGQ+8VV1HGnO+2L2KA4yi5NWA= +github.com/decred/dcrd/addrmgr/v2 v2.0.4 h1:3MWJiO2STogQwNRF3W4yjCzSJtaxqtw+UI3x2+bYeOg= +github.com/decred/dcrd/addrmgr/v2 v2.0.4/go.mod h1:661DIS/De2iLNLMwIKazUQfQypUqJ5om7PXNX0fEMms= +github.com/decred/dcrd/bech32 v1.1.4 h1:wFlLM7Oic0MlIhQZdCQhdIqVc4CNaQ0vNR9fgCoWfe0= +github.com/decred/dcrd/bech32 v1.1.4/go.mod h1:jliqHZmCbVfT06Lh1mQywEKFVidRclbBJIUmwdoKhu0= +github.com/decred/dcrd/blockchain/stake/v5 v5.0.1 h1:KDm6myUPi8j2TTL7LZ+iT+R/pIbxd8qG89fjJNitzx0= +github.com/decred/dcrd/blockchain/stake/v5 v5.0.1/go.mod h1:y1tMD1TssTlPmKDYbSrF3Ujznj+STkXFfYPwoVfe+xA= +github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1 h1:zeI9CHkLM9be4QOBmIAtoPfs6NCgJM1lpmRUYE61I8o= +github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1/go.mod h1:yXZz/EgWdGw5nqMEvyKj/iXZ9I2VSyO95xKj6mRUMIM= +github.com/decred/dcrd/certgen v1.2.0 h1:FF6XXV//5q38/c6QbGQdR35ZJz0GPIkejsZZU3oHuBQ= +github.com/decred/dcrd/certgen v1.2.0/go.mod h1:LRh6dF2WPQeDA6QQSZE+SfK7AL6FuFtCRDHZf8DyGzg= +github.com/decred/dcrd/chaincfg/chainhash v1.0.4 h1:zRCv6tdncLfLTKYqu7hrXvs7hW+8FO/NvwoFvGsrluU= +github.com/decred/dcrd/chaincfg/chainhash v1.0.4/go.mod h1:hA86XxlBWwHivMvxzXTSD0ZCG/LoYsFdWnCekkTMCqY= +github.com/decred/dcrd/chaincfg/v3 v3.2.1 h1:x9zKJaU24WAKbxAR1UyFKHlM3oJgP0H9LodokM4X5lM= +github.com/decred/dcrd/chaincfg/v3 v3.2.1/go.mod h1:SDCWDtY7BLj0leXc9FuoA1YjSVKyCIBVAyxwZn6+sXc= +github.com/decred/dcrd/connmgr/v3 v3.1.2 h1:+xNopie2L3YYwwkz51k0h/pASATOBzHtl2O8eodGg04= +github.com/decred/dcrd/connmgr/v3 v3.1.2/go.mod h1:tdbErFiNOuy/sHrX2mwaOk+r1HLs3EBz2EGxsocMPe4= +github.com/decred/dcrd/container/apbf v1.0.1 h1:oepQzRtLADudsrx0AmmowoU1kambozINTMXduH6Mge0= +github.com/decred/dcrd/container/apbf v1.0.1/go.mod h1:paQplssZMsRhwOUcP6LDNDypyb7lwsFHrUKUkAPFWtQ= +github.com/decred/dcrd/container/lru v1.0.0 h1:7foQymtbu18aQWYiY9RnNIeE+kvpiN+fiBQ3+viyJjI= +github.com/decred/dcrd/container/lru v1.0.0/go.mod h1:vlPwj0l+IzAHhQSsbgQnJgO5Cte78+yI065V+Mc5PRQ= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= +github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/crypto/rand v1.0.0 h1:Ah9Asl36OZt09sGSMbJZuL1HfwGdlC38q/ZUeLDVKRg= +github.com/decred/dcrd/crypto/rand v1.0.0/go.mod h1:coa7BbxSTiKH6esi257plGfMFYuGL4MTbQlLYnOdzpE= +github.com/decred/dcrd/crypto/ripemd160 v1.0.2 h1:TvGTmUBHDU75OHro9ojPLK+Yv7gDl2hnUvRocRCjsys= +github.com/decred/dcrd/crypto/ripemd160 v1.0.2/go.mod h1:uGfjDyePSpa75cSQLzNdVmWlbQMBuiJkvXw/MNKRY4M= +github.com/decred/dcrd/database/v3 v3.0.2 h1:rgP7XNZemTs8ZC7bnTKO8JO79Woj5nq+yQYmB9ry7yM= +github.com/decred/dcrd/database/v3 v3.0.2/go.mod h1:3Ge1yoxEOsqd72V5LTA9g0B7mlY0MGbpxeE1fniIXsQ= +github.com/decred/dcrd/dcrec v1.0.1 h1:gDzlndw0zYxM5BlaV17d7ZJV6vhRe9njPBFeg4Db2UY= +github.com/decred/dcrd/dcrec v1.0.1/go.mod h1:CO+EJd8eHFb8WHa84C7ZBkXsNUIywaTHb+UAuI5uo6o= +github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3 h1:l/lhv2aJCUignzls81+wvga0TFlyoZx8QxRMQgXpZik= +github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3/go.mod h1:AKpV6+wZ2MfPRJnTbQ6NPgWrKzbe9RCIlCF/FKzMtM8= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/dcrjson/v4 v4.1.0 h1:WJVogRnYnNxB5hWoGHODvP4fNTG1JycTuHHKt/XucHk= +github.com/decred/dcrd/dcrjson/v4 v4.1.0/go.mod h1:2qVikafVF9/X3PngQVmqkbUbyAl32uik0k/kydgtqMc= +github.com/decred/dcrd/dcrutil/v4 v4.0.2 h1:eIl3E6gGln54qE8nk5o5lLtjh2/9C2Rz63OpD662h+8= +github.com/decred/dcrd/dcrutil/v4 v4.0.2/go.mod h1:iS3JB1ac3R3FgfpTF1kBD+SPNet8TmiW3Br+/Jc5MC8= +github.com/decred/dcrd/gcs/v4 v4.1.0 h1:tpW7JW53yJZlgNwl/n2NL1b8NxHaIPRUyNuLMkB/Hks= +github.com/decred/dcrd/gcs/v4 v4.1.0/go.mod h1:nPTbGM/I3Ihe5KFvUmxZEqQP/jDZQjQ63+WEi/f4lqU= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/decred/dcrd/lru v1.1.1 h1:kWFDaW0OWx6AD6Ki342c+JPmHbiVdE6rK81pT3fuo/Y= github.com/decred/dcrd/lru v1.1.1/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/decred/dcrd/lru v1.1.3 h1:w9EAbvGLyzm6jTjF83UKuqZEiUtJmvRhQDOCEIvSuE0= +github.com/decred/dcrd/lru v1.1.3/go.mod h1:Tw0i0pJyiLEx/oZdHLe1Wdv/Y7EGzAX+sYftnmxBR4o= +github.com/decred/dcrd/math/uint256 v1.0.2 h1:o8peafL5QmuXGTergI3YDpDU0eq5Z0pQi88B8ym4PRA= +github.com/decred/dcrd/math/uint256 v1.0.2/go.mod h1:7M/y9wJJvlyNG/f/X6mxxhxo9dgloZHFiOfbiscl75A= +github.com/decred/dcrd/mixing v0.4.1 h1:W8ZCzhmNyzG1xjJMA3L6FOElmp98Ttnk3dDUxD6irAE= +github.com/decred/dcrd/mixing v0.4.1/go.mod h1:ySvVwTZyVz5YvevA6YjPrB6pJEwTm7IkHohTfaiHh2c= +github.com/decred/dcrd/peer/v3 v3.1.2 h1:Qe7SpqDtfM0HARmDYwr4WjUu16X6HQ7ZWNnHqE1swiw= +github.com/decred/dcrd/peer/v3 v3.1.2/go.mod h1:M9FxNkHuEBtsRW5gwzIH4cJTWk5xSkxy9zG+TEL1N2Y= +github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0 h1:l0DnCcILTNrpy8APF3FLN312ChpkQaAuW30aC/RgBaw= +github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0/go.mod h1:j+kkRPXPJB5S9VFOsx8SQLcU7PTFkPKRc1aCHN4ENzA= +github.com/decred/dcrd/txscript/v4 v4.1.1 h1:R4M2+jMujgQA91899SkL0cW66d6DC76Gx+1W1oEHjc0= +github.com/decred/dcrd/txscript/v4 v4.1.1/go.mod h1:7ybmJoI+b6dxvQ+0aXdZpkyrj0PbnylJCzFxD1g8+/A= +github.com/decred/dcrd/wire v1.7.0 h1:5JHiDjEQeS4XUl4PfnTZYLwAD/E/+LwBmPRec/fP76o= +github.com/decred/dcrd/wire v1.7.0/go.mod h1:lAqrzV0SU4kyV6INLEJgDtUjJaTaVKrbF4LHtaYl+zU= +github.com/decred/go-socks v1.1.0 h1:dnENcc0KIqQo3HSXdgboXAHgqsCIutkqq6ntQjYtm2U= +github.com/decred/go-socks v1.1.0/go.mod h1:sDhHqkZH0X4JjSa02oYOGhcGHYp12FsY1jQ/meV8md0= +github.com/decred/slog v1.2.0 h1:soHAxV52B54Di3WtKLfPum9OFfWqwtf/ygf9njdfnPM= +github.com/decred/slog v1.2.0/go.mod h1:kVXlGnt6DHy2fV5OjSeuvCJ0OmlmTF6LFpEPMu/fOY0= github.com/deso-protocol/go-deadlock v1.0.0 h1:mw0pHy/19zgC+JFBStuQt1+1Ehv5OKA5NxXqecnL5ic= github.com/deso-protocol/go-deadlock v1.0.0/go.mod h1:K0Wd2OV2x7ck7SMYDraWerpKjFKUeBqaFcwz21tmkb8= github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+LsH/kRdL5l/Yzvg= @@ -210,6 +280,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= @@ -242,6 +314,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -285,11 +359,17 @@ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= +github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jrick/bitset v1.0.0 h1:Ws0PXV3PwXqWK2n7Vz6idCdrV/9OrBXgHEJi27ZB9Dw= +github.com/jrick/bitset v1.0.0/go.mod h1:ZOYB5Uvkla7wIEY4FEssPVi3IQXa02arznRaYaAEPe4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/jrick/logrotate v1.1.2 h1:6ePk462NCX7TfKtNp5JJ7MbA2YIslkpfgP03TlTYMN0= +github.com/jrick/logrotate v1.1.2/go.mod h1:f9tdWggSVK3iqavGpyvegq5IhNois7KXmasU6/N96OQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -306,6 +386,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -477,6 +559,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -546,6 +630,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -609,6 +695,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -622,6 +710,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -664,6 +754,8 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -671,6 +763,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= @@ -704,6 +798,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -796,6 +891,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= +lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w= mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= diff --git a/lib/blockchain.go b/lib/blockchain.go index e4796ec54..a03b55e39 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -18,6 +18,8 @@ import ( "github.com/decred/dcrd/lru" + lru2 "github.com/decred/dcrd/container/lru" + "github.com/deso-protocol/core/collections" "github.com/google/uuid" @@ -527,6 +529,115 @@ type CheckpointBlockInfoAndError struct { Error error } +type BlockIndex struct { + db *badger.DB + snapshot *Snapshot + blockIndexByHash *lru2.Map[BlockHash, *BlockNode] + tipNode *BlockNode +} + +func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { + return &BlockIndex{ + db: db, + snapshot: snapshot, + blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + tipNode: tipNode, + } +} + +func newBlockIndexByHashFromMap(input map[BlockHash]*BlockNode) *lru2.Map[BlockHash, *BlockNode] { + newMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + for key, val := range input { + newMap.Put(key, val) + } + return newMap +} + +func (bi *BlockIndex) setTipNode(tipNode *BlockNode) { + bi.tipNode = tipNode +} + +func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { + bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) +} + +func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, bool, error) { + val, exists := bi.blockIndexByHash.Get(*blockHash) + if exists { + return val, true, nil + } + // NOTE: this can be sped up significantly by mapping blockHash to blockNode or blockHash to blockHeight + // in badger, so we can avoid deserializing the full block. + fullBlock, err := GetBlock(blockHash, bi.db, bi.snapshot) + if err != nil { + return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem reading block from db") + } + nodeHeader := fullBlock.Header + fullBlockHash, err := fullBlock.Hash() + if err != nil { + return nil, false, errors.Wrapf(err, "_initChain: Problem hashing block") + } + blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(nodeHeader.Height), fullBlockHash, false) + if blockNode == nil { + return nil, false, nil + } + bi.addNewBlockNodeToBlockIndex(blockNode) + return blockNode, true, nil +} + +func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { + val, exists := bi.blockIndexByHash.Get(*blockHash) + if exists { + return val, true + } + if height > math.MaxUint32 { + glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) + } + bn := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) + if bn == nil { + return nil, false + } + bi.addNewBlockNodeToBlockIndex(bn) + return bn, true +} + +func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { + if height > math.MaxUint32 { + glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) + } + prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) + _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false) + blockNodes := []*BlockNode{} + for _, val := range valsFound { + blockNode, err := DeserializeBlockNode(val) + if err != nil { + glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) + continue + } + bi.addNewBlockNodeToBlockIndex(blockNode) + blockNodes = append(blockNodes, blockNode) + } + return blockNodes +} + +type BestChain struct { + db *badger.DB + IsHeaderChain bool + Chain []*BlockNode + ChainMap map[BlockHash]*BlockNode + tipNode *BlockNode +} + +func NewBestChain(db *badger.DB, isHeaderChain bool, tipNode *BlockNode) *BestChain { + return &BestChain{ + db: db, + IsHeaderChain: isHeaderChain, + Chain: []*BlockNode{}, + ChainMap: make(map[BlockHash]*BlockNode), + tipNode: tipNode, + } +} + type Blockchain struct { db *badger.DB postgres *Postgres @@ -556,10 +667,10 @@ type Blockchain struct { // // An in-memory index of the "tree" of blocks we are currently aware of. // This index includes forks and side-chains. - blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode] - // blockIndexByHeight is an in-memory map of block height to block nodes. This is - // used to quickly find the safe blocks from which the chain can be extended for PoS - blockIndexByHeight map[uint64]map[BlockHash]*BlockNode + //blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode] + //// blockIndexByHeight is an in-memory map of block height to block nodes. This is + //// used to quickly find the safe blocks from which the chain can be extended for PoS + //blockIndexByHeight map[uint64]map[BlockHash]*BlockNode // An in-memory slice of the blocks on the main chain only. The end of // this slice is the best known tip that we have at any given time. bestChain []*BlockNode @@ -568,6 +679,10 @@ type Blockchain struct { bestHeaderChain []*BlockNode bestHeaderChainMap map[BlockHash]*BlockNode + blockIndex *BlockIndex + //bestChain *BestChain + //bestHeaderChain *BestChain + // We keep track of orphan blocks with the following data structures. Orphans // are not written to disk and are only cached in memory. Moreover we only keep // up to MaxOrphansInMemory of them in order to prevent memory exhaustion. @@ -705,52 +820,33 @@ func getCheckpointBlockInfoFromProviderHelper(provider string) *CheckpointBlockI } func (bc *Blockchain) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { - bc.blockIndexByHash.Set(*blockNode.Hash, blockNode) - if _, exists := bc.blockIndexByHeight[uint64(blockNode.Height)]; !exists { - bc.blockIndexByHeight[uint64(blockNode.Height)] = make(map[BlockHash]*BlockNode) - } - bc.blockIndexByHeight[uint64(blockNode.Height)][*blockNode.Hash] = blockNode + bc.blockIndex.blockIndexByHash.Put(*blockNode.Hash, blockNode) } func (bc *Blockchain) CopyBlockIndexes() ( - _blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode], - _blockIndexByHeight map[uint64]map[BlockHash]*BlockNode, + _blockIndexByHash *lru2.Map[BlockHash, *BlockNode], ) { - newBlockIndexByHash := collections.NewConcurrentMap[BlockHash, *BlockNode]() - newBlockIndexByHeight := make(map[uint64]map[BlockHash]*BlockNode) - bc.blockIndexByHash.Iterate(func(kk BlockHash, vv *BlockNode) { - newBlockIndexByHash.Set(kk, vv) - blockHeight := uint64(vv.Height) - if _, exists := newBlockIndexByHeight[blockHeight]; !exists { - newBlockIndexByHeight[blockHeight] = make(map[BlockHash]*BlockNode) - } - newBlockIndexByHeight[blockHeight][kk] = vv - }) - return newBlockIndexByHash, newBlockIndexByHeight -} - -func (bc *Blockchain) constructBlockIndexByHeight() map[uint64]map[BlockHash]*BlockNode { - newBlockIndex := make(map[uint64]map[BlockHash]*BlockNode) - bc.blockIndexByHash.Iterate(func(_ BlockHash, blockNode *BlockNode) { - blockHeight := uint64(blockNode.Height) - if _, exists := newBlockIndex[blockHeight]; !exists { - newBlockIndex[blockHeight] = make(map[BlockHash]*BlockNode) - } - newBlockIndex[blockHeight][*blockNode.Hash] = blockNode - }) - return newBlockIndex + newBlockIndexByHash := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + for _, key := range bc.blockIndex.blockIndexByHash.Keys() { + val, _ := bc.blockIndex.blockIndexByHash.Get(key) + newBlockIndexByHash.Put(key, val) + } + return newBlockIndexByHash } +// TODO: read through to DB. func (bc *Blockchain) getAllBlockNodesIndexedAtHeight(blockHeight uint64) []*BlockNode { - return collections.MapValues(bc.blockIndexByHeight[blockHeight]) + return bc.blockIndex.GetBlockNodesByHeight(blockHeight) } func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { - blocksAtHeight, hasNestedMapAtHeight := bc.blockIndexByHeight[blockHeight] - if !hasNestedMapAtHeight { - return false - } - return len(blocksAtHeight) > 0 + //blocksAtHeight, hasNestedMapAtHeight := bc.blockIndexByHeight[blockHeight] + //if !hasNestedMapAtHeight { + // return false + //} + //return len(blocksAtHeight) > 0 + blockNodes := bc.blockIndex.GetBlockNodesByHeight(blockHeight) + return len(blockNodes) > 0 } func (bc *Blockchain) CopyBestChain() ([]*BlockNode, map[BlockHash]*BlockNode) { @@ -840,26 +936,48 @@ func (bc *Blockchain) _initChain() error { // to previous blocks we've read in and error if they don't. This works because // reading blocks in height order as we do here ensures that we'll always // add a block's parents, if they exist, before adding the block itself. + //var err error + //if bc.postgres != nil { + // bc.blockIndexByHash, err = bc.postgres.GetBlockIndex() + //} else { + // bc.blockIndexByHash, err = GetBlockIndex(bc.db, false /*bitcoinNodes*/, bc.params) + //} + //if err != nil { + // return errors.Wrapf(err, "_initChain: Problem reading block index from db") + //} + //bc.blockIndexByHeight = bc.constructBlockIndexByHeight() + + // For postgres, we still load the entire block index into memory. This is because var err error + var tipNode *BlockNode if bc.postgres != nil { - bc.blockIndexByHash, err = bc.postgres.GetBlockIndex() + bc.blockIndex.blockIndexByHash, err = bc.postgres.GetBlockIndex() + var exists bool + tipNode, exists = bc.blockIndex.blockIndexByHash.Get(*bestBlockHash) + if !exists { + return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) + } } else { - bc.blockIndexByHash, err = GetBlockIndex(bc.db, false /*bitcoinNodes*/, bc.params) - } - if err != nil { - return errors.Wrapf(err, "_initChain: Problem reading block index from db") + var tipNodeExists bool + // For badger, we only need the tip block to get started. + // Weird hack required for the genesis block. + if bestBlockHash.IsEqual(GenesisBlockHash) { + tipNode, tipNodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(bestBlockHash, 0) + } else { + tipNode, tipNodeExists, err = bc.blockIndex.GetBlockNodeByHashOnly(bestBlockHash) + if err != nil { + return errors.Wrapf(err, "_initChain: Problem reading best block from db") + } + if !tipNodeExists { + return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) + } + } } - bc.blockIndexByHeight = bc.constructBlockIndexByHeight() + bc.blockIndex.setTipNode(tipNode) // At this point the blockIndexByHash should contain a full node tree with all // nodes pointing to valid parent nodes. { - // Find the tip node with the best node hash. - tipNode, exists := bc.blockIndexByHash.Get(*bestBlockHash) - if !exists { - return fmt.Errorf("_initChain(block): Best hash (%#v) not found in block index", bestBlockHash) - } - // Walk back from the best node to the genesis block and store them all // in bestChain. bc.bestChain, err = GetBestChain(tipNode) @@ -873,12 +991,6 @@ func (bc *Blockchain) _initChain() error { // TODO: This code is a bit repetitive but this seemed clearer than factoring it out. { - // Find the tip node with the best node hash. - tipNode, exists := bc.blockIndexByHash.Get(*bestHeaderHash) - if !exists { - return fmt.Errorf("_initChain(header): Best hash (%#v) not found in block index", bestHeaderHash) - } - // Walk back from the best node to the genesis block and store them all // in bestChain. bc.bestHeaderChain, err = GetBestChain(tipNode) @@ -992,12 +1104,13 @@ func NewBlockchain( eventManager: eventManager, archivalMode: archivalMode, - blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), - blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), - bestChainMap: make(map[BlockHash]*BlockNode), - + blockIndex: NewBlockIndex(db, snapshot, nil), + //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), + //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), + bestChainMap: make(map[BlockHash]*BlockNode), + //bestChain: NewBestChain(db, false), bestHeaderChainMap: make(map[BlockHash]*BlockNode), - + //bestHeaderChain: NewBestChain(db, true), blockViewCache: lru.NewKVCache(100), // TODO: parameterize snapshotCache: NewSnapshotCache(), @@ -1065,14 +1178,16 @@ func fastLog2Floor(n uint32) uint8 { // // This function MUST be called with the chain state lock held (for reads). func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32, - blockIndex *collections.ConcurrentMap[BlockHash, *BlockNode], bestChainList []*BlockNode, + blockIndex *BlockIndex, bestChainList []*BlockNode, bestChainMap map[BlockHash]*BlockNode) (*BlockNode, uint32) { // There are no block locators so a specific block is being requested // as identified by the stop hash. - stopNode, stopNodeExists := blockIndex.Get(*stopHash) + stopNode, stopNodeExists, stopNodeError := blockIndex.GetBlockNodeByHashOnly(stopHash) if len(locator) == 0 { - if !stopNodeExists { + if stopNodeError != nil || !stopNodeExists || stopNode == nil { + // TODO: what should we really do here? + glog.Errorf("locateInventory: Block %v is not known", stopHash) // No blocks with the stop hash were found so there is // nothing to do. return nil, 0 @@ -1104,7 +1219,8 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // Calculate how many entries are needed. tip := bestChainList[len(bestChainList)-1] total := uint32((tip.Header.Height - startNode.Header.Height) + 1) - if stopNodeExists && stopNode.Header.Height >= startNode.Header.Height { + if stopNodeError != nil && stopNodeExists && stopNode != nil && + stopNode.Header.Height >= startNode.Header.Height { _, bestChainContainsStopNode := bestChainMap[*stopNode.Hash] if bestChainContainsStopNode { @@ -1126,7 +1242,7 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // // This function MUST be called with the ChainLock held (for reads). func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, - blockIndex *collections.ConcurrentMap[BlockHash, *BlockNode], bestChainList []*BlockNode, + blockIndex *BlockIndex, bestChainList []*BlockNode, bestChainMap map[BlockHash]*BlockNode) []*MsgDeSoHeader { // Find the node after the first known block in the locator and the @@ -1177,7 +1293,7 @@ func (bc *Blockchain) LocateBestBlockChainHeaders( // where it's currently called is single-threaded via a channel in server.go. Going to // avoid messing with it for now. headers := locateHeaders(locator, stopHash, maxHeaders, - bc.blockIndexByHash, bc.bestChain, bc.bestChainMap) + bc.blockIndex, bc.bestChain, bc.bestChainMap) return headers } @@ -1256,8 +1372,11 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { } func (bc *Blockchain) HeaderLocatorWithNodeHash(blockHash *BlockHash) ([]*BlockHash, error) { - node, exists := bc.blockIndexByHash.Get(*blockHash) - if !exists { + node, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) + if err != nil { + return nil, fmt.Errorf("Blockchain.HeaderLocatorWithNodeHash: Problem getting node for hash %v: %v", blockHash, err) + } + if !exists || node == nil { return nil, fmt.Errorf("Blockchain.HeaderLocatorWithNodeHash: Node for hash %v is not in our blockIndexByHash", blockHash) } @@ -1336,9 +1455,9 @@ func (bc *Blockchain) GetBlockNodesToFetch( return blockNodesToFetch } -func (bc *Blockchain) HasHeader(headerHash *BlockHash) bool { - _, exists := bc.blockIndexByHash.Get(*headerHash) - return exists +func (bc *Blockchain) HasHeader(headerHash *BlockHash) (bool, error) { + _, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(headerHash) + return exists, errors.Wrap(err, "Blockchain.HasHeader: ") } func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) *BlockNode { @@ -1349,42 +1468,12 @@ func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) *BlockNode { return bc.bestHeaderChain[blockHeight] } -func (bc *Blockchain) HasBlock(blockHash *BlockHash) bool { - node, nodeExists := bc.blockIndexByHash.Get(*blockHash) - if !nodeExists { - glog.V(2).Infof("Blockchain.HasBlock: Node with hash %v does not exist in node index", blockHash) - return false - } - - if (node.Status & StatusBlockProcessed) == 0 { - glog.V(2).Infof("Blockchain.HasBlock: Node %v does not have StatusBlockProcessed so we don't have the block", node) - return false - } - - // Node exists with StatusBlockProcess set means we have it. - return true -} - -func (bc *Blockchain) HasBlockInBlockIndex(blockHash *BlockHash) bool { +func (bc *Blockchain) HasBlockInBlockIndex(blockHash *BlockHash) (bool, error) { bc.ChainLock.RLock() defer bc.ChainLock.RUnlock() - _, exists := bc.blockIndexByHash.Get(*blockHash) - return exists -} - -// This needs to hold a lock on the blockchain because it read from an in-memory map that is -// not thread-safe. -func (bc *Blockchain) GetBlockHeaderFromIndex(blockHash *BlockHash) *MsgDeSoHeader { - bc.ChainLock.RLock() - defer bc.ChainLock.RUnlock() - - block, blockExists := bc.blockIndexByHash.Get(*blockHash) - if !blockExists { - return nil - } - - return block.Header + _, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) + return exists, errors.Wrap(err, "Blockchain.HasBlockInBlockIndex: ") } // Don't need a lock because blocks don't get removed from the db after they're added @@ -1690,13 +1779,11 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { func (bc *Blockchain) SetBestChainMap( bestChain []*BlockNode, bestChainMap map[BlockHash]*BlockNode, - blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode], - blockIndexByHeight map[uint64]map[BlockHash]*BlockNode, + blockIndexByHash *lru2.Map[BlockHash, *BlockNode], ) { bc.bestChain = bestChain bc.bestChainMap = bestChainMap - bc.blockIndexByHash = blockIndexByHash - bc.blockIndexByHeight = blockIndexByHeight + bc.blockIndex.blockIndexByHash = blockIndexByHash } func (bc *Blockchain) _validateOrphanBlockPoW(desoBlock *MsgDeSoBlock) error { @@ -2025,7 +2112,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // index. If it does, then return an error. We should generally // expect that processHeaderPoW will only be called on headers we // haven't seen before. - _, nodeExists := bc.blockIndexByHash.Get(*headerHash) + _, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, blockHeader.Height) if nodeExists { return false, false, HeaderErrorDuplicateHeader } @@ -2050,7 +2137,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if blockHeader.PrevBlockHash == nil { return false, false, HeaderErrorNilPrevHash } - parentNode, parentNodeExists := bc.blockIndexByHash.Get(*blockHeader.PrevBlockHash) + parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) if !parentNodeExists { // This block is an orphan if its parent doesn't exist and we don't // process unconnectedTxns. @@ -2167,9 +2254,8 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if bc.isSyncing() { bc.addNewBlockNodeToBlockIndex(newNode) } else { - newBlockIndexByHash, newBlockIndexByHeight := bc.CopyBlockIndexes() - bc.blockIndexByHash = newBlockIndexByHash - bc.blockIndexByHeight = newBlockIndexByHeight + newBlockIndexByHash := bc.CopyBlockIndexes() + bc.blockIndex.blockIndexByHash = newBlockIndexByHash bc.addNewBlockNodeToBlockIndex(newNode) } @@ -2312,7 +2398,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.Start("Blockchain.ProcessBlock: BlockNode") // See if a node for the block exists in our node index. - nodeToValidate, nodeExists := bc.blockIndexByHash.Get(*blockHash) + // TODO: validate that current height - 1 > 0 + nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set // the header validation status for it. @@ -2333,7 +2420,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Reset the pointers after having presumably added the header to the // block index. - nodeToValidate, nodeExists = bc.blockIndexByHash.Get(*blockHash) + // TODO: validate that current height - 1 > 0 + nodeToValidate, nodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) } // At this point if the node still doesn't exist or if the header's validation // failed then we should return an error for the block. Note that at this point @@ -2352,7 +2440,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // In this case go ahead and return early. If its parents are truly legitimate then we // should re-request it and its parents from a node and reprocess it // once it is no longer an orphan. - parentNode, parentNodeExists := bc.blockIndexByHash.Get(*blockHeader.PrevBlockHash) + // TODO: validate that current height - 1 > 0 + parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { return false, true, nil } diff --git a/lib/db_utils.go b/lib/db_utils.go index c51609681..2bb61e0f7 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5188,6 +5188,13 @@ func _heightHashToNodeIndexPrefix(bitcoinNodes bool) []byte { return prefix } +func _heightHashToNodePrefixByHeight(height uint32, bitcoinNodes bool) []byte { + prefix := _heightHashToNodeIndexPrefix(bitcoinNodes) + heightBytes := make([]byte, 4) + binary.BigEndian.PutUint32(heightBytes[:], height) + return append(prefix, heightBytes[:]...) +} + func _heightHashToNodeIndexKey(height uint32, hash *BlockHash, bitcoinNodes bool) []byte { prefix := _heightHashToNodeIndexPrefix(bitcoinNodes) diff --git a/lib/peer.go b/lib/peer.go index 9fda1bc7b..aece3eeff 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -330,7 +330,12 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { } else if invVect.Type == InvTypeBlock { // For blocks, we check that the hash isn't known to us either in our // main header chain or in side chains. - if pp.srv.blockchain.HasHeader(¤tHash) { + exists, err := pp.srv.blockchain.HasHeader(¤tHash) + if exists { + continue + } + if err != nil { + glog.Errorf("Server._handleInv: Error checking if block exists: %v", err) continue } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index bede52036..ff433b196 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -110,7 +110,8 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // process headers. func (bc *Blockchain) healPointersForOrphanChildren(blockNode *BlockNode) { // Fetch all potential children of this blockNode from the block index. - blockNodesAtNextHeight, exists := bc.blockIndexByHeight[blockNode.Header.Height+1] + blockNodesAtNextHeight := bc.blockIndex.GetBlockNodesByHeight(blockNode.Header.Height + 1) + exists := len(blockNodesAtNextHeight) > 0 if !exists { // No children of this blockNode exist in the block index. Exit early. return @@ -137,8 +138,7 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas _headerBlockNode *BlockNode, _isOrphan bool, _err error, ) { // Look up the header in the block index to check if it has already been validated and indexed. - blockNode, blockNodeExists := bc.blockIndexByHash.Get(*headerHash) - + blockNode, blockNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, header.Height) // ------------------------------------ Base Cases ----------------------------------- // // The header is already validated. Exit early. @@ -157,7 +157,8 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas } // The header is an orphan. No need to store it in the block index. Exit early. - parentBlockNode, parentBlockNodeExists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + // TODO: validate that height - 1 > 0 + parentBlockNode, parentBlockNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !parentBlockNodeExists { return nil, true, nil } @@ -310,7 +311,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // We expect the utxoView for the parent block to be valid because we check that all ancestor blocks have // been validated. - parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash) + parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) if err != nil { // This should never happen. If the parent is validated and extends from the tip, then we should // be able to build a UtxoView for it. This failure can only happen due to transient or badger issues. @@ -401,7 +402,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // Now that we've processed this block, we check for any blocks that were previously // stored as orphans, which are children of this block. We can process them now. - blockNodesAtNextHeight := bc.blockIndexByHeight[uint64(blockNode.Height)+1] + blockNodesAtNextHeight := bc.blockIndex.GetBlockNodesByHeight(uint64(blockNode.Height) + 1) for _, blockNodeAtNextHeight := range blockNodesAtNextHeight { if blockNodeAtNextHeight.Header.PrevBlockHash.IsEqual(blockNode.Hash) && blockNodeAtNextHeight.IsStored() && @@ -569,7 +570,7 @@ func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock) (_success if err != nil { return false, errors.Wrap(err, "checkAndStoreArchivalBlock: Problem hashing block") } - blockNode, exists := bc.blockIndexByHash.Get(*blockHash) + blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, block.Header.Height) // If the blockNode doesn't exist, or the block is not committed, or it's already stored, then we're not dealing // with an archival block. Archival blocks must have an existing blockNode, be committed, and not be stored. if !exists || !blockNode.IsCommitted() || blockNode.IsStored() { @@ -687,7 +688,8 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi } // Base case - Check if the block is validated or validate failed. If so, we can return early. - blockNode, exists := bc.blockIndexByHash.Get(*blockHash) + // TODO: validate height doesn't overflow uint32 + blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, block.Header.Height) if exists && (blockNode.IsValidateFailed() || blockNode.IsValidated()) { return blockNode, nil } @@ -708,7 +710,7 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi // Run the validation for the parent and update the block index with the parent's status. We first // check if the parent has a cached status. If so, we use the cached status. Otherwise, we run // the full validation algorithm on it, then index it and use the result. - parentBlockNode, err := bc.validatePreviouslyIndexedBlockPoS(block.Header.PrevBlockHash, verifySignatures) + parentBlockNode, err := bc.validatePreviouslyIndexedBlockPoS(block.Header.PrevBlockHash, block.Header.Height-1, verifySignatures) if err != nil { return blockNode, errors.Wrapf(err, "validateAndIndexBlockPoS: Problem validating previously indexed block: ") } @@ -789,10 +791,11 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi // cached block, and runs the validateAndIndexBlockPoS algorithm on it. It returns the resulting BlockNode. func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( blockHash *BlockHash, + blockHeight uint64, verifySignatures bool, ) (*BlockNode, error) { // Check if the block is already in the block index. If so, we check its current status first. - blockNode, exists := bc.blockIndexByHash.Get(*blockHash) + blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeight) if !exists { // We should never really hit this if the block has already been cached in the block index first. // We check here anyway to be safe. @@ -814,7 +817,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( return nil, errors.Wrapf(err, "validatePreviouslyIndexedBlockPoS: Problem fetching block from DB") } // Build utxoView for the block's parent. - parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash) + parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) if err != nil { // This should never happen. If the parent is validated and extends from the tip, then we should // be able to build a UtxoView for it. This failure can only happen due to transient or badger issues. @@ -890,7 +893,7 @@ func (bc *Blockchain) isValidBlockHeaderPoS(header *MsgDeSoHeader) error { // greater than its parent's timestamp. func (bc *Blockchain) isBlockTimestampValidRelativeToParentPoS(header *MsgDeSoHeader) error { // Validate that the timestamp is not less than its parent. - parentBlockNode, exists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + parentBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !exists { // Note: this should never happen as we only call this function after // we've validated that all ancestors exist in the block index. @@ -1050,7 +1053,7 @@ func (bc *Blockchain) hasValidBlockHeightPoS(header *MsgDeSoHeader) error { return RuleErrorPoSBlockBeforeCutoverHeight } // Validate that the block height is exactly one greater than its parent. - parentBlockNode, exists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + parentBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !exists { // Note: this should never happen as we only call this function after // we've validated that all ancestors exist in the block index. @@ -1065,7 +1068,7 @@ func (bc *Blockchain) hasValidBlockHeightPoS(header *MsgDeSoHeader) error { // hasValidBlockViewPoS validates the view for a given block header func (bc *Blockchain) hasValidBlockViewPoS(header *MsgDeSoHeader) error { // Validate that the view is greater than the latest uncommitted block. - parentBlockNode, exists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + parentBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !exists { // Note: this should never happen as we only call this function after // we've validated that all ancestors exist in the block index. @@ -1094,7 +1097,7 @@ func (bc *Blockchain) hasValidBlockViewPoS(header *MsgDeSoHeader) error { func (bc *Blockchain) hasValidProposerRandomSeedSignaturePoS(header *MsgDeSoHeader) (bool, error) { // Validate that the leader proposed a valid random seed signature. - parentBlock, exists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + parentBlock, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !exists { // Note: this should never happen as we only call this function after // we've validated that all ancestors exist in the block index. @@ -1319,11 +1322,13 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( return nil, nil, errors.New("getStoredLineageFromCommittedTip: No committed blocks found") } currentHash := header.PrevBlockHash.NewBlockHash() + currentHeight := header.Height - 1 ancestors := []*BlockNode{} prevHeight := header.Height prevView := header.GetView() for { - currentBlock, exists := bc.blockIndexByHash.Get(*currentHash) + // TODO: is currentHeight correct here? + currentBlock, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(currentHash, currentHeight) if !exists { return nil, []*BlockHash{currentHash}, RuleErrorMissingAncestorBlock } @@ -1366,8 +1371,8 @@ func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock) (* if err != nil { return nil, errors.Wrapf(err, "getOrCreateBlockNodeFromBlockIndex: Problem hashing block %v", block) } - blockNode, _ := bc.blockIndexByHash.Get(*hash) - prevBlockNode, _ := bc.blockIndexByHash.Get(*block.Header.PrevBlockHash) + blockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(hash, block.Header.Height) + prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) if blockNode != nil { // If the block node already exists, we should set its parent if it doesn't have one already. if blockNode.Parent == nil { @@ -1676,7 +1681,7 @@ func (bc *Blockchain) runCommitRuleOnBestChain(verifySignatures bool) error { } } for ii := 0; ii < len(uncommittedAncestors); ii++ { - if err := bc.commitBlockPoS(uncommittedAncestors[ii].Hash, verifySignatures); err != nil { + if err := bc.commitBlockPoS(uncommittedAncestors[ii].Hash, uint64(uncommittedAncestors[ii].Height), verifySignatures); err != nil { return errors.Wrapf(err, "runCommitRuleOnBestChain: Problem committing block %v", uncommittedAncestors[ii].Hash.String()) } @@ -1708,7 +1713,7 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen // commitBlockPoS commits the block with the given hash. Specifically, this updates the // BlockStatus to include StatusBlockCommitted and flushes the view after connecting the block // to the DB and updates relevant badger indexes with info about the block. -func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, verifySignatures bool) error { +func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, blockHeight uint64, verifySignatures bool) error { // block must be in the best chain. we grab the block node from there. blockNode, exists := bc.bestChainMap[*blockHash] if !exists { @@ -1720,7 +1725,7 @@ func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, verifySignatures bool return errors.Errorf("commitBlockPoS: Block %v is already committed", blockHash.String()) } // Connect a view up to block we are committing. - utxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockHash) + utxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockHash, uint64(blockNode.Height)) if err != nil { return errors.Wrapf(err, "commitBlockPoS: Problem initializing UtxoView: ") } @@ -1855,9 +1860,9 @@ func (bc *Blockchain) GetUncommittedBlocks(tipHash *BlockHash) ([]*BlockNode, er if currentParentHash == nil { return nil, errors.Errorf("GetUncommittedBlocks: Block %v has nil PrevBlockHash", currentBlock.Hash) } - currentBlock, _ = bc.blockIndexByHash.Get(*currentParentHash) + currentBlock, _ = bc.blockIndex.GetBlockNodeByHashAndHeight(currentParentHash, currentBlock.Header.Height-1) if currentBlock == nil { - return nil, errors.Errorf("GetUncommittedBlocks: Block %v not found in block index", currentBlock.Hash) + return nil, errors.Errorf("GetUncommittedBlocks: Block %v not found in block index", currentParentHash) } } return collections.Reverse(uncommittedBlockNodes), nil @@ -1892,7 +1897,8 @@ func (viewAndUtxoOps *BlockViewAndUtxoOps) Copy() *BlockViewAndUtxoOps { // GetUncommittedTipView builds a UtxoView to the uncommitted tip. func (bc *Blockchain) GetUncommittedTipView() (*UtxoView, error) { // Connect the uncommitted blocks to the tip so that we can validate subsequent blocks - blockViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*bc.BlockTip().Hash) + blockTip := bc.BlockTip() + blockViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockTip.Hash, uint64(blockTip.Height)) if err != nil { return nil, errors.Wrapf(err, "GetUncommittedTipView: Problem getting UtxoView at block hash") } @@ -1917,12 +1923,12 @@ func (bc *Blockchain) getCachedBlockViewAndUtxoOps(blockHash BlockHash) (*BlockV // all uncommitted ancestors of this block. Then it checks the block view cache to see if we have already // computed this view. If not, connecting the uncommitted ancestor blocks and saving to the cache. The // returned UtxoOps and FullBlock should NOT be modified. -func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash) ( +func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, blockHeight uint64) ( *BlockViewAndUtxoOps, error) { // Always fetch the lineage from the committed tip to the block provided first to // ensure that a valid UtxoView is returned. uncommittedAncestors := []*BlockNode{} - currentBlock, _ := bc.blockIndexByHash.Get(blockHash) + currentBlock, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(&blockHash, blockHeight) if currentBlock == nil { return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: Block %v not found in block index", blockHash) } @@ -1945,7 +1951,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash) ( if currentParentHash == nil { return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: Block %v has nil PrevBlockHash", currentBlock.Hash) } - currentBlock, _ = bc.blockIndexByHash.Get(*currentParentHash) + currentBlock, _ = bc.blockIndex.GetBlockNodeByHashAndHeight(currentParentHash, currentBlock.Header.Height-1) if currentBlock == nil { return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: Block %v not found in block index", currentParentHash) } @@ -2108,8 +2114,8 @@ func (bc *Blockchain) GetProofOfStakeGenesisQuorumCertificate() (*QuorumCertific func (bc *Blockchain) GetFinalCommittedPoWBlock() (*BlockNode, error) { // Fetch the block node for the cutover block - blockNodes, blockNodesExist := bc.blockIndexByHeight[bc.params.GetFinalPoWBlockHeight()] - if !blockNodesExist { + blockNodes := bc.blockIndex.GetBlockNodesByHeight(bc.params.GetFinalPoWBlockHeight()) + if len(blockNodes) == 0 { return nil, errors.Errorf("Error fetching cutover block nodes before height %d", bc.params.GetFinalPoWBlockHeight()) } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 52ea4d696..2bd140003 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -252,7 +252,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) bc.bestChain = []*BlockNode{genesisBlock} - bc.blockIndexByHash.Set(*genesisBlock.Hash, genesisBlock) + bc.blockIndex.blockIndexByHash.Put(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) randomBLSPrivateKey := _generateRandomBLSPrivateKey(t) @@ -301,7 +301,7 @@ func TestHasValidBlockHeight(t *testing.T) { require.Equal(t, err, RuleErrorInvalidPoSBlockHeight) block.Header.Height = 2 - bc.blockIndexByHash = collections.NewConcurrentMap[BlockHash, *BlockNode]() + bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot, nil) err = bc.hasValidBlockHeightPoS(block.Header) require.Equal(t, err, RuleErrorMissingParentBlock) } @@ -329,7 +329,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndexByHash = collections.NewConcurrentMapFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -377,17 +377,16 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { newHash, err := block.Hash() require.NoError(t, err) // Check the block index by hash - blockNodeFromIndex, exists := bc.blockIndexByHash.Get(*newHash) + blockNodeFromIndex, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(newHash, uint64(blockNode.Height)) require.True(t, exists) require.True(t, blockNodeFromIndex.Hash.IsEqual(blockNode.Hash)) require.Equal(t, blockNodeFromIndex.Height, uint32(2)) require.True(t, blockNodeFromIndex.IsStored()) require.False(t, blockNodeFromIndex.IsValidated()) // Check the block index by height - byHeightBlockNodes, exists := bc.blockIndexByHeight[2] - require.True(t, exists) + byHeightBlockNodes := bc.blockIndex.GetBlockNodesByHeight(2) require.Len(t, byHeightBlockNodes, 1) - require.True(t, byHeightBlockNodes[*newHash].Hash.IsEqual(newHash)) + require.True(t, byHeightBlockNodes[0].Hash.IsEqual(newHash)) require.True(t, bc.hasBlockNodesIndexedAtHeight(2)) require.Len(t, bc.getAllBlockNodesIndexedAtHeight(2), 1) // Check the DB for the block @@ -401,18 +400,17 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { // Okay now we update the status of the block to include validated. blockNode, err = bc.storeValidatedBlockInBlockIndex(block) require.NoError(t, err) - blockNodeFromIndex, exists = bc.blockIndexByHash.Get(*newHash) + blockNodeFromIndex, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(newHash, uncommittedBlock.Header.Height) require.True(t, exists) require.True(t, blockNodeFromIndex.Hash.IsEqual(blockNode.Hash)) require.Equal(t, blockNodeFromIndex.Height, uint32(2)) require.True(t, blockNodeFromIndex.IsStored()) require.True(t, blockNodeFromIndex.IsValidated()) // Check the block index by height. - byHeightBlockNodes, exists = bc.blockIndexByHeight[2] - require.True(t, exists) + byHeightBlockNodes = bc.blockIndex.GetBlockNodesByHeight(2) require.Len(t, byHeightBlockNodes, 1) - require.True(t, byHeightBlockNodes[*newHash].Hash.IsEqual(newHash)) - require.True(t, byHeightBlockNodes[*newHash].IsValidated()) + require.True(t, byHeightBlockNodes[0].Hash.IsEqual(newHash)) + require.True(t, byHeightBlockNodes[0].IsValidated()) require.True(t, bc.hasBlockNodesIndexedAtHeight(2)) require.Len(t, bc.getAllBlockNodesIndexedAtHeight(2), 1) @@ -428,18 +426,18 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { blockNode, err = bc.storeBlockInBlockIndex(block) require.NoError(t, err) // Make sure the blockIndexByHash is correct. - updatedBlockNode, exists := bc.blockIndexByHash.Get(*updatedBlockHash) + updatedBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(updatedBlockHash, uint64(blockNode.Height)) require.True(t, exists) require.True(t, updatedBlockNode.Hash.IsEqual(updatedBlockHash)) require.Equal(t, updatedBlockNode.Height, uint32(2)) require.True(t, updatedBlockNode.IsStored()) require.False(t, updatedBlockNode.IsValidated()) // Make sure the blockIndexByHeight is correct - byHeightBlockNodes, exists = bc.blockIndexByHeight[2] + byHeightBlockNodes = bc.blockIndex.GetBlockNodesByHeight(2) require.True(t, exists) require.Len(t, byHeightBlockNodes, 2) - require.True(t, byHeightBlockNodes[*newHash].Hash.IsEqual(newHash)) - require.True(t, byHeightBlockNodes[*updatedBlockHash].Hash.IsEqual(updatedBlockHash)) + require.True(t, byHeightBlockNodes[0].Hash.IsEqual(newHash) || byHeightBlockNodes[1].Hash.IsEqual(newHash)) + require.True(t, byHeightBlockNodes[0].Hash.IsEqual(updatedBlockHash) || byHeightBlockNodes[1].Hash.IsEqual(updatedBlockHash)) require.True(t, bc.hasBlockNodesIndexedAtHeight(2)) require.Len(t, bc.getAllBlockNodesIndexedAtHeight(2), 2) @@ -477,7 +475,7 @@ func TestHasValidBlockViewPoS(t *testing.T) { genesisNode, block2, } - bc.blockIndexByHash = collections.NewConcurrentMapFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -810,7 +808,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) bc.bestChain = []*BlockNode{genesisNode} - bc.blockIndexByHash = collections.NewConcurrentMapFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) block := &MsgDeSoBlock{ @@ -847,7 +845,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) bc.bestChain = append(bc.bestChain, block2) - bc.blockIndexByHash.Set(*hash2, block2) + bc.blockIndex.blockIndexByHash.Put(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) require.Equal(t, err, RuleErrorDoesNotExtendCommittedTip) @@ -1319,9 +1317,9 @@ func TestTryApplyNewTip(t *testing.T) { bc.addTipBlockToBestChain(bn1) bc.addTipBlockToBestChain(bn2) bc.addTipBlockToBestChain(bn3) - bc.blockIndexByHash.Set(*hash1, bn1) - bc.blockIndexByHash.Set(*hash2, bn2) - bc.blockIndexByHash.Set(*hash3, bn3) + bc.blockIndex.blockIndexByHash.Put(*hash1, bn1) + bc.blockIndex.blockIndexByHash.Put(*hash2, bn2) + bc.blockIndex.blockIndexByHash.Put(*hash3, bn3) // Simple reorg. Just replacing the uncommitted tip. newBlock := &MsgDeSoBlock{ @@ -1403,8 +1401,8 @@ func TestTryApplyNewTip(t *testing.T) { Height: 6, }, } - bc.blockIndexByHash.Set(*hash4, bn4) - bc.blockIndexByHash.Set(*hash5, bn5) + bc.blockIndex.blockIndexByHash.Put(*hash4, bn4) + bc.blockIndex.blockIndexByHash.Put(*hash5, bn5) // Set new block's parent to hash5 newBlockNode.Header.PrevBlockHash = hash5 @@ -1839,7 +1837,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { futureBlockHash, err = futureBlock.Hash() require.NoError(t, err) - futureBlockNode, exists := testMeta.chain.blockIndexByHash.Get(*futureBlockHash) + futureBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(futureBlockHash, futureBlock.Header.Height) require.True(t, exists) require.False(t, futureBlockNode.IsCommitted()) require.True(t, futureBlockNode.IsStored()) @@ -1848,10 +1846,12 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { } var timeoutBlockHash *BlockHash + var timeoutBlockHeight uint64 { // Okay let's timeout view 15 var timeoutBlock *MsgDeSoBlock timeoutBlock = _generateRealBlock(testMeta, 15, 16, 381, blockHash3, true) + timeoutBlockHeight = timeoutBlock.Header.Height success, _, _, err := testMeta.chain.ProcessBlockPoS(timeoutBlock, 15, true) fmt.Println(err) require.True(t, success) @@ -1877,7 +1877,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { _, exists := testMeta.chain.bestChainMap[*timeoutBlockHash] require.False(t, exists) - timeoutBlockNode, exists := testMeta.chain.blockIndexByHash.Get(*timeoutBlockHash) + timeoutBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(timeoutBlockHash, timeoutBlockHeight) require.True(t, exists) require.False(t, timeoutBlockNode.IsCommitted()) } @@ -1905,7 +1905,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { require.Len(t, missingBlockHashes, 1) require.True(t, missingBlockHashes[0].IsEqual(dummyParentBlockHash)) require.NoError(t, err) - orphanBlockInIndex, orphanBlockExists := testMeta.chain.blockIndexByHash.Get(*orphanBlockHash) + orphanBlockInIndex, orphanBlockExists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(orphanBlockHash, orphanBlock.Header.Height) require.True(t, orphanBlockExists) require.NotNil(t, orphanBlockInIndex) require.True(t, orphanBlockInIndex.IsStored()) @@ -1918,7 +1918,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { require.Len(t, missingBlockHashes, 0) require.NoError(t, err) - orphanBlockInIndex, orphanBlockExists = testMeta.chain.blockIndexByHash.Get(*orphanBlockHash) + orphanBlockInIndex, orphanBlockExists = testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(orphanBlockHash, orphanBlock.Header.Height) require.True(t, orphanBlockExists) require.NotNil(t, orphanBlockInIndex) require.True(t, orphanBlockInIndex.IsStored()) @@ -1944,7 +1944,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { require.True(t, missingBlockHashes[0].IsEqual(randomHash)) require.NoError(t, err) - malformedOrphanBlockInIndex, malformedOrphanBlockExists := testMeta.chain.blockIndexByHash.Get(*malformedOrphanBlockHash) + malformedOrphanBlockInIndex, malformedOrphanBlockExists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(malformedOrphanBlockHash, malformedOrphanBlock.Header.Height) require.True(t, malformedOrphanBlockExists) require.True(t, malformedOrphanBlockInIndex.IsValidateFailed()) require.True(t, malformedOrphanBlockInIndex.IsStored()) @@ -2067,7 +2067,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := realBlock.Hash() require.NoError(t, err) - blockNode, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + blockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, realBlock.Header.Height) require.True(t, exists) require.True(t, blockNode.IsStored()) require.False(t, blockNode.IsValidateFailed()) @@ -2088,7 +2088,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := realBlock.Hash() require.NoError(t, err) - blockNode, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + blockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, realBlock.Header.Height) require.True(t, exists) require.True(t, blockNode.IsStored()) require.True(t, blockNode.IsValidateFailed()) @@ -2115,7 +2115,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := realBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, realBlock.Header.Height) require.False(t, exists) } @@ -2167,7 +2167,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := realBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, realBlock.Header.Height) require.False(t, exists) } { @@ -2185,7 +2185,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := nextEpochBlock.Hash() require.NoError(t, err) - blockNode, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + blockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, nextEpochBlock.Header.Height) require.True(t, exists) require.True(t, blockNode.IsStored()) require.False(t, blockNode.IsValidateFailed()) @@ -2210,7 +2210,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := nextEpochBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, nextEpochBlock.Header.Height) require.False(t, exists) } { @@ -2261,7 +2261,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := nextEpochBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, nextEpochBlock.Header.Height) require.False(t, exists) } { @@ -2290,7 +2290,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // The block shouldn't be in the block index. blockHash, err := twoEpochsInFutureBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, twoEpochsInFutureBlock.Header.Height) require.False(t, exists) } { @@ -2307,7 +2307,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // The block should be in the block index. blockHash, err := prevEpochBlock.Hash() require.NoError(t, err) - blockNode, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + blockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, prevEpochBlock.Header.Height) require.True(t, exists) require.True(t, blockNode.IsStored()) require.False(t, blockNode.IsValidateFailed()) @@ -2375,7 +2375,7 @@ func TestHasValidProposerRandomSeedSignaturePoS(t *testing.T) { require.NoError(t, err) realBlockHash, err := realBlock.Hash() require.NoError(t, err) - realBlockNode, exists := testMeta.chain.blockIndexByHash.Get(*realBlockHash) + realBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(realBlockHash, realBlock.Header.Height) require.True(t, exists) require.True(t, realBlockNode.IsStored()) require.False(t, realBlockNode.IsValidateFailed()) @@ -2461,13 +2461,13 @@ func _generateRealBlockWithFailingTxn(testMeta *TestMeta, blockHeight uint64, vi } // TODO: Get real seed signature. - prevBlock, exists := testMeta.chain.blockIndexByHash.Get(*prevBlockHash) + prevBlock, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(prevBlockHash, blockHeight-1) require.True(testMeta.t, exists) // Always update the testMeta latestBlockView - latestBlockViewAndUtxoOps, err := testMeta.chain.getUtxoViewAndUtxoOpsAtBlockHash(*prevBlockHash) + latestBlockViewAndUtxoOps, err := testMeta.chain.getUtxoViewAndUtxoOpsAtBlockHash(*prevBlockHash, blockHeight-1) require.NoError(testMeta.t, err) latestBlockView := latestBlockViewAndUtxoOps.UtxoView - latestBlockNode, latestBlockNodeExists := testMeta.chain.blockIndexByHash.Get(*prevBlockHash) + latestBlockNode, latestBlockNodeExists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(prevBlockHash, blockHeight-1) require.True(testMeta.t, latestBlockNodeExists) latestBlockHeight := latestBlockNode.Height testMeta.posMempool.UpdateLatestBlock(latestBlockView, uint64(latestBlockHeight)) @@ -2524,7 +2524,7 @@ func _generateDummyBlock(testMeta *TestMeta, blockHeight uint64, view uint64, se blockNode, err := testMeta.chain.storeBlockInBlockIndex(msgDesoBlock) require.NoError(testMeta.t, err) require.True(testMeta.t, blockNode.IsStored()) - _, exists := testMeta.chain.blockIndexByHash.Get(*newBlockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(newBlockHash, msgDesoBlock.Header.Height) require.True(testMeta.t, exists) // Remove the transactions from this block from the mempool. // This prevents nonce reuse issues when trying to make failing blocks. @@ -2547,7 +2547,7 @@ func _generateBlockAndAddToBestChain(testMeta *TestMeta, blockHeight uint64, vie require.NoError(testMeta.t, err) require.True(testMeta.t, blockNode.IsStored()) require.True(testMeta.t, blockNode.IsValidated()) - newBlockNode, exists := testMeta.chain.blockIndexByHash.Get(*newBlockHash) + newBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(newBlockHash, msgDesoBlock.Header.Height) require.True(testMeta.t, exists) testMeta.chain.addTipBlockToBestChain(newBlockNode) // Update the latest block view @@ -2678,7 +2678,7 @@ func _getFullRealBlockTemplate( // Get leader voting private key. leaderVotingPrivateKey := testMeta.pubKeyToBLSKeyMap[leaderPublicKey] // Get hash of last block - chainTip, _ := testMeta.chain.blockIndexByHash.Get(*blockTemplate.Header.PrevBlockHash) + chainTip, _ := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockTemplate.Header.PrevBlockHash, blockTemplate.Header.Height-1) chainTipHash := chainTip.Hash // Get the vote signature payload // Hack to get view numbers working properly w/ PoW blocks. diff --git a/lib/pos_consensus.go b/lib/pos_consensus.go index d29421a13..a33983240 100644 --- a/lib/pos_consensus.go +++ b/lib/pos_consensus.go @@ -250,7 +250,10 @@ func (fc *FastHotStuffConsensus) handleBlockProposalEvent( // Fetch the parent block parentBlockHash := BlockHashFromConsensusInterface(event.QC.GetBlockHash()) - parentBlock, parentBlockExists := fc.blockchain.blockIndexByHash.Get(*parentBlockHash) + parentBlock, parentBlockExists, err := fc.blockchain.blockIndex.GetBlockNodeByHashOnly(parentBlockHash) + if err != nil { + return errors.Errorf("Error fetching parent block: %v", parentBlockHash) + } if !parentBlockExists { return errors.Errorf("Error fetching parent block: %v", parentBlockHash) } @@ -487,7 +490,8 @@ func (fc *FastHotStuffConsensus) HandleLocalTimeoutEvent(event *consensus.FastHo tipBlockHash := BlockHashFromConsensusInterface(event.TipBlockHash) // Fetch the HighQC from the Blockchain struct - tipBlockNode, tipBlockExists := fc.blockchain.blockIndexByHash.Get(*tipBlockHash) + // TODO: validate that TipHeight is a uint32 + tipBlockNode, tipBlockExists := fc.blockchain.blockIndex.GetBlockNodeByHashAndHeight(tipBlockHash, event.TipBlockHeight) if !tipBlockExists { return errors.Errorf("FastHotStuffConsensus.HandleLocalTimeoutEvent: Error fetching tip block: %v", tipBlockHash) } @@ -558,13 +562,17 @@ func (fc *FastHotStuffConsensus) HandleValidatorTimeout(pp *Peer, msg *MsgDeSoVa // If we don't have the highQC's block on hand, then we need to request it from the peer. We do // that first before storing the timeout message locally in the FastHotStuffEventLoop. This // prevents spamming of timeout messages by peers. - if !fc.blockchain.HasBlockInBlockIndex(msg.HighQC.BlockHash) { - err := errors.Errorf("FastHotStuffConsensus.HandleValidatorTimeout: Missing highQC's block: %v", msg.HighQC.BlockHash) + hasBlockInBlockIndex, err := fc.blockchain.HasBlockInBlockIndex(msg.HighQC.BlockHash) + if err != nil { + return nil, errors.Wrapf(err, "FastHotStuffConsensus.HandleValidatorTimeout: Error fetching block: ") + } + if !hasBlockInBlockIndex { + err = errors.Errorf("FastHotStuffConsensus.HandleValidatorTimeout: Missing highQC's block: %v", msg.HighQC.BlockHash) return []*BlockHash{msg.HighQC.BlockHash}, err } // Process the timeout message locally in the FastHotStuffEventLoop - if err := fc.fastHotStuffEventLoop.ProcessValidatorTimeout(msg); err != nil { + if err = fc.fastHotStuffEventLoop.ProcessValidatorTimeout(msg); err != nil { // If we can't process the timeout locally, then it must somehow be malformed, stale, // or a duplicate vote/timeout for the same view. glog.Errorf("FastHotStuffConsensus.HandleValidatorTimeout: Error processing timeout msg: %v", err) @@ -693,7 +701,7 @@ func (fc *FastHotStuffConsensus) tryProcessBlockAsNewTip(block *MsgDeSoBlock) ([ return nil, errors.Errorf("Error hashing tip block: %v", err) } - utxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*tipBlockHash) + utxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*tipBlockHash, tipBlock.Height) if err != nil { return nil, errors.Errorf("Error fetching UtxoView for tip block: %v", err) } @@ -733,13 +741,16 @@ func (fc *FastHotStuffConsensus) produceUnsignedBlockForBlockProposalEvent( parentBlockHash := BlockHashFromConsensusInterface(event.QC.GetBlockHash()) // Fetch the parent block - parentBlock, parentBlockExists := fc.blockchain.blockIndexByHash.Get(*parentBlockHash) + parentBlock, parentBlockExists, err := fc.blockchain.blockIndex.GetBlockNodeByHashOnly(parentBlockHash) + if err != nil { + return nil, errors.Errorf("Error fetching parent block: %v", parentBlockHash) + } if !parentBlockExists { return nil, errors.Errorf("Error fetching parent block: %v", parentBlockHash) } // Build a UtxoView at the parent block - parentUtxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*parentBlockHash) + parentUtxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*parentBlockHash, uint64(parentBlock.Height)) if err != nil { // This should never happen as long as the parent block is a descendant of the committed tip. return nil, errors.Errorf("Error fetching UtxoView for parent block: %v", parentBlockHash) diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index ac9ea0ed9..900afb71a 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -112,9 +112,11 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { params: &DeSoTestnetParams, blockchain: &Blockchain{ ChainLock: deadlock.RWMutex{}, - blockIndexByHash: collections.NewConcurrentMapFromMap(map[BlockHash]*BlockNode{ - *blockHash: {Header: blockHeader}, - }), + blockIndex: &BlockIndex{ + blockIndexByHash: newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ + *blockHash: {Header: blockHeader}, + }), + }, params: &DeSoTestnetParams, }, fastHotStuffEventLoop: &consensus.MockFastHotStuffEventLoop{ diff --git a/lib/postgres.go b/lib/postgres.go index 4b43e47c0..0c5991756 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/deso-protocol/core/collections" + lru2 "github.com/decred/dcrd/container/lru" "net/url" "regexp" "strings" @@ -1305,16 +1305,16 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { } // GetBlockIndex gets all the PGBlocks and creates a map of BlockHash to BlockNode as needed by blockchain.go -func (postgres *Postgres) GetBlockIndex() (*collections.ConcurrentMap[BlockHash, *BlockNode], error) { +func (postgres *Postgres) GetBlockIndex() (*lru2.Map[BlockHash, *BlockNode], error) { var blocks []PGBlock err := postgres.db.Model(&blocks).Select() if err != nil { return nil, err } - blockMap := collections.NewConcurrentMap[BlockHash, *BlockNode]() + blockMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, block := range blocks { - blockMap.Set(*block.Hash, &BlockNode{ + blockMap.Put(*block.Hash, &BlockNode{ Hash: block.Hash, Height: uint32(block.Height), DifficultyTarget: block.DifficultyTarget, @@ -1333,7 +1333,8 @@ func (postgres *Postgres) GetBlockIndex() (*collections.ConcurrentMap[BlockHash, } // Setup parent pointers - blockMap.Iterate(func(key BlockHash, blockNode *BlockNode) { + for _, key := range blockMap.Keys() { + blockNode, _ := blockMap.Get(key) // Genesis block has nil parent parentHash := blockNode.Header.PrevBlockHash if parentHash != nil { @@ -1343,7 +1344,7 @@ func (postgres *Postgres) GetBlockIndex() (*collections.ConcurrentMap[BlockHash, } blockNode.Parent = parent } - }) + } return blockMap, nil } diff --git a/lib/server.go b/lib/server.go index b8c3bb5c5..21756a665 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1086,7 +1086,13 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // have this issue. Hitting duplicates after we're done syncing is // fine and can happen in certain cases. headerHash, _ := headerReceived.Hash() - if srv.blockchain.HasHeader(headerHash) { + hasHeader, err := srv.blockchain.HasHeader(headerHash) + // TODO: what should we really do here? + if err != nil { + glog.Errorf("Server._handleHeaderBundle: Error checking if header %v exists: %v", headerHash, err) + return + } + if hasHeader { if srv.blockchain.isSyncing() { glog.Warningf("Server._handleHeaderBundle: Duplicate header %v received from peer %v "+ @@ -1309,7 +1315,12 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // we're either not aware of or that we don't think is the best chain. // Doing things this way makes it so that when we request blocks we // are 100% positive the peer has them. - if !srv.blockchain.HasHeader(msg.TipHash) { + hasHeader, err := srv.blockchain.HasHeader(msg.TipHash) + if err != nil { + glog.Errorf("Server._handleHeaderBundle: Error checking if header %v exists: %v", msg.TipHash, err) + return + } + if !hasHeader { glog.V(1).Infof("Server._handleHeaderBundle: Peer's tip is not in our "+ "blockchain so not requesting anything else from them. Our block "+ "tip %v, their tip %v:%d, peer: %v", diff --git a/lib/state_change_syncer.go b/lib/state_change_syncer.go index 7498690b7..e825dfbb6 100644 --- a/lib/state_change_syncer.go +++ b/lib/state_change_syncer.go @@ -847,7 +847,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser // TODO: Have Z look at if we need to do some caching in the uncommitted blocks logic. // First connect the uncommitted blocks to the mempool view. for _, uncommittedBlock := range uncommittedBlocks { - utxoViewAndOpsAtBlockHash, err := server.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*uncommittedBlock.Hash) + utxoViewAndOpsAtBlockHash, err := server.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*uncommittedBlock.Hash, uint64(uncommittedBlock.Height)) if err != nil { mempoolUtxoView.EventManager.stateSyncerFlushed(&StateSyncerFlushedEvent{ FlushId: originalCommittedFlushId, diff --git a/lib/txindex.go b/lib/txindex.go index 9f8e15f47..5fe9cb515 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -225,7 +225,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // The only thing we can really do in this case is rebuild the entire index // from scratch. To do that, we return all the blocks in the index to detach // and all the blocks in the real chain to attach. - txindexTipNode, _ := txi.TXIndexChain.blockIndexByHash.Get(*txindexTipHash.Hash) + txindexTipNode, _ := txi.TXIndexChain.blockIndex.GetBlockNodeByHashAndHeight(txindexTipHash.Hash, uint64(txindexTipHash.Height)) // Get the committed tip. committedTip, _ := txi.CoreChain.GetCommittedTip() @@ -368,13 +368,13 @@ func (txi *TXIndex) Update() error { // Delete this block from the chain db so we don't get duplicate block errors. // Remove this block from our bestChain data structures. - newBlockIndexByHash, newBlockIndexByHeight := txi.TXIndexChain.CopyBlockIndexes() + newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() newBestChain, newBestChainMap := txi.TXIndexChain.CopyBestChain() newBestChain = newBestChain[:len(newBestChain)-1] delete(newBestChainMap, *(blockToDetach.Hash)) - newBlockIndexByHash.Remove(*(blockToDetach.Hash)) + newBlockIndex.Delete(*(blockToDetach.Hash)) - txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndexByHash, newBlockIndexByHeight) + txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndex) // At this point the entries for the block should have been removed // from both our Txindex chain and our transaction index mappings. @@ -408,7 +408,7 @@ func (txi *TXIndex) Update() error { utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) if blockToAttach.Header.PrevBlockHash != nil && !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { var utxoViewAndUtxoOps *BlockViewAndUtxoOps - utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash) + utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) if err != nil { return fmt.Errorf("Update: Problem getting UtxoView at block hash %v: %v", blockToAttach.Header.PrevBlockHash, err) From f3e8e4c233e83ca9bb72aa0860a1c8845faa3c03 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Sep 2024 15:01:51 -0400 Subject: [PATCH 02/62] best chain and header chain with read through cache --- lib/block_view_lockups_test.go | 4 +- lib/block_view_test.go | 12 +- lib/blockchain.go | 368 ++++++++++++++++++++++----------- lib/db_utils.go | 87 +++++++- lib/pos_blockchain.go | 48 +++-- lib/pos_blockchain_test.go | 54 ++--- lib/server.go | 58 ++++-- lib/state_change_syncer.go | 6 +- lib/txindex.go | 4 +- 9 files changed, 443 insertions(+), 198 deletions(-) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index 6d25cb1fa..faaa6a3e3 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2462,7 +2462,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk2.Header.Height)) // Update the tip - testMeta.chain.bestChain = testMeta.chain.bestChain[:len(testMeta.chain.bestChain)-1] + testMeta.chain.bestChain.Chain = testMeta.chain.bestChain.Chain[:len(testMeta.chain.bestChain.Chain)-1] // Validate the state update utxoView = NewUtxoView( @@ -2517,7 +2517,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk1.Header.Height)) // Update the tip - testMeta.chain.bestChain = testMeta.chain.bestChain[:len(testMeta.chain.bestChain)-1] + testMeta.chain.bestChain.Chain = testMeta.chain.bestChain.Chain[:len(testMeta.chain.bestChain.Chain)-1] // Verify we return back to the initial state utxoView = NewUtxoView( diff --git a/lib/block_view_test.go b/lib/block_view_test.go index 35f019271..2717aeb50 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -702,8 +702,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te require.NoError(err) // sanity-check that the last block hash is the same as the last header hash. require.Equal(true, bytes.Equal( - tm.chain.bestChain[len(tm.chain.bestChain)-1].Hash.ToBytes(), - tm.chain.bestHeaderChain[len(tm.chain.bestHeaderChain)-1].Hash.ToBytes())) + tm.chain.bestChain.GetTip().Hash.ToBytes(), + tm.chain.bestHeaderChain.GetTip().Hash.ToBytes())) // Last block shouldn't be nil, and the number of expectedTxns should be the same as in the testVectorBlock + 1, // because of the additional block reward. require.NotNil(lastBlock) @@ -791,10 +791,10 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // TODO: if ever needed we can call tm.chain.eventManager.blockDisconnected() here. // Update the block and header metadata chains. - tm.chain.bestChain = tm.chain.bestChain[:len(tm.chain.bestChain)-1] - tm.chain.bestHeaderChain = tm.chain.bestHeaderChain[:len(tm.chain.bestHeaderChain)-1] - delete(tm.chain.bestChainMap, *lastBlockHash) - delete(tm.chain.bestHeaderChainMap, *lastBlockHash) + tm.chain.bestChain.Chain = tm.chain.bestChain.Chain[:len(tm.chain.bestChain.Chain)-1] + tm.chain.bestHeaderChain.Chain = tm.chain.bestHeaderChain.Chain[:len(tm.chain.bestHeaderChain.Chain)-1] + tm.chain.bestChain.ChainMap.Delete(*lastBlockHash) + tm.chain.bestHeaderChain.ChainMap.Delete(*lastBlockHash) // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. diff --git a/lib/blockchain.go b/lib/blockchain.go index a03b55e39..f1e35d305 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -536,12 +536,11 @@ type BlockIndex struct { tipNode *BlockNode } -func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { +func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { return &BlockIndex{ db: db, snapshot: snapshot, blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? - tipNode: tipNode, } } @@ -566,18 +565,14 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, if exists { return val, true, nil } - // NOTE: this can be sped up significantly by mapping blockHash to blockNode or blockHash to blockHeight - // in badger, so we can avoid deserializing the full block. - fullBlock, err := GetBlock(blockHash, bi.db, bi.snapshot) + height, err := GetHeightForHash(bi.db, bi.snapshot, blockHash) if err != nil { - return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem reading block from db") - } - nodeHeader := fullBlock.Header - fullBlockHash, err := fullBlock.Hash() - if err != nil { - return nil, false, errors.Wrapf(err, "_initChain: Problem hashing block") + if errors.Is(err, badger.ErrKeyNotFound) { + return nil, false, nil + } + return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem getting height for hash") } - blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(nodeHeader.Height), fullBlockHash, false) + blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if blockNode == nil { return nil, false, nil } @@ -622,20 +617,94 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { type BestChain struct { db *badger.DB + snapshot *Snapshot IsHeaderChain bool - Chain []*BlockNode - ChainMap map[BlockHash]*BlockNode - tipNode *BlockNode + Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. + ChainMap *lru2.Map[BlockHash, *BlockNode] } -func NewBestChain(db *badger.DB, isHeaderChain bool, tipNode *BlockNode) *BestChain { +func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool) *BestChain { return &BestChain{ db: db, + snapshot: snapshot, IsHeaderChain: isHeaderChain, Chain: []*BlockNode{}, - ChainMap: make(map[BlockHash]*BlockNode), - tipNode: tipNode, + ChainMap: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), + } +} + +func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { + bestChain.Chain = append(bestChain.Chain, tipNode) + bestChain.ChainMap.Put(*tipNode.Hash, tipNode) +} + +func (bestChain *BestChain) GetTip() *BlockNode { + if len(bestChain.Chain) == 0 { + return nil + } + return bestChain.Chain[len(bestChain.Chain)-1] +} + +func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, error) { + if height > math.MaxUint32 { + return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) + } + + prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) + _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) + if len(valsFound) == 0 { + return nil, false, nil + } + for _, val := range valsFound { + blockNode, err := DeserializeBlockNode(val) + if err != nil { + glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) + continue + } + if blockNode.IsCommitted() { + return blockNode, true, nil + } } + // TODO: how to return uncommitted blocks by height. We probably just need to iterate backwards through the + // best chain. + return nil, false, nil +} + +func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { + val, exists := bestChain.ChainMap.Get(*blockHash) + if exists { + return val, true + } + if height > math.MaxUint32 { + glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) + } + bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) + if bn == nil { + return nil, false + } + bestChain.ChainMap.Put(*blockHash, bn) + return bn, true +} + +func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bool, error) { + val, exists := bestChain.ChainMap.Get(*blockHash) + if exists { + return val, true, nil + } + + height, err := GetHeightForHash(bestChain.db, bestChain.snapshot, blockHash) + if err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return nil, false, nil + } + return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") + } + bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) + if bn == nil { + return nil, false, nil + } + bestChain.ChainMap.Put(*blockHash, bn) + return bn, true, nil } type Blockchain struct { @@ -673,15 +742,15 @@ type Blockchain struct { //blockIndexByHeight map[uint64]map[BlockHash]*BlockNode // An in-memory slice of the blocks on the main chain only. The end of // this slice is the best known tip that we have at any given time. - bestChain []*BlockNode - bestChainMap map[BlockHash]*BlockNode - - bestHeaderChain []*BlockNode - bestHeaderChainMap map[BlockHash]*BlockNode + //bestChain []*BlockNode + //bestChainMap map[BlockHash]*BlockNode + // + //bestHeaderChain []*BlockNode + //bestHeaderChainMap map[BlockHash]*BlockNode - blockIndex *BlockIndex - //bestChain *BestChain - //bestHeaderChain *BestChain + blockIndex *BlockIndex + bestChain *BestChain + bestHeaderChain *BestChain // We keep track of orphan blocks with the following data structures. Orphans // are not written to disk and are only cached in memory. Moreover we only keep @@ -849,23 +918,25 @@ func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { return len(blockNodes) > 0 } -func (bc *Blockchain) CopyBestChain() ([]*BlockNode, map[BlockHash]*BlockNode) { +func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Map[BlockHash, *BlockNode]) { newBestChain := []*BlockNode{} - newBestChainMap := make(map[BlockHash]*BlockNode) - newBestChain = append(newBestChain, bc.bestChain...) - for kk, vv := range bc.bestChainMap { - newBestChainMap[kk] = vv + newBestChainMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBestChain = append(newBestChain, bc.bestChain.Chain...) + for _, key := range bc.bestChain.ChainMap.Keys() { + val, _ := bc.bestChain.ChainMap.Get(key) + newBestChainMap.Put(key, val) } return newBestChain, newBestChainMap } -func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, map[BlockHash]*BlockNode) { +func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Map[BlockHash, *BlockNode]) { newBestChain := []*BlockNode{} - newBestChainMap := make(map[BlockHash]*BlockNode) - newBestChain = append(newBestChain, bc.bestHeaderChain...) - for kk, vv := range bc.bestHeaderChainMap { - newBestChainMap[kk] = vv + newBestChainMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) + for _, key := range bc.bestHeaderChain.ChainMap.Keys() { + val, _ := bc.bestHeaderChain.ChainMap.Get(key) + newBestChainMap.Put(key, val) } return newBestChain, newBestChainMap @@ -873,8 +944,9 @@ func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, map[BlockHash]*BlockN // IsFullyStored determines if there are block nodes that haven't been fully stored or processed in the best block chain. func (bc *Blockchain) IsFullyStored() bool { + // TODO: figure out how to iterate over best chain w/o having entire thing in memory. if bc.ChainState() == SyncStateFullyCurrent { - for _, blockNode := range bc.bestChain { + for _, blockNode := range bc.bestChain.Chain { if !blockNode.Status.IsFullyProcessed() { return false } @@ -971,21 +1043,26 @@ func (bc *Blockchain) _initChain() error { if !tipNodeExists { return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) } + // Walk back the last 100 hours of blocks. + currBlockCounter := 1 + for currBlockCounter < 3600*100 && tipNode.Header.PrevBlockHash != nil { + bc.blockIndex.GetBlockNodeByHashAndHeight(tipNode.Header.PrevBlockHash, tipNode.Header.Height-1) + currBlockCounter++ + } } } - bc.blockIndex.setTipNode(tipNode) // At this point the blockIndexByHash should contain a full node tree with all // nodes pointing to valid parent nodes. { // Walk back from the best node to the genesis block and store them all // in bestChain. - bc.bestChain, err = GetBestChain(tipNode) + bc.bestChain.Chain, err = GetBestChain(tipNode) if err != nil { return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") } - for _, bestChainNode := range bc.bestChain { - bc.bestChainMap[*bestChainNode.Hash] = bestChainNode + for _, bestChainNode := range bc.bestChain.Chain { + bc.bestChain.ChainMap.Put(*bestChainNode.Hash, bestChainNode) } } @@ -993,12 +1070,12 @@ func (bc *Blockchain) _initChain() error { { // Walk back from the best node to the genesis block and store them all // in bestChain. - bc.bestHeaderChain, err = GetBestChain(tipNode) + bc.bestHeaderChain.Chain, err = GetBestChain(tipNode) if err != nil { return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") } - for _, bestHeaderChainNode := range bc.bestHeaderChain { - bc.bestHeaderChainMap[*bestHeaderChainNode.Hash] = bestHeaderChainNode + for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { + bc.bestHeaderChain.ChainMap.Put(*bestHeaderChainNode.Hash, bestHeaderChainNode) } } @@ -1050,9 +1127,9 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// currentHeaderTip := bc.headerTip() _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) - bc.bestHeaderChain, bc.bestHeaderChainMap = updateBestChainInMemory( - bc.bestHeaderChain, - bc.bestHeaderChainMap, + bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + bc.bestHeaderChain.Chain, + bc.bestHeaderChain.ChainMap, blocksToDetach, blocksToAttach, ) @@ -1104,15 +1181,15 @@ func NewBlockchain( eventManager: eventManager, archivalMode: archivalMode, - blockIndex: NewBlockIndex(db, snapshot, nil), + blockIndex: NewBlockIndex(db, snapshot), //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), - bestChainMap: make(map[BlockHash]*BlockNode), - //bestChain: NewBestChain(db, false), - bestHeaderChainMap: make(map[BlockHash]*BlockNode), - //bestHeaderChain: NewBestChain(db, true), - blockViewCache: lru.NewKVCache(100), // TODO: parameterize - snapshotCache: NewSnapshotCache(), + //bestChainMap: make(map[BlockHash]*BlockNode), + bestChain: NewBestChain(db, snapshot, false), + //bestHeaderChainMap: make(map[BlockHash]*BlockNode), + bestHeaderChain: NewBestChain(db, snapshot, true), + blockViewCache: lru.NewKVCache(100), // TODO: parameterize + snapshotCache: NewSnapshotCache(), checkpointSyncingProviders: checkpointSyncingProviders, @@ -1178,8 +1255,7 @@ func fastLog2Floor(n uint32) uint8 { // // This function MUST be called with the chain state lock held (for reads). func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32, - blockIndex *BlockIndex, bestChainList []*BlockNode, - bestChainMap map[BlockHash]*BlockNode) (*BlockNode, uint32) { + blockIndex *BlockIndex, bestChain *BestChain) (*BlockNode, uint32) { // There are no block locators so a specific block is being requested // as identified by the stop hash. @@ -1198,9 +1274,22 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // Find the most recent locator block hash in the main chain. In the // case none of the hashes in the locator are in the main chain, fall // back to the genesis block. - startNode := bestChainList[0] + startNode, startNodeExists, err := bestChain.GetBlockByHeight(0) + if err != nil { + glog.Errorf("locateInventory: Problem getting block by height: %v", err) + return nil, 0 + } + if !startNodeExists { + glog.Errorf("locateInventory: Genesis block not found") + return nil, 0 + } for _, hash := range locator { - node, bestChainContainsNode := bestChainMap[*hash] + // TODO: replace w/ read-through cache call. + node, bestChainContainsNode, err := bestChain.GetBlockByHash(hash) + if err != nil { + glog.Errorf("locateInventory: Problem getting block by hash: %v", err) + continue + } if bestChainContainsNode { startNode = node break @@ -1211,18 +1300,25 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // is no next block it means the most recently known block is the tip of // the best chain, so there is nothing more to do. nextNodeHeight := uint32(startNode.Header.Height) + 1 - if uint32(len(bestChainList)) <= nextNodeHeight { + startNode, startNodeExists, err = bestChain.GetBlockByHeight(uint64(nextNodeHeight)) + if err != nil { + glog.Errorf("locateInventory: Problem getting block by height: %v", err) + return nil, 0 + } + if !startNodeExists { return nil, 0 } - startNode = bestChainList[nextNodeHeight] // Calculate how many entries are needed. - tip := bestChainList[len(bestChainList)-1] - total := uint32((tip.Header.Height - startNode.Header.Height) + 1) + total := (bestChain.GetTip().Height - startNode.Height) + 1 if stopNodeError != nil && stopNodeExists && stopNode != nil && stopNode.Header.Height >= startNode.Header.Height { - _, bestChainContainsStopNode := bestChainMap[*stopNode.Hash] + _, bestChainContainsStopNode, err := bestChain.GetBlockByHash(stopNode.Hash) + if err != nil { + glog.Errorf("locateInventory: Problem getting block by hash: %v", err) + return nil, 0 + } if bestChainContainsStopNode { total = uint32((stopNode.Header.Height - startNode.Header.Height) + 1) } @@ -1242,14 +1338,13 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // // This function MUST be called with the ChainLock held (for reads). func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, - blockIndex *BlockIndex, bestChainList []*BlockNode, - bestChainMap map[BlockHash]*BlockNode) []*MsgDeSoHeader { + blockIndex *BlockIndex, bestChain *BestChain) []*MsgDeSoHeader { // Find the node after the first known block in the locator and the // total number of nodes after it needed while respecting the stop hash // and max entries. node, total := locateInventory(locator, stopHash, maxHeaders, - blockIndex, bestChainList, bestChainMap) + blockIndex, bestChain) if total == 0 { return nil } @@ -1264,7 +1359,15 @@ func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, if uint32(len(headers)) == total { break } - node = bestChainList[node.Header.Height+1] + var nodeExists bool + node, nodeExists, err = bestChain.GetBlockByHeight(node.Header.Height + 1) + if err != nil { + glog.Errorf("locateHeaders: Problem getting block by height: %v", err) + break + } + if !nodeExists { + break + } } return headers } @@ -1293,7 +1396,7 @@ func (bc *Blockchain) LocateBestBlockChainHeaders( // where it's currently called is single-threaded via a channel in server.go. Going to // avoid messing with it for now. headers := locateHeaders(locator, stopHash, maxHeaders, - bc.blockIndex, bc.bestChain, bc.bestChainMap) + bc.blockIndex, bc.bestChain) return headers } @@ -1355,8 +1458,17 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { // ancestors must be too, so use a much faster O(1) lookup in // that case. Otherwise, fall back to walking backwards through // the nodes of the other chain to the correct ancestor. - if _, exists := bc.bestHeaderChainMap[*tip.Hash]; exists { - tip = bc.bestHeaderChain[height] + _, exists, err := bc.bestHeaderChain.GetBlockByHash(tip.Hash) + if err != nil { + glog.Errorf("LatestLocator: Problem getting block by hash: %v", err) + exists = false + } + if exists { + var innerExists bool + tip, innerExists = bc.bestHeaderChain.GetBlockByHashAndHeight(tip.Hash, uint64(height)) + if !innerExists { + glog.Errorf("LatestLocator: Block %v not found in best header chain", tip.Hash) + } } else { tip = tip.Ancestor(uint32(height)) } @@ -1405,7 +1517,7 @@ func (bc *Blockchain) GetBlockNodesToFetch( // If the tip of the best block chain is in the main header chain, make that // the start point for our fetch. - headerNodeStart, blockTipExistsInBestHeaderChain := bc.bestHeaderChainMap[*bestBlockTip.Hash] + headerNodeStart, blockTipExistsInBestHeaderChain := bc.bestHeaderChain.GetBlockByHashAndHeight(bestBlockTip.Hash, uint64(bestBlockTip.Height)) if !blockTipExistsInBestHeaderChain { // If the hash of the tip of the best blockchain is not in the best header chain, then // this is a case where the header chain has forked off from the best block @@ -1424,7 +1536,17 @@ func (bc *Blockchain) GetBlockNodesToFetch( // an error and set it to the genesis block. glog.Errorf("GetBlockToFetch: headerNode was nil after iterating " + "backward through best header chain; using genesis block") - headerNodeStart = bc.bestHeaderChain[0] + var err error + var genesisBlockExists bool + headerNodeStart, genesisBlockExists, err = bc.bestHeaderChain.GetBlockByHeight(0) + if err != nil { + glog.Errorf("GetBlockToFetch: Problem getting genesis block: %v", err) + return nil + } + if !genesisBlockExists { + glog.Errorf("GetBlockToFetch: Genesis block not found") + return nil + } } } @@ -1434,14 +1556,22 @@ func (bc *Blockchain) GetBlockNodesToFetch( currentHeight := headerNodeStart.Height + 1 blockNodesToFetch := []*BlockNode{} heightLimit := maxHeight - if heightLimit >= uint32(len(bc.bestHeaderChain)) { - heightLimit = uint32(len(bc.bestHeaderChain) - 1) + if heightLimit >= bc.bestHeaderChain.GetTip().Height { + heightLimit = bc.bestHeaderChain.GetTip().Height - 1 } for currentHeight <= heightLimit && len(blockNodesToFetch) < numBlocks { // Get the current hash and increment the height. - currentNode := bc.bestHeaderChain[currentHeight] + currentNode, currentNodeExists, err := bc.bestHeaderChain.GetBlockByHeight(uint64(currentHeight)) + if err != nil { + glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) + return nil + } + if !currentNodeExists { + glog.Errorf("GetBlockToFetch: Block at height %d not found", currentHeight) + return nil + } currentHeight++ if _, exists := blocksToIgnore[*currentNode.Hash]; exists { @@ -1460,12 +1590,12 @@ func (bc *Blockchain) HasHeader(headerHash *BlockHash) (bool, error) { return exists, errors.Wrap(err, "Blockchain.HasHeader: ") } -func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) *BlockNode { - if blockHeight >= uint32(len(bc.bestHeaderChain)) { - return nil +// TODO: delete me? +func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) (*BlockNode, bool, error) { + if blockHeight >= bc.bestChain.GetTip().Height { + return nil, false, nil } - - return bc.bestHeaderChain[blockHeight] + return bc.bestChain.GetBlockByHeight(uint64(blockHeight)) } func (bc *Blockchain) HasBlockInBlockIndex(blockHash *BlockHash) (bool, error) { @@ -1487,14 +1617,14 @@ func (bc *Blockchain) GetBlock(blockHash *BlockHash) *MsgDeSoBlock { return blk } -func (bc *Blockchain) GetBlockAtHeight(height uint32) *MsgDeSoBlock { - numBlocks := uint32(len(bc.bestChain)) - - if height >= numBlocks { - return nil +func (bc *Blockchain) GetBlockAtHeight(height uint32) (*MsgDeSoBlock, error) { + bn, bnExists, err := bc.bestChain.GetBlockByHeight(uint64(height)) + if !bnExists || err != nil { + glog.Errorf("Blockchain.GetBlockAtHeight: Problem getting block by height: %v", err) + return nil, err } - return bc.GetBlock(bc.bestChain[height].Hash) + return bc.GetBlock(bn.Hash), nil } // GetBlockNodeWithHash looks for a block node in the bestChain list that matches the hash. @@ -1502,7 +1632,11 @@ func (bc *Blockchain) GetBlockNodeWithHash(hash *BlockHash) *BlockNode { if hash == nil { return nil } - return bc.bestChainMap[*hash] + bn, bnExists, err := bc.bestChain.GetBlockByHash(hash) + if !bnExists || err != nil { + return nil + } + return bn } // isTipMaxed compares the tip height to the MaxSyncBlockHeight height. @@ -1665,7 +1799,11 @@ func (bc *Blockchain) checkArchivalMode() bool { } firstSnapshotHeight := bc.snapshot.CurrentEpochSnapshotMetadata.FirstSnapshotBlockHeight - for _, blockNode := range bc.bestChain { + // @diamondhands - can we spot check just a few blocks such as firstSnapshotHeight - 1, + // firstSnapshotHeight / 2 - 1, and firstSnapshotHeight / 4 - 1 to see if they are stored? + // TODO: figure out how to iterate over best chain to checkArchivalMode + // when we only have a portion of best chain in memory. + for _, blockNode := range bc.bestChain.Chain { if uint64(blockNode.Height) > firstSnapshotHeight { return false } @@ -1715,13 +1853,7 @@ func (bc *Blockchain) isHyperSyncCondition() bool { // main chain for blocks, which is why separate functions are required for // each of them. func (bc *Blockchain) headerTip() *BlockNode { - if len(bc.bestHeaderChain) == 0 { - return nil - } - - // Note this should always work because we should have the genesis block - // in here. - return bc.bestHeaderChain[len(bc.bestHeaderChain)-1] + return bc.bestHeaderChain.GetTip() } func (bc *Blockchain) HeaderTip() *BlockNode { @@ -1753,15 +1885,7 @@ func (bc *Blockchain) Snapshot() *Snapshot { // invalidate and chop off the headers corresponding to those blocks and // their ancestors so the two generally stay in sync. func (bc *Blockchain) blockTip() *BlockNode { - var tip *BlockNode - - if len(bc.bestChain) == 0 { - return nil - } - - tip = bc.bestChain[len(bc.bestChain)-1] - - return tip + return bc.bestChain.GetTip() } func (bc *Blockchain) BlockTip() *BlockNode { @@ -1769,20 +1893,20 @@ func (bc *Blockchain) BlockTip() *BlockNode { } func (bc *Blockchain) BestChain() []*BlockNode { - return bc.bestChain + return bc.bestChain.Chain } func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { - bc.bestChain = bestChain + bc.bestChain.Chain = bestChain } func (bc *Blockchain) SetBestChainMap( bestChain []*BlockNode, - bestChainMap map[BlockHash]*BlockNode, + bestChainMap *lru2.Map[BlockHash, *BlockNode], blockIndexByHash *lru2.Map[BlockHash, *BlockNode], ) { - bc.bestChain = bestChain - bc.bestChainMap = bestChainMap + bc.bestChain.Chain = bestChain + bc.bestChain.ChainMap = bestChainMap bc.blockIndex.blockIndexByHash = blockIndexByHash } @@ -2071,14 +2195,14 @@ func GetReorgBlocks(tip *BlockNode, newNode *BlockNode) (_commonAncestor *BlockN return commonAncestor, detachBlocks, attachBlocks } -func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap map[BlockHash]*BlockNode, detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( - chainList []*BlockNode, chainMap map[BlockHash]*BlockNode) { +func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Map[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( + chainList []*BlockNode, chainMap *lru2.Map[BlockHash, *BlockNode]) { // Remove the nodes we detached from the end of the best chain node list. tipIndex := len(mainChainList) - 1 for blockOffset := 0; blockOffset < len(detachBlocks); blockOffset++ { blockIndex := tipIndex - blockOffset - delete(mainChainMap, *mainChainList[blockIndex].Hash) + mainChainMap.Delete(*mainChainList[blockIndex].Hash) } mainChainList = mainChainList[:len(mainChainList)-len(detachBlocks)] @@ -2087,7 +2211,7 @@ func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap map[BlockH // first, with the new tip at the end. for _, attachNode := range attachBlocks { mainChainList = append(mainChainList, attachNode) - mainChainMap[*attachNode.Hash] = attachNode + mainChainMap.Put(*attachNode.Hash, attachNode) } return mainChainList, mainChainMap @@ -2268,8 +2392,8 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B isMainChain = true _, detachBlocks, attachBlocks := GetReorgBlocks(headerTip, newNode) - bc.bestHeaderChain, bc.bestHeaderChainMap = updateBestChainInMemory( - bc.bestHeaderChain, bc.bestHeaderChainMap, detachBlocks, attachBlocks) + bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) // Note that we don't store the best header hash here and so this is an // in-memory-only adjustment. See the comment above on preventing attacks. @@ -2759,8 +2883,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Now that we've set the best chain in the db, update our in-memory data // structure to reflect this. Do a quick check first to make sure it's consistent. - lastIndex := len(bc.bestChain) - 1 - bestChainHash := bc.bestChain[lastIndex].Hash + lastIndex := len(bc.bestChain.Chain) - 1 + bestChainHash := bc.bestChain.Chain[lastIndex].Hash if !bestChainHash.IsEqual(nodeToValidate.Header.PrevBlockHash) { return false, false, fmt.Errorf("ProcessBlock: Last block in bestChain "+ @@ -2771,13 +2895,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If we're syncing there's no risk of concurrency issues. Otherwise, we // need to make a copy in order to be save. if bc.isSyncing() { - bc.bestChain = append(bc.bestChain, nodeToValidate) - bc.bestChainMap[*nodeToValidate.Hash] = nodeToValidate + bc.bestChain.PushNewTip(nodeToValidate) } else { newBestChain, newBestChainMap := bc.CopyBestChain() - newBestChain = append(newBestChain, nodeToValidate) - newBestChainMap[*nodeToValidate.Hash] = nodeToValidate - bc.bestChain, bc.bestChainMap = newBestChain, newBestChainMap + bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap + bc.bestChain.PushNewTip(nodeToValidate) } // This node is on the main chain so set this variable. @@ -3041,7 +3163,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures newBestChain, newBestChainMap := bc.CopyBestChain() newBestChain, newBestChainMap = updateBestChainInMemory( newBestChain, newBestChainMap, detachBlocks, attachBlocks) - bc.bestChain, bc.bestChainMap = newBestChain, newBestChainMap + bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap // If we made it here then this block is on the main chain. isMainChain = true diff --git a/lib/db_utils.go b/lib/db_utils.go index 2bb61e0f7..ec9350f5a 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -604,7 +604,11 @@ type DBPrefixes struct { // When reading and writing data to this prefixes, please acquire the snapshotDbMutex in the snapshot. PrefixHypersyncSnapshotDBPrefix []byte `prefix_id:"[97]"` - // NEXT_TAG: 98 + // PrefixHashToHeight is used to store the height of a block given its hash. + // This helps us map a block hash to its height so we can look up the full info + // in PrefixHeightHashToNodeInfo. + PrefixHashToHeight []byte `prefix_id:"[98]"` + // NEXT_TAG: 99 } // DecodeStateKey decodes a state key into a DeSoEncoder type. This is useful for encoders which don't have a stored @@ -5206,6 +5210,12 @@ func _heightHashToNodeIndexKey(height uint32, hash *BlockHash, bitcoinNodes bool return key } +func _hashToHeightIndexKey(hash *BlockHash) []byte { + key := append([]byte{}, Prefixes.PrefixHashToHeight...) + key = append(key, hash[:]...) + return key +} + func GetHeightHashToNodeInfoWithTxn(txn *badger.Txn, snap *Snapshot, height uint32, hash *BlockHash, bitcoinNodes bool) *BlockNode { @@ -5245,9 +5255,44 @@ func PutHeightHashToNodeInfoWithTxn(txn *badger.Txn, snap *Snapshot, if err := DBSetWithTxn(txn, snap, key, serializedNode, eventManager); err != nil { return err } + + hashToHeightKey := _hashToHeightIndexKey(node.Hash) + if err = DBSetWithTxn(txn, snap, hashToHeightKey, UintToBuf(uint64(node.Height)), eventManager); err != nil { + return err + } + return nil } +func PutHashToHeightBatch(handle *badger.DB, snap *Snapshot, hashToHeight map[BlockHash]uint32, eventManager *EventManager) error { + return handle.Update(func(txn *badger.Txn) error { + for hash, height := range hashToHeight { + key := _hashToHeightIndexKey(&hash) + if err := DBSetWithTxn(txn, snap, key, UintToBuf(uint64(height)), eventManager); err != nil { + return errors.Wrap(err, "PutHashToHeightBatch: Problem setting hash to height") + } + } + return nil + }) +} + +func GetHeightForHash(db *badger.DB, snap *Snapshot, hash *BlockHash) (uint64, error) { + var height uint64 + err := db.View(func(txn *badger.Txn) error { + key := _hashToHeightIndexKey(hash) + heightBytes, err := DBGetWithTxn(txn, snap, key) + if err != nil { + return err + } + height = DecodeUint64(heightBytes) + return nil + }) + if err != nil { + return 0, err + } + return height, nil +} + func PutHeightHashToNodeInfoBatch(handle *badger.DB, snap *Snapshot, nodes []*BlockNode, bitcoinNodes bool, eventManager *EventManager) error { @@ -5551,9 +5596,47 @@ func GetBlockIndex(handle *badger.DB, bitcoinNodes bool, params *DeSoParams) ( return blockIndex, nil } +func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager *EventManager) error { + return handle.View(func(txn *badger.Txn) error { + prefix := _heightHashToNodeIndexPrefix(false) + opts := badger.DefaultIteratorOptions + opts.Prefix = prefix + // We don't need values for this migration. + opts.PrefetchValues = false + nodeIterator := txn.NewIterator(opts) + defer nodeIterator.Close() + hashToHeightMap := make(map[BlockHash]uint32) + for nodeIterator.Seek(prefix); nodeIterator.ValidForPrefix(prefix); nodeIterator.Next() { + item := nodeIterator.Item().Key() + + // Parse the key to get the height and hash. + height := binary.BigEndian.Uint32(item[1:5]) + hash := BlockHash{} + copy(hash[:], item[5:]) + hashToHeightMap[hash] = height + if len(hashToHeightMap) < 10000 { + continue + } + innerErr := PutHashToHeightBatch(handle, snapshot, hashToHeightMap, eventManager) + if innerErr != nil { + return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") + } + hashToHeightMap = make(map[BlockHash]uint32) + } + if len(hashToHeightMap) > 0 { + innerErr := PutHashToHeightBatch(handle, snapshot, hashToHeightMap, eventManager) + if innerErr != nil { + return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") + } + } + return nil + }) +} + func GetBestChain(tipNode *BlockNode) ([]*BlockNode, error) { reversedBestChain := []*BlockNode{} - for tipNode != nil { + maxBestChainInitLength := 3600 * 100 // Cache up to 100 hours of blocks. + for tipNode != nil && len(reversedBestChain) < maxBestChainInitLength { if (tipNode.Status&StatusBlockValidated) == 0 && (tipNode.Status&StatusBitcoinHeaderValidated) == 0 { diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index ff433b196..6adb79f12 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -55,7 +55,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // If the incoming header is already part of the best header chain, then we can exit early. // The header is not part of a fork, and is already an ancestor of the current header chain tip. - if _, isInBestHeaderChain := bc.bestHeaderChainMap[*headerHash]; isInBestHeaderChain { + if _, isInBestHeaderChain := bc.bestHeaderChain.GetBlockByHashAndHeight(headerHash, header.Height); isInBestHeaderChain { return true, false, nil } @@ -91,9 +91,9 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentTip, blockNode) - bc.bestHeaderChain, bc.bestHeaderChainMap = updateBestChainInMemory( - bc.bestHeaderChain, - bc.bestHeaderChainMap, + bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + bc.bestHeaderChain.Chain, + bc.bestHeaderChain.ChainMap, blocksToDetach, blocksToAttach, ) @@ -1639,8 +1639,7 @@ func (bc *Blockchain) shouldReorg(blockNode *BlockNode, currentView uint64) bool // addTipBlockToBestChain adds the block as the new tip of the best chain. func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { - bc.bestChain = append(bc.bestChain, blockNode) - bc.bestChainMap[*blockNode.Hash] = blockNode + bc.bestChain.PushNewTip(blockNode) } // removeTipBlockFromBestChain removes the current tip from the best chain. It @@ -1649,9 +1648,9 @@ func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { // the bestChain slice and bestChainMap map. func (bc *Blockchain) removeTipBlockFromBestChain() *BlockNode { // Remove the last block from the best chain. - lastBlock := bc.bestChain[len(bc.bestChain)-1] - delete(bc.bestChainMap, *lastBlock.Hash) - bc.bestChain = bc.bestChain[:len(bc.bestChain)-1] + lastBlock := bc.bestChain.Chain[len(bc.bestChain.Chain)-1] + bc.bestChain.ChainMap.Delete(*lastBlock.Hash) + bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] return lastBlock } @@ -1674,9 +1673,9 @@ func (bc *Blockchain) runCommitRuleOnBestChain(verifySignatures bool) error { return errors.New("runCommitRuleOnBestChain: No committed blocks found") } uncommittedAncestors := []*BlockNode{} - for ii := idx + 1; ii < len(bc.bestChain); ii++ { - uncommittedAncestors = append(uncommittedAncestors, bc.bestChain[ii]) - if bc.bestChain[ii].Hash.IsEqual(blockToCommit) { + for ii := idx + 1; ii < len(bc.bestChain.Chain); ii++ { + uncommittedAncestors = append(uncommittedAncestors, bc.bestChain.Chain[ii]) + if bc.bestChain.Chain[ii].Hash.IsEqual(blockToCommit) { break } } @@ -1698,8 +1697,16 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen ) { // TODO: Is it sufficient that the current block's header points to the parent // or does it need to have something to do with the QC? - parent := bc.bestChainMap[*currentBlock.Header.PrevBlockHash] - grandParent := bc.bestChainMap[*parent.Header.PrevBlockHash] + parent, exists := bc.bestChain.GetBlockByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + if !exists { + glog.Errorf("canCommitGrandparent: Parent block %v not found in best chain map", currentBlock.Header.PrevBlockHash.String()) + return nil, false + } + grandParent, exists := bc.bestChain.GetBlockByHashAndHeight(parent.Header.PrevBlockHash, uint64(parent.Height-1)) + if !exists { + glog.Errorf("canCommitGrandparent: Grandparent block %v not found in best chain map", parent.Header.PrevBlockHash.String()) + return nil, false + } if grandParent.IsCommitted() { return nil, false } @@ -1715,7 +1722,7 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen // to the DB and updates relevant badger indexes with info about the block. func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, blockHeight uint64, verifySignatures bool) error { // block must be in the best chain. we grab the block node from there. - blockNode, exists := bc.bestChainMap[*blockHash] + blockNode, exists := bc.bestChain.GetBlockByHashAndHeight(blockHash, blockHeight) if !exists { return errors.Errorf("commitBlockPoS: Block %v not found in best chain map", blockHash.String()) } @@ -1844,7 +1851,10 @@ func (bc *Blockchain) GetUncommittedBlocks(tipHash *BlockHash) ([]*BlockNode, er } bc.ChainLock.RLock() defer bc.ChainLock.RUnlock() - tipBlock, exists := bc.bestChainMap[*tipHash] + tipBlock, exists, err := bc.bestChain.GetBlockByHash(tipHash) + if err != nil { + return nil, errors.Wrapf(err, "GetUncommittedBlocks: Problem getting block %v", tipHash.String()) + } if !exists { return nil, errors.Errorf("GetUncommittedBlocks: Block %v not found in best chain map", tipHash.String()) } @@ -2015,9 +2025,9 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc // GetCommittedTip returns the highest committed block and its index in the best chain. func (bc *Blockchain) GetCommittedTip() (*BlockNode, int) { - for ii := len(bc.bestChain) - 1; ii >= 0; ii-- { - if bc.bestChain[ii].IsCommitted() { - return bc.bestChain[ii], ii + for ii := len(bc.bestChain.Chain) - 1; ii >= 0; ii-- { + if bc.bestChain.Chain[ii].IsCommitted() { + return bc.bestChain.Chain[ii], ii } } return nil, -1 diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 2bd140003..64daa127c 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -251,7 +251,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.bestChain = []*BlockNode{genesisBlock} + bc.bestChain.Chain = []*BlockNode{genesisBlock} bc.blockIndex.blockIndexByHash.Put(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) @@ -301,7 +301,7 @@ func TestHasValidBlockHeight(t *testing.T) { require.Equal(t, err, RuleErrorInvalidPoSBlockHeight) block.Header.Height = 2 - bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot, nil) + bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot) err = bc.hasValidBlockHeightPoS(block.Header) require.Equal(t, err, RuleErrorMissingParentBlock) } @@ -471,7 +471,7 @@ func TestHasValidBlockViewPoS(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.bestChain = []*BlockNode{ + bc.bestChain.Chain = []*BlockNode{ genesisNode, block2, } @@ -807,7 +807,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { Height: 1, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.bestChain = []*BlockNode{genesisNode} + bc.bestChain.Chain = []*BlockNode{genesisNode} bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) @@ -844,7 +844,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 2, PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.bestChain = append(bc.bestChain, block2) + bc.bestChain.Chain = append(bc.bestChain.Chain, block2) bc.blockIndex.blockIndexByHash.Put(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) @@ -1235,7 +1235,7 @@ func TestShouldReorg(t *testing.T) { hash1 := NewBlockHash(RandomBytes(32)) hash2 := NewBlockHash(RandomBytes(32)) hash3 := NewBlockHash(RandomBytes(32)) - bc.bestChain = []*BlockNode{ + bc.bestChain.Chain = []*BlockNode{ { Hash: hash1, Status: StatusBlockStored | StatusBlockValidated | StatusBlockCommitted, @@ -1249,7 +1249,7 @@ func TestShouldReorg(t *testing.T) { newBlock := &BlockNode{ Header: &MsgDeSoHeader{ ProposedInView: 2, - PrevBlockHash: bc.bestChain[1].Hash, + PrevBlockHash: bc.bestChain.Chain[1].Hash, }, } @@ -1335,7 +1335,7 @@ func TestTryApplyNewTip(t *testing.T) { ancestors, _, err := bc.getStoredLineageFromCommittedTip(newBlock.Header) require.NoError(t, err) checkBestChainForHash := func(hash *BlockHash) bool { - return collections.Any(bc.bestChain, func(bn *BlockNode) bool { + return collections.Any(bc.bestChain.Chain, func(bn *BlockNode) bool { return bn.Hash.IsEqual(hash) }) } @@ -1349,30 +1349,30 @@ func TestTryApplyNewTip(t *testing.T) { require.NoError(t, err) require.True(t, appliedNewTip) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap := bc.bestChainMap[*hash3] + _, hash3ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash3) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) require.Len(t, connectedBlockHashes, 1) require.Len(t, disconnectedBlockHashes, 1) // newBlock should be in the best chain and the best chain map and should be the tip. - _, newBlockExistsInBestChainMap := bc.bestChainMap[*newBlockHash] + _, newBlockExistsInBestChainMap := bc.bestChain.ChainMap.Get(*newBlockHash) require.True(t, newBlockExistsInBestChainMap) require.True(t, checkBestChainForHash(newBlockHash)) require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // Make sure block 2 and block 1 are still in the best chain. - _, hash2ExistsInBestChainMap := bc.bestChainMap[*hash2] + _, hash2ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash2) require.True(t, hash2ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash2)) - _, hash1ExistsInBestChainMap := bc.bestChainMap[*hash1] + _, hash1ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash1) require.True(t, hash1ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash1)) // Remove newBlock from the best chain and block index to reset the state. - bc.bestChain = bc.bestChain[:len(bc.bestChain)-1] - delete(bc.bestChainMap, *newBlockHash) + bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] + bc.bestChain.ChainMap.Delete(*newBlockHash) // Add block 3 back bc.addTipBlockToBestChain(bn3) @@ -1420,19 +1420,19 @@ func TestTryApplyNewTip(t *testing.T) { // newBlockHash should be tip. require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap = bc.bestChainMap[*hash3] + _, hash3ExistsInBestChainMap = bc.bestChain.ChainMap.Get(*hash3) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) // hash 2 should no longer be in the best chain or best chain map - _, hash2ExistsInBestChainMap = bc.bestChainMap[*hash2] + _, hash2ExistsInBestChainMap = bc.bestChain.ChainMap.Get(*hash2) require.False(t, hash2ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash2)) // hash 4 should be in the best chain and the best chain map - _, hash4ExistsInBestChainMap := bc.bestChainMap[*hash4] + _, hash4ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash4) require.True(t, hash4ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash4)) // hash 5 should be in the best chain and the best chain map - _, hash5ExistsInBestChainMap := bc.bestChainMap[*hash5] + _, hash5ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash5) require.True(t, hash5ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash5)) @@ -1441,10 +1441,10 @@ func TestTryApplyNewTip(t *testing.T) { require.Len(t, disconnectedBlockHashes, 2) // Reset the state of the best chain. - delete(bc.bestChainMap, *hash4) - delete(bc.bestChainMap, *hash5) - delete(bc.bestChainMap, *newBlockHash) - bc.bestChain = bc.bestChain[:len(bc.bestChain)-3] + bc.bestChain.ChainMap.Delete(*hash4) + bc.bestChain.ChainMap.Delete(*hash5) + bc.bestChain.ChainMap.Delete(*newBlockHash) + bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] // Add block 2 and 3 back. bc.addTipBlockToBestChain(bn2) @@ -1514,8 +1514,8 @@ func TestCanCommitGrandparent(t *testing.T) { PrevBlockHash: hash1, }, } - bc.bestChainMap[*hash1] = bn1 - bc.bestChainMap[*hash2] = bn2 + bc.bestChain.ChainMap.Put(*hash1, bn1) + bc.bestChain.ChainMap.Put(*hash2, bn2) // define incoming block hash3 := NewBlockHash(RandomBytes(32)) @@ -1657,7 +1657,7 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, committedHash := range committedBlocks { // Okay so let's make sure the block is committed. - blockNode, exists := testMeta.chain.bestChainMap[*committedHash] + blockNode, exists := testMeta.chain.bestChain.ChainMap.Get(*committedHash) require.True(testMeta.t, exists) require.True(testMeta.t, blockNode.IsCommitted()) @@ -1682,7 +1682,7 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, uncommittedBlockHash := range uncommittedBlocks { // Okay so let's make sure the block is uncommitted. - blockNode, exists := testMeta.chain.bestChainMap[*uncommittedBlockHash] + blockNode, exists := testMeta.chain.bestChain.ChainMap.Get(*uncommittedBlockHash) require.True(testMeta.t, exists) require.False(testMeta.t, blockNode.IsCommitted()) // TODO: Verify DB results?? Kinda silly to make sure everything is missing. @@ -1874,7 +1874,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { // Timeout block will no longer be in best chain, and will still be in an uncommitted state in the block index _verifyCommitRuleHelper(testMeta, []*BlockHash{blockHash1, blockHash2}, []*BlockHash{blockHash3, reorgBlockHash}, blockHash2) _verifyRandomSeedHashHelper(testMeta, reorgBlock) - _, exists := testMeta.chain.bestChainMap[*timeoutBlockHash] + _, exists := testMeta.chain.bestChain.ChainMap.Get(*timeoutBlockHash) require.False(t, exists) timeoutBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(timeoutBlockHash, timeoutBlockHeight) diff --git a/lib/server.go b/lib/server.go index 21756a665..debcd2b3e 100644 --- a/lib/server.go +++ b/lib/server.go @@ -861,7 +861,8 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } // Go through the block nodes in the blockchain and download the blocks if they're not stored. - for _, blockNode := range srv.blockchain.bestChain { + // TODO: need to figure out a way to get all the blocks in the best chain so we can download historical blocks. + for _, blockNode := range srv.blockchain.bestChain.Chain { // We find the first block that's not stored and get ready to download blocks starting from this block onwards. if blockNode.Status&StatusBlockStored == 0 { maxBlocksInFlight := MaxBlocksInFlight @@ -876,8 +877,8 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { blockNodesToFetch := []*BlockNode{} // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. var heightLimit int - for heightLimit = len(srv.blockchain.bestChain) - 1; heightLimit >= 0; heightLimit-- { - if !srv.blockchain.bestChain[heightLimit].Status.IsFullyProcessed() { + for heightLimit = len(srv.blockchain.bestChain.Chain) - 1; heightLimit >= 0; heightLimit-- { + if !srv.blockchain.bestChain.Chain[heightLimit].Status.IsFullyProcessed() { break } } @@ -888,7 +889,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to // the array index. - currentNode := srv.blockchain.bestChain[currentHeight] + currentNode := srv.blockchain.bestChain.Chain[currentHeight] currentHeight++ // If we've already requested this block then we don't request it again. @@ -1015,9 +1016,11 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b var hasSeenCheckpointBlockHash bool var checkpointBlockNode *BlockNode if isHeaderChain { - checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestHeaderChainMap[*checkpointBlockInfo.Hash] + checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestHeaderChain.GetBlockByHashAndHeight( + checkpointBlockInfo.Hash, checkpointBlockInfo.Height) } else { - checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestChainMap[*checkpointBlockInfo.Hash] + checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestChain.GetBlockByHashAndHeight( + checkpointBlockInfo.Hash, checkpointBlockInfo.Height) } // If we haven't seen the checkpoint block hash yet, we skip signature verification. if !hasSeenCheckpointBlockHash { @@ -1047,9 +1050,11 @@ func (srv *Server) getCheckpointSyncingStatus(isHeaders bool) string { } hasSeenCheckPointBlockHash := false if isHeaders { - _, hasSeenCheckPointBlockHash = srv.blockchain.bestHeaderChainMap[*checkpointBlockInfo.Hash] + _, hasSeenCheckPointBlockHash = srv.blockchain.bestHeaderChain.GetBlockByHashAndHeight( + checkpointBlockInfo.Hash, checkpointBlockInfo.Height) } else { - _, hasSeenCheckPointBlockHash = srv.blockchain.bestChainMap[*checkpointBlockInfo.Hash] + _, hasSeenCheckPointBlockHash = srv.blockchain.bestChain.GetBlockByHashAndHeight( + checkpointBlockInfo.Hash, checkpointBlockInfo.Height) } if !hasSeenCheckPointBlockHash { return fmt.Sprintf("", checkpointBlockInfo.String()) @@ -1234,11 +1239,21 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // expected height at which the snapshot should be taking place. We do this to make sure that the // snapshot we receive from the peer is up-to-date. // TODO: error handle if the hash doesn't exist for some reason. + expectedSnapshotHeightBlock, expectedSnapshotHeightblockExists, err := + srv.blockchain.bestHeaderChain.GetBlockByHeight(expectedSnapshotHeight) + if err != nil { + glog.Errorf("Server._handleHeaderBundle: Problem getting expected snapshot height block, error (%v)", err) + return + } + if !expectedSnapshotHeightblockExists || expectedSnapshotHeightBlock == nil { + glog.Errorf("Server._handleHeaderBundle: Expected snapshot height block doesn't exist.") + return + } srv.HyperSyncProgress.SnapshotMetadata = &SnapshotEpochMetadata{ SnapshotBlockHeight: expectedSnapshotHeight, FirstSnapshotBlockHeight: expectedSnapshotHeight, CurrentEpochChecksumBytes: []byte{}, - CurrentEpochBlockHash: srv.blockchain.bestHeaderChain[expectedSnapshotHeight].Hash, + CurrentEpochBlockHash: expectedSnapshotHeightBlock.Hash, } srv.HyperSyncProgress.PrefixProgress = []*SyncPrefixProgress{} srv.HyperSyncProgress.Completed = false @@ -1646,9 +1661,17 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { srv.snapshot.PrintChecksum("Finished hyper sync. Checksum is:") glog.Infof(CLog(Magenta, fmt.Sprintf("Metadata checksum: (%v)", srv.HyperSyncProgress.SnapshotMetadata.CurrentEpochChecksumBytes))) - + blockNode, exists, err := srv.blockchain.bestHeaderChain.GetBlockByHeight(msg.SnapshotMetadata.SnapshotBlockHeight) + if err != nil { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error (%v)", err) + return + } + if !exists { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") + return + } glog.Infof(CLog(Yellow, fmt.Sprintf("Best header chain %v best block chain %v", - srv.blockchain.bestHeaderChain[msg.SnapshotMetadata.SnapshotBlockHeight], srv.blockchain.bestChain))) + blockNode, srv.blockchain.bestChain.Chain))) // Verify that the state checksum matches the one in HyperSyncProgress snapshot metadata. // If the checksums don't match, it means that we've been interacting with a peer that was misbehaving. @@ -1691,14 +1714,21 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { // being too large and possibly causing an error in badger. var blockNodeBatch []*BlockNode for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { - currentNode := srv.blockchain.bestHeaderChain[ii] + currentNode, currentNodeExists, err := srv.blockchain.bestHeaderChain.GetBlockByHeight(ii) + if err != nil { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) + break + } + if !currentNodeExists { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") + break + } // Do not set the StatusBlockStored flag, because we still need to download the past blocks. currentNode.Status |= StatusBlockProcessed currentNode.Status |= StatusBlockValidated currentNode.Status |= StatusBlockCommitted srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) - srv.blockchain.bestChainMap[*currentNode.Hash] = currentNode - srv.blockchain.bestChain = append(srv.blockchain.bestChain, currentNode) + srv.blockchain.bestChain.PushNewTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) if len(blockNodeBatch) < 10000 { continue diff --git a/lib/state_change_syncer.go b/lib/state_change_syncer.go index e825dfbb6..0172e30f9 100644 --- a/lib/state_change_syncer.go +++ b/lib/state_change_syncer.go @@ -777,7 +777,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser return true, nil } - blockHeight := uint64(server.blockchain.bestChain[len(server.blockchain.bestChain)-1].Height) + blockHeight := uint64(server.blockchain.bestChain.GetTip().Height) stateChangeSyncer.MempoolFlushId = originalCommittedFlushId @@ -804,7 +804,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser mempoolUtxoView.Snapshot = nil server.blockchain.ChainLock.RLock() - mempoolUtxoView.TipHash = server.blockchain.bestChain[len(server.blockchain.bestChain)-1].Hash + mempoolUtxoView.TipHash = server.blockchain.bestChain.GetTip().Hash server.blockchain.ChainLock.RUnlock() // A new transaction is created so that we can simulate writes to the db without actually writing to the db. @@ -815,7 +815,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser defer txn.Discard() glog.V(2).Infof("Time since mempool sync start: %v", time.Since(startTime)) startTime = time.Now() - err = mempoolUtxoView.FlushToDbWithTxn(txn, uint64(server.blockchain.bestChain[len(server.blockchain.bestChain)-1].Height)) + err = mempoolUtxoView.FlushToDbWithTxn(txn, uint64(server.blockchain.bestChain.GetTip().Height)) if err != nil { mempoolUtxoView.EventManager.stateSyncerFlushed(&StateSyncerFlushedEvent{ FlushId: originalCommittedFlushId, diff --git a/lib/txindex.go b/lib/txindex.go index 5fe9cb515..3c1a1f78b 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -371,8 +371,8 @@ func (txi *TXIndex) Update() error { newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() newBestChain, newBestChainMap := txi.TXIndexChain.CopyBestChain() newBestChain = newBestChain[:len(newBestChain)-1] - delete(newBestChainMap, *(blockToDetach.Hash)) - newBlockIndex.Delete(*(blockToDetach.Hash)) + newBestChainMap.Delete(*blockToDetach.Hash) + newBlockIndex.Delete(*blockToDetach.Hash) txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndex) From a2e5b4bebcca69420591353ee8d5c321fefd4dcb Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Sep 2024 15:05:32 -0400 Subject: [PATCH 03/62] check bestHeaderChain.ChainMap ONLY when processing pos headers --- lib/pos_blockchain.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 6adb79f12..1b8e1d082 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -55,7 +55,10 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // If the incoming header is already part of the best header chain, then we can exit early. // The header is not part of a fork, and is already an ancestor of the current header chain tip. - if _, isInBestHeaderChain := bc.bestHeaderChain.GetBlockByHashAndHeight(headerHash, header.Height); isInBestHeaderChain { + // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly + // updated. This is necessary because the block index may have been updated with the header but the + // bestHeaderChain.ChainMap may not have been updated yet. + if _, isInBestHeaderChain := bc.bestHeaderChain.ChainMap.Get(*headerHash); isInBestHeaderChain { return true, false, nil } From ff6ee241c08bf2a643c8fbfce21c59bcbc3d9e9a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Sep 2024 17:49:20 -0400 Subject: [PATCH 04/62] updates for locating headers and faster cache usage --- lib/blockchain.go | 63 +++++++++++++++++++++++++++++++++++++++++------ lib/constants.go | 2 ++ lib/server.go | 15 +++++++++++ 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index f1e35d305..d0ed35a09 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -61,7 +61,7 @@ const ( // have room for multiple forks each an entire history's length with this value). If // each node takes up 100 bytes of space this amounts to around 500MB, which also seems // like a reasonable size. - MaxBlockIndexNodes = 5000000 + MaxBlockIndexNodes = 50000000 // TODO: trim this down somehow... ) type BlockStatus uint32 @@ -533,7 +533,6 @@ type BlockIndex struct { db *badger.DB snapshot *Snapshot blockIndexByHash *lru2.Map[BlockHash, *BlockNode] - tipNode *BlockNode } func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { @@ -552,10 +551,6 @@ func newBlockIndexByHashFromMap(input map[BlockHash]*BlockNode) *lru2.Map[BlockH return newMap } -func (bi *BlockIndex) setTipNode(tipNode *BlockNode) { - bi.tipNode = tipNode -} - func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) } @@ -572,6 +567,7 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem getting height for hash") } + // TODO: cache current height to exit early? blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if blockNode == nil { return nil, false, nil @@ -588,6 +584,7 @@ func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height u if height > math.MaxUint32 { glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) } + // TODO: cache current height to exit early? bn := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false @@ -600,6 +597,7 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { if height > math.MaxUint32 { glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) } + // TODO: cache current height to exit early? prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false) blockNodes := []*BlockNode{} @@ -650,6 +648,33 @@ func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, e return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) } + if height > uint64(bestChain.GetTip().Height) { + return nil, false, nil + } + + currTip := bestChain.GetTip() + if currTip != nil { + currNode := &BlockNode{} + *currNode = *currTip + if currNode.Height == uint32(height) { + return currNode, true, nil + } + for currNode != nil && !currNode.IsCommitted() { + if currNode.Height < uint32(height) { + break + } + if currNode.Height == uint32(height) { + return currNode, true, nil + } + var currNodeExists bool + currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( + currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) + if !currNodeExists { + break + } + } + } + prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) if len(valsFound) == 0 { @@ -678,6 +703,9 @@ func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height if height > math.MaxUint32 { glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) } + if height > uint64(bestChain.GetTip().Height) { + return nil, false + } bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false @@ -699,10 +727,16 @@ func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bo } return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") } + if height > uint64(bestChain.GetTip().Height) { + return nil, false, nil + } bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false, nil } + if !bn.IsCommitted() { + return nil, false, nil + } bestChain.ChainMap.Put(*blockHash, bn) return bn, true, nil } @@ -1465,9 +1499,14 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { } if exists { var innerExists bool - tip, innerExists = bc.bestHeaderChain.GetBlockByHashAndHeight(tip.Hash, uint64(height)) + tip, innerExists, err = bc.bestHeaderChain.GetBlockByHeight(uint64(height)) + if err != nil { + glog.Errorf("LatestLocator: Problem getting block by height: %v", err) + break + } if !innerExists { - glog.Errorf("LatestLocator: Block %v not found in best header chain", tip.Hash) + glog.Errorf("LatestLocator: Block %v not found in best header chain", height) + break } } else { tip = tip.Ancestor(uint32(height)) @@ -1896,6 +1935,14 @@ func (bc *Blockchain) BestChain() []*BlockNode { return bc.bestChain.Chain } +func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash) (*BlockNode, bool, error) { + return bc.bestChain.GetBlockByHash(blockHash) +} + +func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64) (*BlockNode, bool, error) { + return bc.bestChain.GetBlockByHeight(height) +} + func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { bc.bestChain.Chain = bestChain } diff --git a/lib/constants.go b/lib/constants.go index e9001c8c3..5629a50a1 100644 --- a/lib/constants.go +++ b/lib/constants.go @@ -1955,3 +1955,5 @@ const DefaultMainnetCheckpointProvider = "https://node.deso.org" const DefaultTestnetCheckpointProvider = "https://test.deso.org" const RoutePathGetCommittedTipBlockInfo = "/api/v0/get-committed-tip-block-info" + +const BlockIndexMigrationFileName = "block_index_migration.txt" diff --git a/lib/server.go b/lib/server.go index debcd2b3e..35eafe4be 100644 --- a/lib/server.go +++ b/lib/server.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "net" + "path/filepath" "reflect" "runtime" "strings" @@ -363,6 +364,16 @@ func ValidateHyperSyncFlags(isHypersync bool, syncType NodeSyncType) { } } +func RunBlockIndexMigrationOnce(db *badger.DB, dataDir string) error { + glog.V(0).Info("Running block index migration") + blockIndexMigrationFileName := filepath.Join(dataDir, BlockIndexMigrationFileName) + hasRunMigration, err := ReadBoolFromFile(blockIndexMigrationFileName) + if err == nil && hasRunMigration { + return nil + } + return RunBlockIndexMigration(db, nil, nil) +} + // NewServer initializes all of the internal data structures. Right now this basically // looks as follows: // - ConnectionManager starts and keeps track of peers. @@ -439,6 +450,10 @@ func NewServer( _shouldRestart bool, ) { + if err := RunBlockIndexMigrationOnce(_db, _dataDir); err != nil { + return nil, errors.Wrapf(err, "NewServer: Problem running block index migration"), true + } + var err error // Only initialize state change syncer if the directories are defined. From 71b1eaa9a78729c0d3a12e95619ca240ff7e82a2 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 25 Sep 2024 17:50:00 -0400 Subject: [PATCH 05/62] more fixes --- lib/blockchain.go | 108 +++++++++++++++++++++++++++++------------- lib/db_utils.go | 2 +- lib/pos_blockchain.go | 2 +- lib/server.go | 24 +++++++--- lib/txindex.go | 2 +- 5 files changed, 94 insertions(+), 44 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index d0ed35a09..ab7f24b9d 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -467,6 +467,7 @@ func CalcNextDifficultyTarget( maxRetargetTimeSecs := targetSecs * params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget + // TODO: this needs to be replaced with a call to GetBlockNodeByHeight firstNode := lastNode.Ancestor(firstNodeHeight) if firstNode == nil { return nil, fmt.Errorf("CalcNextDifficultyTarget: Problem getting block at "+ @@ -619,15 +620,17 @@ type BestChain struct { IsHeaderChain bool Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. ChainMap *lru2.Map[BlockHash, *BlockNode] + params *DeSoParams } -func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool) *BestChain { +func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { return &BestChain{ db: db, snapshot: snapshot, IsHeaderChain: isHeaderChain, Chain: []*BlockNode{}, ChainMap: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), + params: params, } } @@ -648,31 +651,42 @@ func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, e return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) } - if height > uint64(bestChain.GetTip().Height) { + currTip := bestChain.GetTip() + if currTip != nil && height > uint64(currTip.Height) { return nil, false, nil } - currTip := bestChain.GetTip() if currTip != nil { - currNode := &BlockNode{} - *currNode = *currTip - if currNode.Height == uint32(height) { - return currNode, true, nil - } - for currNode != nil && !currNode.IsCommitted() { - if currNode.Height < uint32(height) { - break - } - if currNode.Height == uint32(height) { - return currNode, true, nil - } - var currNodeExists bool - currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( - currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) - if !currNodeExists { - break + currTipHeight := currTip.Height + delta := currTipHeight - uint32(height) + if delta < uint32(len(bestChain.Chain)) { + targetNode := bestChain.Chain[uint32(len(bestChain.Chain)-1)-delta] + if uint64(targetNode.Height) == height { + return targetNode, true, nil } } + //targetIndex := (len(bestChain.Chain)-1) + //// We can probably do some binary search thing here instead. + //currNode := &BlockNode{} + //*currNode = *currTip + //if currNode.Height == uint32(height) { + // return currNode, true, nil + //} + //// During syncing, we don't write to the DB, so we need to iterate backwards through the best chain. + //for currNode != nil && !currNode.IsStored() { + // if currNode.Height < uint32(height) { + // break + // } + // if currNode.Height == uint32(height) { + // return currNode, true, nil + // } + // var currNodeExists bool + // currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( + // currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) + // if !currNodeExists { + // break + // } + //} } prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) @@ -1160,7 +1174,7 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// currentHeaderTip := bc.headerTip() - _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) + _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode, bc.blockIndex) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, @@ -1219,9 +1233,9 @@ func NewBlockchain( //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), //bestChainMap: make(map[BlockHash]*BlockNode), - bestChain: NewBestChain(db, snapshot, false), + bestChain: NewBestChain(db, snapshot, false, params), //bestHeaderChainMap: make(map[BlockHash]*BlockNode), - bestHeaderChain: NewBestChain(db, snapshot, true), + bestHeaderChain: NewBestChain(db, snapshot, true, params), blockViewCache: lru.NewKVCache(100), // TODO: parameterize snapshotCache: NewSnapshotCache(), @@ -1509,6 +1523,7 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { break } } else { + // TODO: this needs to be replaced with a read-through cache call. tip = tip.Ancestor(uint32(height)) } @@ -2099,7 +2114,8 @@ func (bc *Blockchain) MarkBlockInvalid(node *BlockNode, errOccurred RuleError) { //} } -func _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode) *BlockNode { +// node1 is the current tip and node2 is a new node. +func _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, blockIndex *BlockIndex) *BlockNode { if node1 == nil || node2 == nil { // If either node is nil then there can't be a common ancestor. return nil @@ -2107,18 +2123,33 @@ func _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode) *BlockNode { // Get the two nodes to be at the same height. if node1.Height > node2.Height { - node1 = node1.Ancestor(node2.Height) + node1Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) + if !exists { + return nil + } + return _FindCommonAncestor(node1Parent, node2, blockIndex) } else if node1.Height < node2.Height { - node2 = node2.Ancestor(node1.Height) + node2Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) + if !exists { + return nil + } + return _FindCommonAncestor(node1, node2Parent, blockIndex) } // Iterate the nodes backward until they're either the same or we // reach the end of the lists. We only need to check node1 for nil // since they're the same height and we are iterating both back // in tandem. - for node1 != nil && !node1.Hash.IsEqual(node2.Hash) { - node1 = node1.Parent - node2 = node2.Parent + if !node1.Hash.IsEqual(node2.Hash) { + node1Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) + if !exists { + return nil + } + node2Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) + if !exists { + return nil + } + return _FindCommonAncestor(node1Parent, node2Parent, blockIndex) } // By now either node1 == node2 and we found the common ancestor or @@ -2198,9 +2229,10 @@ func CheckTransactionSanity(txn *MsgDeSoTxn, blockHeight uint32, params *DeSoPar return nil } -func GetReorgBlocks(tip *BlockNode, newNode *BlockNode) (_commonAncestor *BlockNode, _detachNodes []*BlockNode, _attachNodes []*BlockNode) { +func GetReorgBlocks(tip *BlockNode, newNode *BlockNode, blockIndex *BlockIndex) ( + _commonAncestor *BlockNode, _detachNodes []*BlockNode, _attachNodes []*BlockNode) { // Find the common ancestor of this block and the main header chain. - commonAncestor := _FindCommonAncestor(tip, newNode) + commonAncestor := _FindCommonAncestor(tip, newNode, blockIndex) // Log a warning if the reorg is going to be a big one. numBlocks := tip.Height - commonAncestor.Height @@ -2229,8 +2261,16 @@ func GetReorgBlocks(tip *BlockNode, newNode *BlockNode) (_commonAncestor *BlockN // attachNodes will have the new node as its first element and work back to // the node right after the common ancestor as its last element. attachBlocks := []*BlockNode{} - for currentBlock := newNode; *currentBlock.Hash != *commonAncestor.Hash; currentBlock = currentBlock.Parent { + currentBlock := &BlockNode{} + *currentBlock = *newNode + for *currentBlock.Hash != *commonAncestor.Hash { attachBlocks = append(attachBlocks, currentBlock) + var exists bool + currentBlock, exists = blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + if !exists { + // TODO: what should we do here? + glog.Fatal("GetReorgBlocks: Failed to find parent of block") + } } // Reverse attachBlocks so that the node right after the common ancestor // will be the first element and the node at the end of the list will be @@ -2438,7 +2478,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - _, detachBlocks, attachBlocks := GetReorgBlocks(headerTip, newNode) + _, detachBlocks, attachBlocks := GetReorgBlocks(headerTip, newNode, bc.blockIndex) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) @@ -3003,7 +3043,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Find the common ancestor of this block and the main chain. // TODO: Reorgs with postgres? - commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(currentTip, nodeToValidate) + commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(currentTip, nodeToValidate, bc.blockIndex) // Log a warning if the reorg is going to be a big one. numBlocks := currentTip.Height - commonAncestor.Height if numBlocks > 10 { diff --git a/lib/db_utils.go b/lib/db_utils.go index ec9350f5a..e9520a44c 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5284,7 +5284,7 @@ func GetHeightForHash(db *badger.DB, snap *Snapshot, hash *BlockHash) (uint64, e if err != nil { return err } - height = DecodeUint64(heightBytes) + height, _ = Uvarint(heightBytes) return nil }) if err != nil { diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 1b8e1d082..a839ec850 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -93,7 +93,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. - _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentTip, blockNode) + _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentTip, blockNode, bc.blockIndex) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, diff --git a/lib/server.go b/lib/server.go index 35eafe4be..f19b91c3c 100644 --- a/lib/server.go +++ b/lib/server.go @@ -365,13 +365,21 @@ func ValidateHyperSyncFlags(isHypersync bool, syncType NodeSyncType) { } func RunBlockIndexMigrationOnce(db *badger.DB, dataDir string) error { - glog.V(0).Info("Running block index migration") blockIndexMigrationFileName := filepath.Join(dataDir, BlockIndexMigrationFileName) + glog.V(0).Info("FileName: ", blockIndexMigrationFileName) hasRunMigration, err := ReadBoolFromFile(blockIndexMigrationFileName) if err == nil && hasRunMigration { + glog.V(0).Info("Block index migration has already been run") return nil } - return RunBlockIndexMigration(db, nil, nil) + glog.V(0).Info("Running block index migration") + if err = RunBlockIndexMigration(db, nil, nil); err != nil { + return errors.Wrapf(err, "Problem running block index migration") + } + if err = SaveBoolToFile(blockIndexMigrationFileName, true); err != nil { + return errors.Wrapf(err, "Problem saving block index migration file") + } + return nil } // NewServer initializes all of the internal data structures. Right now this basically @@ -1405,6 +1413,8 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { glog.V(1).Infof("Server._handleHeaderBundle: *Syncing* headers for blocks starting at "+ "header tip %v out of %d from peer %v", headerTip.Header, msg.TipHeight, pp) + glog.V(0).Infof("Server._handleHeaderBundle: Num Headers in header chain: (chain map: %v) - (chain: %v) ", + srv.blockchain.bestHeaderChain.ChainMap.Len(), len(srv.blockchain.bestHeaderChain.Chain)) } func (srv *Server) _handleGetBlocks(pp *Peer, msg *MsgDeSoGetBlocks) { @@ -1682,12 +1692,12 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { return } if !exists { - glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") - return + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist: (%v)", msg.SnapshotMetadata.SnapshotBlockHeight) + //return + } else { + glog.Infof(CLog(Yellow, fmt.Sprintf("Best header chain %v best block chain %v", + blockNode, srv.blockchain.bestChain.Chain))) } - glog.Infof(CLog(Yellow, fmt.Sprintf("Best header chain %v best block chain %v", - blockNode, srv.blockchain.bestChain.Chain))) - // Verify that the state checksum matches the one in HyperSyncProgress snapshot metadata. // If the checksums don't match, it means that we've been interacting with a peer that was misbehaving. checksumBytes, err := srv.snapshot.Checksum.ToBytes() diff --git a/lib/txindex.go b/lib/txindex.go index 3c1a1f78b..36e8ac416 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -242,7 +242,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // At this point, we know our txindex tip is in our block index so // there must be a common ancestor between the tip and the block tip. - commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(&derefedTxindexTipNode, committedTip) + commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(&derefedTxindexTipNode, committedTip, txi.TXIndexChain.blockIndex) return txindexTipNode, committedTip, commonAncestor, detachBlocks, attachBlocks } From 17a56bd6ae64ad91b3c095a3de8c569f9ac2fa1e Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 14:44:51 -0400 Subject: [PATCH 06/62] more fixes --- lib/blockchain.go | 94 +++++++++++++++++++++++++++++++------------ lib/pos_blockchain.go | 2 +- lib/server.go | 13 +----- lib/txindex.go | 5 ++- 4 files changed, 74 insertions(+), 40 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index ab7f24b9d..004af0456 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1174,7 +1174,7 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// currentHeaderTip := bc.headerTip() - _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode, bc.blockIndex) + _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode, true) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, @@ -1644,6 +1644,14 @@ func (bc *Blockchain) HasHeader(headerHash *BlockHash) (bool, error) { return exists, errors.Wrap(err, "Blockchain.HasHeader: ") } +func (bc *Blockchain) HasHeaderByHashAndHeight(headerHash *BlockHash, height uint64) bool { + if height > uint64(bc.headerTip().Height) { + return false + } + _, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, height) + return exists +} + // TODO: delete me? func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) (*BlockNode, bool, error) { if blockHeight >= bc.bestChain.GetTip().Height { @@ -2114,42 +2122,62 @@ func (bc *Blockchain) MarkBlockInvalid(node *BlockNode, errOccurred RuleError) { //} } -// node1 is the current tip and node2 is a new node. -func _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, blockIndex *BlockIndex) *BlockNode { +// Note: we make some assumptions that we only care about ancestors in the best chain. +func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, isHeaderChain bool) *BlockNode { if node1 == nil || node2 == nil { // If either node is nil then there can't be a common ancestor. return nil } + chainToUse := &BestChain{} + if isHeaderChain { + chainToUse = bc.bestHeaderChain + } else { + chainToUse = bc.bestChain + } - // Get the two nodes to be at the same height. - if node1.Height > node2.Height { - node1Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) - if !exists { + committedTip, _ := bc.GetCommittedTip() + // If both nodes are at a height greater than the committed tip, then we know the + // committed tip is a common ancestor. + // TODO: this isn't necessarily the GREATEST common ancestor. It's just a common ancestor. + if node1.Height > committedTip.Height && node2.Height > committedTip.Height { + // If both nodes are at a height greater than the committed tip, then we know that + // we have valid parent pointers and can use the Ancestor function to get use to the right place. + if node1.Height > node2.Height { + node1 = node1.Ancestor(node2.Height) + } else if node2.Height > node1.Height { + node2 = node2.Ancestor(node1.Height) + } + } else { + var exists bool + var err error + // Get the two nodes to be at the same height. + if node1.Height > node2.Height { + node1, exists, err = chainToUse.GetBlockByHeight(uint64(node2.Height)) + } else if node1.Height < node2.Height { + node2, exists, err = chainToUse.GetBlockByHeight(uint64(node1.Height)) + } + if err != nil { return nil } - return _FindCommonAncestor(node1Parent, node2, blockIndex) - } else if node1.Height < node2.Height { - node2Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) if !exists { return nil } - return _FindCommonAncestor(node1, node2Parent, blockIndex) } // Iterate the nodes backward until they're either the same or we // reach the end of the lists. We only need to check node1 for nil // since they're the same height and we are iterating both back // in tandem. + var exists bool if !node1.Hash.IsEqual(node2.Hash) { - node1Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) + node1, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) if !exists { return nil } - node2Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) + node2, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) if !exists { return nil } - return _FindCommonAncestor(node1Parent, node2Parent, blockIndex) } // By now either node1 == node2 and we found the common ancestor or @@ -2229,17 +2257,24 @@ func CheckTransactionSanity(txn *MsgDeSoTxn, blockHeight uint32, params *DeSoPar return nil } -func GetReorgBlocks(tip *BlockNode, newNode *BlockNode, blockIndex *BlockIndex) ( +func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode, isHeaderChain bool) ( _commonAncestor *BlockNode, _detachNodes []*BlockNode, _attachNodes []*BlockNode) { // Find the common ancestor of this block and the main header chain. - commonAncestor := _FindCommonAncestor(tip, newNode, blockIndex) + commonAncestor := bc._FindCommonAncestor(tip, newNode, isHeaderChain) + + if commonAncestor == nil { + glog.Fatalf("No common ancestor found between tip and new node: tip hash (%v), newNode hash (%v)", tip.Hash, newNode.Hash) + return + } // Log a warning if the reorg is going to be a big one. - numBlocks := tip.Height - commonAncestor.Height - if numBlocks > 10 { - glog.Warningf("GetReorgBlocks: Proceeding with reorg of (%d) blocks from "+ - "block (%v) at height (%d) to block (%v) at height of (%d)", - numBlocks, tip, tip.Height, newNode, newNode.Height) + if tip != nil && commonAncestor != nil { + numBlocks := tip.Height - commonAncestor.Height + if numBlocks > 10 { + glog.Warningf("GetReorgBlocks: Proceeding with reorg of (%d) blocks from "+ + "block (%v) at height (%d) to block (%v) at height of (%d)", + numBlocks, tip, tip.Height, newNode, newNode.Height) + } } // Get the blocks to detach. Start at the tip and work backwards to the @@ -2249,7 +2284,14 @@ func GetReorgBlocks(tip *BlockNode, newNode *BlockNode, blockIndex *BlockIndex) // detachBlocks will have the current tip as its first element and parents // of the tip thereafter. detachBlocks := []*BlockNode{} - for currentBlock := tip; *currentBlock.Hash != *commonAncestor.Hash; currentBlock = currentBlock.Parent { + currentBlock := &BlockNode{} + *currentBlock = *tip + for currentBlock != nil && *currentBlock.Hash != *commonAncestor.Hash { + var exists bool + currentBlock, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + if !exists { + glog.Fatalf("GetReorgBlocks: Failed to find parent of block. Parent hash %v", currentBlock.Header.PrevBlockHash) + } detachBlocks = append(detachBlocks, currentBlock) } @@ -2261,12 +2303,12 @@ func GetReorgBlocks(tip *BlockNode, newNode *BlockNode, blockIndex *BlockIndex) // attachNodes will have the new node as its first element and work back to // the node right after the common ancestor as its last element. attachBlocks := []*BlockNode{} - currentBlock := &BlockNode{} + currentBlock = &BlockNode{} *currentBlock = *newNode for *currentBlock.Hash != *commonAncestor.Hash { attachBlocks = append(attachBlocks, currentBlock) var exists bool - currentBlock, exists = blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + currentBlock, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) if !exists { // TODO: what should we do here? glog.Fatal("GetReorgBlocks: Failed to find parent of block") @@ -2478,7 +2520,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - _, detachBlocks, attachBlocks := GetReorgBlocks(headerTip, newNode, bc.blockIndex) + _, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode, true) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) @@ -3043,7 +3085,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Find the common ancestor of this block and the main chain. // TODO: Reorgs with postgres? - commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(currentTip, nodeToValidate, bc.blockIndex) + commonAncestor, detachBlocks, attachBlocks := bc.GetReorgBlocks(currentTip, nodeToValidate, false) // Log a warning if the reorg is going to be a big one. numBlocks := currentTip.Height - commonAncestor.Height if numBlocks > 10 { diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index a839ec850..46a0ce78e 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -93,7 +93,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. - _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentTip, blockNode, bc.blockIndex) + _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode, true) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, diff --git a/lib/server.go b/lib/server.go index f19b91c3c..71f003c8b 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1114,12 +1114,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // have this issue. Hitting duplicates after we're done syncing is // fine and can happen in certain cases. headerHash, _ := headerReceived.Hash() - hasHeader, err := srv.blockchain.HasHeader(headerHash) - // TODO: what should we really do here? - if err != nil { - glog.Errorf("Server._handleHeaderBundle: Error checking if header %v exists: %v", headerHash, err) - return - } + hasHeader := srv.blockchain.HasHeaderByHashAndHeight(headerHash, headerReceived.Height) if hasHeader { if srv.blockchain.isSyncing() { @@ -1353,11 +1348,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // we're either not aware of or that we don't think is the best chain. // Doing things this way makes it so that when we request blocks we // are 100% positive the peer has them. - hasHeader, err := srv.blockchain.HasHeader(msg.TipHash) - if err != nil { - glog.Errorf("Server._handleHeaderBundle: Error checking if header %v exists: %v", msg.TipHash, err) - return - } + hasHeader := srv.blockchain.HasHeaderByHashAndHeight(msg.TipHash, uint64(msg.TipHeight)) if !hasHeader { glog.V(1).Infof("Server._handleHeaderBundle: Peer's tip is not in our "+ "blockchain so not requesting anything else from them. Our block "+ diff --git a/lib/txindex.go b/lib/txindex.go index 36e8ac416..4f2998d86 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -171,7 +171,8 @@ func (txi *TXIndex) Start() { txi.updateWaitGroup.Done() return default: - if txi.CoreChain.ChainState() == SyncStateFullyCurrent { + chainState := txi.CoreChain.ChainState() + if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && txi.CoreChain.headerTip().Height-txi.CoreChain.blockTip().Height < 10) { if !txi.CoreChain.IsFullyStored() { glog.V(1).Infof("TXIndex: Waiting, blockchain is not fully stored") break @@ -242,7 +243,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // At this point, we know our txindex tip is in our block index so // there must be a common ancestor between the tip and the block tip. - commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(&derefedTxindexTipNode, committedTip, txi.TXIndexChain.blockIndex) + commonAncestor, detachBlocks, attachBlocks := txi.CoreChain.GetReorgBlocks(&derefedTxindexTipNode, committedTip, false) return txindexTipNode, committedTip, commonAncestor, detachBlocks, attachBlocks } From d15392733a30711ebe78526a4f640dbac43f6e4e Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 16:02:00 -0400 Subject: [PATCH 07/62] more fix, specifically around .Ancestor --- lib/block_producer.go | 4 +- lib/blockchain.go | 107 ++++++++++++++++++----------------------- lib/blockchain_test.go | 35 +++++++++++--- lib/pos_blockchain.go | 2 +- lib/txindex.go | 2 +- 5 files changed, 79 insertions(+), 71 deletions(-) diff --git a/lib/block_producer.go b/lib/block_producer.go index c54f6771e..a52318102 100644 --- a/lib/block_producer.go +++ b/lib/block_producer.go @@ -368,8 +368,8 @@ func (desoBlockProducer *DeSoBlockProducer) _getBlockTemplate(publicKey []byte) blockRet.Header.TransactionMerkleRoot = merkleRoot // Compute the next difficulty target given the current tip. - diffTarget, err := CalcNextDifficultyTarget( - lastNode, CurrentHeaderVersion, desoBlockProducer.params) + diffTarget, err := desoBlockProducer.chain.CalcNextDifficultyTarget( + lastNode, CurrentHeaderVersion) if err != nil { return nil, nil, nil, errors.Wrapf(err, "DeSoBlockProducer._getBlockTemplate: Problem computing next difficulty: ") } diff --git a/lib/blockchain.go b/lib/blockchain.go index 004af0456..4de5b6988 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -406,7 +406,7 @@ func NewBlockNode( } } -func (nn *BlockNode) Ancestor(height uint32) *BlockNode { +func (nn *BlockNode) Ancestor(height uint32, blockIndex *BlockIndex) *BlockNode { if height > nn.Height { return nil } @@ -414,6 +414,14 @@ func (nn *BlockNode) Ancestor(height uint32) *BlockNode { node := nn for ; node != nil && node.Height != height; node = node.Parent { // Keep iterating node until the condition no longer holds. + if node.Parent == nil { + var exists bool + node.Parent, exists = blockIndex.GetBlockNodeByHashAndHeight( + node.Header.PrevBlockHash, uint64(node.Height-1)) + if !exists { + return nil + } + } } return node @@ -424,24 +432,24 @@ func (nn *BlockNode) Ancestor(height uint32) *BlockNode { // height minus provided distance. // // This function is safe for concurrent access. -func (nn *BlockNode) RelativeAncestor(distance uint32) *BlockNode { - return nn.Ancestor(nn.Height - distance) +func (nn *BlockNode) RelativeAncestor(distance uint32, blockIndex *BlockIndex) *BlockNode { + return nn.Ancestor(nn.Height-distance, blockIndex) } // CalcNextDifficultyTarget computes the difficulty target expected of the // next block. -func CalcNextDifficultyTarget( - lastNode *BlockNode, version uint32, params *DeSoParams) (*BlockHash, error) { +func (bc *Blockchain) CalcNextDifficultyTarget( + lastNode *BlockNode, version uint32) (*BlockHash, error) { // Compute the blocks in each difficulty cycle. - blocksPerRetarget := uint32(params.TimeBetweenDifficultyRetargets / params.TimeBetweenBlocks) + blocksPerRetarget := uint32(bc.params.TimeBetweenDifficultyRetargets / bc.params.TimeBetweenBlocks) // We effectively skip the first difficulty retarget by returning the default // difficulty value for the first cycle. Not doing this (or something like it) // would cause the genesis block's timestamp, which could be off by several days // to significantly skew the first cycle in a way that is mostly annoying for // testing but also suboptimal for the mainnet. - minDiffBytes, err := hex.DecodeString(params.MinDifficultyTargetHex) + minDiffBytes, err := hex.DecodeString(bc.params.MinDifficultyTargetHex) if err != nil { return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem computing min difficulty") } @@ -462,14 +470,19 @@ func CalcNextDifficultyTarget( } // If we get here it means we reached a difficulty retarget point. - targetSecs := int64(params.TimeBetweenDifficultyRetargets / time.Second) - minRetargetTimeSecs := targetSecs / params.MaxDifficultyRetargetFactor - maxRetargetTimeSecs := targetSecs * params.MaxDifficultyRetargetFactor + targetSecs := int64(bc.params.TimeBetweenDifficultyRetargets / time.Second) + minRetargetTimeSecs := targetSecs / bc.params.MaxDifficultyRetargetFactor + maxRetargetTimeSecs := targetSecs * bc.params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget // TODO: this needs to be replaced with a call to GetBlockNodeByHeight - firstNode := lastNode.Ancestor(firstNodeHeight) - if firstNode == nil { + firstNode, exists, err := bc.bestChain.GetBlockByHeight(uint64(firstNodeHeight)) + if err != nil { + return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem getting block at "+ + "beginning of retarget interval at height %d during retarget from height %d", + firstNodeHeight, lastNode.Height) + } + if !exists || firstNode == nil { return nil, fmt.Errorf("CalcNextDifficultyTarget: Problem getting block at "+ "beginning of retarget interval at height %d during retarget from height %d", firstNodeHeight, lastNode.Height) @@ -993,7 +1006,9 @@ func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Map[BlockHash, // IsFullyStored determines if there are block nodes that haven't been fully stored or processed in the best block chain. func (bc *Blockchain) IsFullyStored() bool { // TODO: figure out how to iterate over best chain w/o having entire thing in memory. - if bc.ChainState() == SyncStateFullyCurrent { + chainState := bc.ChainState() + if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && + bc.headerTip().Height-bc.blockTip().Height < 10) { for _, blockNode := range bc.bestChain.Chain { if !blockNode.Status.IsFullyProcessed() { return false @@ -1174,7 +1189,7 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// currentHeaderTip := bc.headerTip() - _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode, true) + _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, @@ -1523,8 +1538,7 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { break } } else { - // TODO: this needs to be replaced with a read-through cache call. - tip = tip.Ancestor(uint32(height)) + tip = tip.Ancestor(uint32(height), bc.blockIndex) } // Once 11 entries have been included, start doubling the @@ -2123,45 +2137,18 @@ func (bc *Blockchain) MarkBlockInvalid(node *BlockNode, errOccurred RuleError) { } // Note: we make some assumptions that we only care about ancestors in the best chain. -func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, isHeaderChain bool) *BlockNode { +func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode) *BlockNode { if node1 == nil || node2 == nil { // If either node is nil then there can't be a common ancestor. return nil } - chainToUse := &BestChain{} - if isHeaderChain { - chainToUse = bc.bestHeaderChain - } else { - chainToUse = bc.bestChain - } - committedTip, _ := bc.GetCommittedTip() - // If both nodes are at a height greater than the committed tip, then we know the - // committed tip is a common ancestor. - // TODO: this isn't necessarily the GREATEST common ancestor. It's just a common ancestor. - if node1.Height > committedTip.Height && node2.Height > committedTip.Height { - // If both nodes are at a height greater than the committed tip, then we know that - // we have valid parent pointers and can use the Ancestor function to get use to the right place. - if node1.Height > node2.Height { - node1 = node1.Ancestor(node2.Height) - } else if node2.Height > node1.Height { - node2 = node2.Ancestor(node1.Height) - } - } else { - var exists bool - var err error - // Get the two nodes to be at the same height. - if node1.Height > node2.Height { - node1, exists, err = chainToUse.GetBlockByHeight(uint64(node2.Height)) - } else if node1.Height < node2.Height { - node2, exists, err = chainToUse.GetBlockByHeight(uint64(node1.Height)) - } - if err != nil { - return nil - } - if !exists { - return nil - } + // If both nodes are at a height greater than the committed tip, then we know that + // we have valid parent pointers and can use the Ancestor function to get use to the right place. + if node1.Height > node2.Height { + node1 = node1.Ancestor(node2.Height, bc.blockIndex) + } else if node2.Height > node1.Height { + node2 = node2.Ancestor(node1.Height, bc.blockIndex) } // Iterate the nodes backward until they're either the same or we @@ -2169,7 +2156,7 @@ func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, is // since they're the same height and we are iterating both back // in tandem. var exists bool - if !node1.Hash.IsEqual(node2.Hash) { + for !node1.Hash.IsEqual(node2.Hash) { node1, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) if !exists { return nil @@ -2257,10 +2244,12 @@ func CheckTransactionSanity(txn *MsgDeSoTxn, blockHeight uint32, params *DeSoPar return nil } -func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode, isHeaderChain bool) ( +func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( _commonAncestor *BlockNode, _detachNodes []*BlockNode, _attachNodes []*BlockNode) { + // TODO: finding common ancestors is very expensive for txindex when txindex is very far + // behind. Currently, it requires loading the entire chain into memory. // Find the common ancestor of this block and the main header chain. - commonAncestor := bc._FindCommonAncestor(tip, newNode, isHeaderChain) + commonAncestor := bc._FindCommonAncestor(tip, newNode) if commonAncestor == nil { glog.Fatalf("No common ancestor found between tip and new node: tip hash (%v), newNode hash (%v)", tip.Hash, newNode.Hash) @@ -2268,7 +2257,7 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode, isHeade } // Log a warning if the reorg is going to be a big one. - if tip != nil && commonAncestor != nil { + if tip != nil { numBlocks := tip.Height - commonAncestor.Height if numBlocks > 10 { glog.Warningf("GetReorgBlocks: Proceeding with reorg of (%d) blocks from "+ @@ -2287,12 +2276,12 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode, isHeade currentBlock := &BlockNode{} *currentBlock = *tip for currentBlock != nil && *currentBlock.Hash != *commonAncestor.Hash { + detachBlocks = append(detachBlocks, currentBlock) var exists bool currentBlock, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) if !exists { glog.Fatalf("GetReorgBlocks: Failed to find parent of block. Parent hash %v", currentBlock.Header.PrevBlockHash) } - detachBlocks = append(detachBlocks, currentBlock) } // Get the blocks to attach. Start at the new node and work backwards to @@ -2452,8 +2441,8 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // the parent block. Note that if the parent block is in the block index // then it has necessarily had its difficulty validated, and so using it to // do this check makes sense. - diffTarget, err := CalcNextDifficultyTarget( - parentNode, blockHeader.Version, bc.params) + diffTarget, err := bc.CalcNextDifficultyTarget( + parentNode, blockHeader.Version) if err != nil { return false, false, errors.Wrapf(err, "ProcessBlock: Problem computing difficulty "+ @@ -2520,7 +2509,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - _, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode, true) + _, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) @@ -3085,7 +3074,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Find the common ancestor of this block and the main chain. // TODO: Reorgs with postgres? - commonAncestor, detachBlocks, attachBlocks := bc.GetReorgBlocks(currentTip, nodeToValidate, false) + commonAncestor, detachBlocks, attachBlocks := bc.GetReorgBlocks(currentTip, nodeToValidate) // Log a warning if the reorg is going to be a big one. numBlocks := currentTip.Height - commonAncestor.Height if numBlocks > 10 { diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index c54c056a1..18c136607 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1222,6 +1222,8 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, @@ -1229,6 +1231,8 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { MaxDifficultyRetargetFactor: 2, } + bc.params = fakeParams + nodes := []*BlockNode{} diffsAsInts := []int64{} for ii := 0; ii < 13; ii++ { @@ -1236,7 +1240,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1273,7 +1277,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { diffsAsInts = []int64{} for ii := 13; ii < 30; ii++ { lastNode := nodes[ii-1] - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1318,6 +1322,8 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, @@ -1325,6 +1331,8 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { MaxDifficultyRetargetFactor: 2, } + bc.params = fakeParams + nodes := []*BlockNode{} diffsAsInts := []int64{} for ii := 0; ii < 13; ii++ { @@ -1332,7 +1340,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1369,7 +1377,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { diffsAsInts = []int64{} for ii := 13; ii < 30; ii++ { lastNode := nodes[ii-1] - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1414,6 +1422,8 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, @@ -1421,6 +1431,8 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { MaxDifficultyRetargetFactor: 2, } + bc.params = fakeParams + nodes := []*BlockNode{} diffsAsInts := []int64{} for ii := 0; ii < 13; ii++ { @@ -1428,7 +1440,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1469,12 +1481,15 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, TimeBetweenBlocks: 2 * time.Second, MaxDifficultyRetargetFactor: 3, } + bc.params = fakeParams nodes := []*BlockNode{} diffsAsInts := []int64{} @@ -1483,7 +1498,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1524,6 +1539,8 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, @@ -1531,6 +1548,8 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { MaxDifficultyRetargetFactor: 2, } + bc.params = fakeParams + nodes := []*BlockNode{} diffsAsInts := []int64{} for ii := 0; ii < 13; ii++ { @@ -1538,7 +1557,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1575,7 +1594,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { diffsAsInts = []int64{} for ii := 13; ii < 34; ii++ { lastNode := nodes[ii-1] - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 46a0ce78e..1d455be28 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -93,7 +93,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. - _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode, true) + _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, diff --git a/lib/txindex.go b/lib/txindex.go index 4f2998d86..b7643ef7a 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -243,7 +243,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // At this point, we know our txindex tip is in our block index so // there must be a common ancestor between the tip and the block tip. - commonAncestor, detachBlocks, attachBlocks := txi.CoreChain.GetReorgBlocks(&derefedTxindexTipNode, committedTip, false) + commonAncestor, detachBlocks, attachBlocks := txi.CoreChain.GetReorgBlocks(&derefedTxindexTipNode, committedTip) return txindexTipNode, committedTip, commonAncestor, detachBlocks, attachBlocks } From 47e1958abadb24cdb430ad72537275732312cfbb Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 16:28:22 -0400 Subject: [PATCH 08/62] bump test timeout --- test.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.Dockerfile b/test.Dockerfile index 3eb8c7501..6262c5c55 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -28,4 +28,4 @@ COPY main.go . # build backend RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/core main.go -ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] +ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "30m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] From 5e5c0c18fec3fc074171df905ca81df3456414a3 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 19:41:20 -0400 Subject: [PATCH 09/62] load all blocks after committed tip in block index --- lib/blockchain.go | 64 +++++++++++++++++++++++++++++++------- lib/db_utils.go | 63 +++++++++++++++++++++++++++++++++++++ lib/pos_blockchain_test.go | 6 ++-- lib/pos_consensus_test.go | 15 ++++----- 4 files changed, 126 insertions(+), 22 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 4de5b6988..86a0d9001 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -544,29 +544,54 @@ type CheckpointBlockInfoAndError struct { } type BlockIndex struct { - db *badger.DB - snapshot *Snapshot - blockIndexByHash *lru2.Map[BlockHash, *BlockNode] + db *badger.DB + snapshot *Snapshot + blockIndexByHash *lru2.Map[BlockHash, *BlockNode] + blockIndexByHeight *lru2.Map[uint64, []*BlockNode] + maxHeightSeen uint64 } func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { return &BlockIndex{ - db: db, - snapshot: snapshot, - blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + db: db, + snapshot: snapshot, + blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + blockIndexByHeight: lru2.NewMap[uint64, []*BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + maxHeightSeen: 0, } } -func newBlockIndexByHashFromMap(input map[BlockHash]*BlockNode) *lru2.Map[BlockHash, *BlockNode] { - newMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) +func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { + newHashToBlockNodeMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newHeightToBlockNodeMap := lru2.NewMap[uint64, []*BlockNode](MaxBlockIndexNodes) + maxHeight := uint64(0) for key, val := range input { - newMap.Put(key, val) + newHashToBlockNodeMap.Put(key, val) + blocksAtHeight, exists := newHeightToBlockNodeMap.Get(uint64(val.Height)) + if !exists { + blocksAtHeight = []*BlockNode{} + } + blocksAtHeight = append(blocksAtHeight, val) + newHeightToBlockNodeMap.Put(uint64(val.Height), append(blocksAtHeight, val)) + if uint64(val.Height) > maxHeight { + maxHeight = uint64(val.Height) + } } - return newMap + bi.blockIndexByHash = newHashToBlockNodeMap + bi.blockIndexByHeight = newHeightToBlockNodeMap + bi.maxHeightSeen = maxHeight } func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) + blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) + if !exists { + blocksAtHeight = []*BlockNode{} + } + bi.blockIndexByHeight.Put(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) + if uint64(blockNode.Height) > bi.maxHeightSeen { + bi.maxHeightSeen = uint64(blockNode.Height) + } } func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, bool, error) { @@ -581,7 +606,9 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem getting height for hash") } - // TODO: cache current height to exit early? + if height > bi.maxHeightSeen { + return nil, false, nil + } blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if blockNode == nil { return nil, false, nil @@ -598,7 +625,9 @@ func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height u if height > math.MaxUint32 { glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) } - // TODO: cache current height to exit early? + if height > bi.maxHeightSeen { + return nil, false + } bn := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false @@ -611,6 +640,13 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { if height > math.MaxUint32 { glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) } + if height > bi.maxHeightSeen { + return []*BlockNode{} + } + blockNodesAtHeight, exists := bi.blockIndexByHeight.Get(height) + if exists { + return blockNodesAtHeight + } // TODO: cache current height to exit early? prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false) @@ -1113,6 +1149,10 @@ func (bc *Blockchain) _initChain() error { currBlockCounter++ } } + if err = bc.blockIndex.LoadBlockIndexFromHeight(tipNode.Height, bc.params); err != nil { + return errors.Wrapf(err, "_initChain: Problem loading block index from db") + } + } // At this point the blockIndexByHash should contain a full node tree with all diff --git a/lib/db_utils.go b/lib/db_utils.go index e9520a44c..cc3e4faee 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5596,6 +5596,69 @@ func GetBlockIndex(handle *badger.DB, bitcoinNodes bool, params *DeSoParams) ( return blockIndex, nil } +func (bi *BlockIndex) LoadBlockIndexFromHeight(height uint32, params *DeSoParams) error { + prefix := _heightHashToNodePrefixByHeight(height, false) + + return bi.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + nodeIterator := txn.NewIterator(opts) + defer nodeIterator.Close() + for nodeIterator.Seek(prefix); nodeIterator.ValidForPrefix(prefix); nodeIterator.Next() { + var blockNode *BlockNode + + // Don't bother checking the key. We assume that the key lines up + // with what we've stored in the value in terms of (height, block hash). + item := nodeIterator.Item() + err := item.Value(func(blockNodeBytes []byte) error { + // Deserialize the block node. + var err error + // TODO: There is room for optimization here by pre-allocating a + // contiguous list of block nodes and then populating that list + // rather than having each blockNode be a stand-alone allocation. + blockNode, err = DeserializeBlockNode(blockNodeBytes) + if err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + + // If we got here it means we read a blockNode successfully. Store it + // into our node index. + bi.addNewBlockNodeToBlockIndex(blockNode) + + // Find the parent of this block, which should already have been read + // in and connect it. Skip the genesis block, which has height 0. Also + // skip the block if its PrevBlockHash is empty, which will be true for + // the BitcoinStartBlockNode. + // + // TODO: There is room for optimization here by keeping a reference to + // the last node we've iterated over and checking if that node is the + // parent. Doing this would avoid an expensive hashmap check to get + // the parent by its block hash. + if blockNode.Height == 0 || (*blockNode.Header.PrevBlockHash == BlockHash{}) { + continue + } + if parent, ok := bi.GetBlockNodeByHashAndHeight(blockNode.Header.PrevBlockHash, uint64(blockNode.Height)); ok { + // We found the parent node so connect it. + blockNode.Parent = parent + } else { + // If we're syncing a DeSo node and we hit a PoS block, we expect there to + // be orphan blocks in the block index. In this case, we don't throw an error. + if params.IsPoSBlockHeight(uint64(blockNode.Height)) { + continue + } + // In this case we didn't find the parent so error. There shouldn't + // be any unconnectedTxns in our block index. + return fmt.Errorf("GetBlockIndex: Could not find parent for blockNode: %+v", blockNode) + } + } + return nil + }) +} + func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager *EventManager) error { return handle.View(func(txn *badger.Txn) error { prefix := _heightHashToNodeIndexPrefix(false) diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 64daa127c..fa50800ce 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -329,7 +329,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -475,7 +475,7 @@ func TestHasValidBlockViewPoS(t *testing.T) { genesisNode, block2, } - bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -808,7 +808,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) bc.bestChain.Chain = []*BlockNode{genesisNode} - bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) block := &MsgDeSoBlock{ diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index 900afb71a..81e785c5b 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -102,6 +102,11 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { currentView := blockHeader.ValidatorsVoteQC.GetView() + 1 nextView := currentView + 1 + blockIndex := NewBlockIndex(nil, nil) + blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + *blockHash: {Header: blockHeader}, + }) + // Create a mock consensus fastHotStuffConsensus := FastHotStuffConsensus{ lock: sync.RWMutex{}, @@ -111,13 +116,9 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { }, params: &DeSoTestnetParams, blockchain: &Blockchain{ - ChainLock: deadlock.RWMutex{}, - blockIndex: &BlockIndex{ - blockIndexByHash: newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ - *blockHash: {Header: blockHeader}, - }), - }, - params: &DeSoTestnetParams, + ChainLock: deadlock.RWMutex{}, + blockIndex: blockIndex, + params: &DeSoTestnetParams, }, fastHotStuffEventLoop: &consensus.MockFastHotStuffEventLoop{ OnIsInitialized: alwaysReturnTrue, From 9a32c2631216532caad73953c51077c416967b44 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 19:55:08 -0400 Subject: [PATCH 10/62] fix GetBlockNodeByHashAndHeight - no need for checking height > maxHeight --- lib/blockchain.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 86a0d9001..8ac3f39ff 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -606,9 +606,6 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem getting height for hash") } - if height > bi.maxHeightSeen { - return nil, false, nil - } blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if blockNode == nil { return nil, false, nil From 5359be82be1977571c321605b3daf845b9d325e6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 00:04:54 -0400 Subject: [PATCH 11/62] bump timeout to 60m --- test.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.Dockerfile b/test.Dockerfile index 6262c5c55..21f1dc087 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -28,4 +28,4 @@ COPY main.go . # build backend RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/core main.go -ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "30m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] +ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "60m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] From 89a845e1f3b89713afcacd0a5e5f88d921cd2204 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 11:33:56 -0400 Subject: [PATCH 12/62] try hashicorp lru cache --- go.mod | 40 +------------- go.sum | 104 ++----------------------------------- lib/block_view_test.go | 4 +- lib/blockchain.go | 85 +++++++++++++++++------------- lib/miner.go | 23 ++++++++ lib/pos_blockchain.go | 2 +- lib/pos_blockchain_test.go | 26 +++++----- lib/postgres.go | 8 +-- lib/txindex.go | 4 +- test.Dockerfile | 2 +- 10 files changed, 100 insertions(+), 198 deletions(-) diff --git a/go.mod b/go.mod index a64a34f67..ddf84244b 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/go-pg/pg/v10 v10.10.0 github.com/golang/glog v1.0.0 github.com/google/uuid v1.2.0 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/holiman/uint256 v1.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/oleiade/lane v1.0.1 @@ -48,46 +49,14 @@ require ( ) require ( - decred.org/cspp/v2 v2.2.0 // indirect github.com/Microsoft/go-winio v0.4.16 // indirect - github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect github.com/bwesterb/go-ristretto v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect - github.com/dchest/siphash v1.2.3 // indirect - github.com/decred/base58 v1.0.5 // indirect - github.com/decred/dcrd v1.9.4 // indirect - github.com/decred/dcrd/addrmgr/v2 v2.0.4 // indirect - github.com/decred/dcrd/bech32 v1.1.4 // indirect - github.com/decred/dcrd/blockchain/stake/v5 v5.0.1 // indirect - github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1 // indirect - github.com/decred/dcrd/certgen v1.2.0 // indirect - github.com/decred/dcrd/chaincfg/chainhash v1.0.4 // indirect - github.com/decred/dcrd/chaincfg/v3 v3.2.1 // indirect - github.com/decred/dcrd/connmgr/v3 v3.1.2 // indirect - github.com/decred/dcrd/container/apbf v1.0.1 // indirect - github.com/decred/dcrd/container/lru v1.0.0 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect - github.com/decred/dcrd/crypto/rand v1.0.0 // indirect - github.com/decred/dcrd/crypto/ripemd160 v1.0.2 // indirect - github.com/decred/dcrd/database/v3 v3.0.2 // indirect - github.com/decred/dcrd/dcrec v1.0.1 // indirect - github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3 // indirect - github.com/decred/dcrd/dcrjson/v4 v4.1.0 // indirect - github.com/decred/dcrd/dcrutil/v4 v4.0.2 // indirect - github.com/decred/dcrd/gcs/v4 v4.1.0 // indirect - github.com/decred/dcrd/math/uint256 v1.0.2 // indirect - github.com/decred/dcrd/mixing v0.4.1 // indirect - github.com/decred/dcrd/peer/v3 v3.1.2 // indirect - github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0 // indirect - github.com/decred/dcrd/txscript/v4 v4.1.1 // indirect - github.com/decred/dcrd/wire v1.7.0 // indirect - github.com/decred/go-socks v1.1.0 // indirect - github.com/decred/slog v1.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect @@ -99,17 +68,12 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 // indirect - github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.8 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jessevdk/go-flags v1.6.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jrick/bitset v1.0.0 // indirect - github.com/jrick/logrotate v1.1.2 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.12.3 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.9 // indirect @@ -128,7 +92,6 @@ require ( github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tinylib/msgp v1.1.2 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df // indirect @@ -152,6 +115,5 @@ require ( gopkg.in/kyokomi/emoji.v1 v1.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.3.0 // indirect mellium.im/sasl v0.2.1 // indirect ) diff --git a/go.sum b/go.sum index 821ddcfdd..c159e2333 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -decred.org/cspp/v2 v2.2.0 h1:VSOUC1w0Wo+QOGS0r1XO6TLnO16X67KuvpDmRRYyr08= -decred.org/cspp/v2 v2.2.0/go.mod h1:9nO3bfvCheOPIFZw5f6sRQ42CjBFB5RKSaJ9Iq6G4MA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -41,8 +39,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= -github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -98,8 +94,6 @@ github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GK github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a h1:clYxJ3Os0EQUKDDVU8M0oipllX0EkuFNBfhVQuIfyF0= -github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a/go.mod h1:z/9Ck1EDixEbBbZ2KH2qNHekEmDLTOZ+FyoIPWWSVOI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -117,78 +111,14 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= -github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/decred/base58 v1.0.5 h1:hwcieUM3pfPnE/6p3J100zoRfGkQxBulZHo7GZfOqic= -github.com/decred/base58 v1.0.5/go.mod h1:s/8lukEHFA6bUQQb/v3rjUySJ2hu+RioCzLukAVkrfw= -github.com/decred/dcrd v1.9.4 h1:DgeZdf8TTtGuZGVs5DnAL0ghbhnrPYfENneqUqEaq8o= -github.com/decred/dcrd v1.9.4/go.mod h1:7I6LovGdFimw3LTPCtOGQ+8VV1HGnO+2L2KA4yi5NWA= -github.com/decred/dcrd/addrmgr/v2 v2.0.4 h1:3MWJiO2STogQwNRF3W4yjCzSJtaxqtw+UI3x2+bYeOg= -github.com/decred/dcrd/addrmgr/v2 v2.0.4/go.mod h1:661DIS/De2iLNLMwIKazUQfQypUqJ5om7PXNX0fEMms= -github.com/decred/dcrd/bech32 v1.1.4 h1:wFlLM7Oic0MlIhQZdCQhdIqVc4CNaQ0vNR9fgCoWfe0= -github.com/decred/dcrd/bech32 v1.1.4/go.mod h1:jliqHZmCbVfT06Lh1mQywEKFVidRclbBJIUmwdoKhu0= -github.com/decred/dcrd/blockchain/stake/v5 v5.0.1 h1:KDm6myUPi8j2TTL7LZ+iT+R/pIbxd8qG89fjJNitzx0= -github.com/decred/dcrd/blockchain/stake/v5 v5.0.1/go.mod h1:y1tMD1TssTlPmKDYbSrF3Ujznj+STkXFfYPwoVfe+xA= -github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1 h1:zeI9CHkLM9be4QOBmIAtoPfs6NCgJM1lpmRUYE61I8o= -github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1/go.mod h1:yXZz/EgWdGw5nqMEvyKj/iXZ9I2VSyO95xKj6mRUMIM= -github.com/decred/dcrd/certgen v1.2.0 h1:FF6XXV//5q38/c6QbGQdR35ZJz0GPIkejsZZU3oHuBQ= -github.com/decred/dcrd/certgen v1.2.0/go.mod h1:LRh6dF2WPQeDA6QQSZE+SfK7AL6FuFtCRDHZf8DyGzg= -github.com/decred/dcrd/chaincfg/chainhash v1.0.4 h1:zRCv6tdncLfLTKYqu7hrXvs7hW+8FO/NvwoFvGsrluU= -github.com/decred/dcrd/chaincfg/chainhash v1.0.4/go.mod h1:hA86XxlBWwHivMvxzXTSD0ZCG/LoYsFdWnCekkTMCqY= -github.com/decred/dcrd/chaincfg/v3 v3.2.1 h1:x9zKJaU24WAKbxAR1UyFKHlM3oJgP0H9LodokM4X5lM= -github.com/decred/dcrd/chaincfg/v3 v3.2.1/go.mod h1:SDCWDtY7BLj0leXc9FuoA1YjSVKyCIBVAyxwZn6+sXc= -github.com/decred/dcrd/connmgr/v3 v3.1.2 h1:+xNopie2L3YYwwkz51k0h/pASATOBzHtl2O8eodGg04= -github.com/decred/dcrd/connmgr/v3 v3.1.2/go.mod h1:tdbErFiNOuy/sHrX2mwaOk+r1HLs3EBz2EGxsocMPe4= -github.com/decred/dcrd/container/apbf v1.0.1 h1:oepQzRtLADudsrx0AmmowoU1kambozINTMXduH6Mge0= -github.com/decred/dcrd/container/apbf v1.0.1/go.mod h1:paQplssZMsRhwOUcP6LDNDypyb7lwsFHrUKUkAPFWtQ= -github.com/decred/dcrd/container/lru v1.0.0 h1:7foQymtbu18aQWYiY9RnNIeE+kvpiN+fiBQ3+viyJjI= -github.com/decred/dcrd/container/lru v1.0.0/go.mod h1:vlPwj0l+IzAHhQSsbgQnJgO5Cte78+yI065V+Mc5PRQ= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/crypto/rand v1.0.0 h1:Ah9Asl36OZt09sGSMbJZuL1HfwGdlC38q/ZUeLDVKRg= -github.com/decred/dcrd/crypto/rand v1.0.0/go.mod h1:coa7BbxSTiKH6esi257plGfMFYuGL4MTbQlLYnOdzpE= -github.com/decred/dcrd/crypto/ripemd160 v1.0.2 h1:TvGTmUBHDU75OHro9ojPLK+Yv7gDl2hnUvRocRCjsys= -github.com/decred/dcrd/crypto/ripemd160 v1.0.2/go.mod h1:uGfjDyePSpa75cSQLzNdVmWlbQMBuiJkvXw/MNKRY4M= -github.com/decred/dcrd/database/v3 v3.0.2 h1:rgP7XNZemTs8ZC7bnTKO8JO79Woj5nq+yQYmB9ry7yM= -github.com/decred/dcrd/database/v3 v3.0.2/go.mod h1:3Ge1yoxEOsqd72V5LTA9g0B7mlY0MGbpxeE1fniIXsQ= -github.com/decred/dcrd/dcrec v1.0.1 h1:gDzlndw0zYxM5BlaV17d7ZJV6vhRe9njPBFeg4Db2UY= -github.com/decred/dcrd/dcrec v1.0.1/go.mod h1:CO+EJd8eHFb8WHa84C7ZBkXsNUIywaTHb+UAuI5uo6o= -github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3 h1:l/lhv2aJCUignzls81+wvga0TFlyoZx8QxRMQgXpZik= -github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3/go.mod h1:AKpV6+wZ2MfPRJnTbQ6NPgWrKzbe9RCIlCF/FKzMtM8= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/decred/dcrd/dcrjson/v4 v4.1.0 h1:WJVogRnYnNxB5hWoGHODvP4fNTG1JycTuHHKt/XucHk= -github.com/decred/dcrd/dcrjson/v4 v4.1.0/go.mod h1:2qVikafVF9/X3PngQVmqkbUbyAl32uik0k/kydgtqMc= -github.com/decred/dcrd/dcrutil/v4 v4.0.2 h1:eIl3E6gGln54qE8nk5o5lLtjh2/9C2Rz63OpD662h+8= -github.com/decred/dcrd/dcrutil/v4 v4.0.2/go.mod h1:iS3JB1ac3R3FgfpTF1kBD+SPNet8TmiW3Br+/Jc5MC8= -github.com/decred/dcrd/gcs/v4 v4.1.0 h1:tpW7JW53yJZlgNwl/n2NL1b8NxHaIPRUyNuLMkB/Hks= -github.com/decred/dcrd/gcs/v4 v4.1.0/go.mod h1:nPTbGM/I3Ihe5KFvUmxZEqQP/jDZQjQ63+WEi/f4lqU= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/decred/dcrd/lru v1.1.1 h1:kWFDaW0OWx6AD6Ki342c+JPmHbiVdE6rK81pT3fuo/Y= -github.com/decred/dcrd/lru v1.1.1/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/decred/dcrd/lru v1.1.3 h1:w9EAbvGLyzm6jTjF83UKuqZEiUtJmvRhQDOCEIvSuE0= github.com/decred/dcrd/lru v1.1.3/go.mod h1:Tw0i0pJyiLEx/oZdHLe1Wdv/Y7EGzAX+sYftnmxBR4o= -github.com/decred/dcrd/math/uint256 v1.0.2 h1:o8peafL5QmuXGTergI3YDpDU0eq5Z0pQi88B8ym4PRA= -github.com/decred/dcrd/math/uint256 v1.0.2/go.mod h1:7M/y9wJJvlyNG/f/X6mxxhxo9dgloZHFiOfbiscl75A= -github.com/decred/dcrd/mixing v0.4.1 h1:W8ZCzhmNyzG1xjJMA3L6FOElmp98Ttnk3dDUxD6irAE= -github.com/decred/dcrd/mixing v0.4.1/go.mod h1:ySvVwTZyVz5YvevA6YjPrB6pJEwTm7IkHohTfaiHh2c= -github.com/decred/dcrd/peer/v3 v3.1.2 h1:Qe7SpqDtfM0HARmDYwr4WjUu16X6HQ7ZWNnHqE1swiw= -github.com/decred/dcrd/peer/v3 v3.1.2/go.mod h1:M9FxNkHuEBtsRW5gwzIH4cJTWk5xSkxy9zG+TEL1N2Y= -github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0 h1:l0DnCcILTNrpy8APF3FLN312ChpkQaAuW30aC/RgBaw= -github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0/go.mod h1:j+kkRPXPJB5S9VFOsx8SQLcU7PTFkPKRc1aCHN4ENzA= -github.com/decred/dcrd/txscript/v4 v4.1.1 h1:R4M2+jMujgQA91899SkL0cW66d6DC76Gx+1W1oEHjc0= -github.com/decred/dcrd/txscript/v4 v4.1.1/go.mod h1:7ybmJoI+b6dxvQ+0aXdZpkyrj0PbnylJCzFxD1g8+/A= -github.com/decred/dcrd/wire v1.7.0 h1:5JHiDjEQeS4XUl4PfnTZYLwAD/E/+LwBmPRec/fP76o= -github.com/decred/dcrd/wire v1.7.0/go.mod h1:lAqrzV0SU4kyV6INLEJgDtUjJaTaVKrbF4LHtaYl+zU= -github.com/decred/go-socks v1.1.0 h1:dnENcc0KIqQo3HSXdgboXAHgqsCIutkqq6ntQjYtm2U= -github.com/decred/go-socks v1.1.0/go.mod h1:sDhHqkZH0X4JjSa02oYOGhcGHYp12FsY1jQ/meV8md0= -github.com/decred/slog v1.2.0 h1:soHAxV52B54Di3WtKLfPum9OFfWqwtf/ygf9njdfnPM= -github.com/decred/slog v1.2.0/go.mod h1:kVXlGnt6DHy2fV5OjSeuvCJ0OmlmTF6LFpEPMu/fOY0= github.com/deso-protocol/go-deadlock v1.0.0 h1:mw0pHy/19zgC+JFBStuQt1+1Ehv5OKA5NxXqecnL5ic= github.com/deso-protocol/go-deadlock v1.0.0/go.mod h1:K0Wd2OV2x7ck7SMYDraWerpKjFKUeBqaFcwz21tmkb8= github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+LsH/kRdL5l/Yzvg= @@ -278,7 +208,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -295,8 +224,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -314,8 +243,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -336,6 +263,8 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -359,17 +288,11 @@ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= -github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jrick/bitset v1.0.0 h1:Ws0PXV3PwXqWK2n7Vz6idCdrV/9OrBXgHEJi27ZB9Dw= -github.com/jrick/bitset v1.0.0/go.mod h1:ZOYB5Uvkla7wIEY4FEssPVi3IQXa02arznRaYaAEPe4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/jrick/logrotate v1.1.2 h1:6ePk462NCX7TfKtNp5JJ7MbA2YIslkpfgP03TlTYMN0= -github.com/jrick/logrotate v1.1.2/go.mod h1:f9tdWggSVK3iqavGpyvegq5IhNois7KXmasU6/N96OQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -386,8 +309,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -559,8 +480,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -628,8 +547,6 @@ golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -662,7 +579,7 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -693,8 +610,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -708,8 +623,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -752,8 +665,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -761,8 +672,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -796,8 +705,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -891,8 +799,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= -lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w= mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= diff --git a/lib/block_view_test.go b/lib/block_view_test.go index 2717aeb50..b69307dbf 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -793,8 +793,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // Update the block and header metadata chains. tm.chain.bestChain.Chain = tm.chain.bestChain.Chain[:len(tm.chain.bestChain.Chain)-1] tm.chain.bestHeaderChain.Chain = tm.chain.bestHeaderChain.Chain[:len(tm.chain.bestHeaderChain.Chain)-1] - tm.chain.bestChain.ChainMap.Delete(*lastBlockHash) - tm.chain.bestHeaderChain.ChainMap.Delete(*lastBlockHash) + tm.chain.bestChain.ChainMap.Remove(*lastBlockHash) + tm.chain.bestHeaderChain.ChainMap.Remove(*lastBlockHash) // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. diff --git a/lib/blockchain.go b/lib/blockchain.go index 8ac3f39ff..89988dec4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -18,7 +18,7 @@ import ( "github.com/decred/dcrd/lru" - lru2 "github.com/decred/dcrd/container/lru" + lru2 "github.com/hashicorp/golang-lru/v2" "github.com/deso-protocol/core/collections" @@ -546,33 +546,35 @@ type CheckpointBlockInfoAndError struct { type BlockIndex struct { db *badger.DB snapshot *Snapshot - blockIndexByHash *lru2.Map[BlockHash, *BlockNode] - blockIndexByHeight *lru2.Map[uint64, []*BlockNode] + blockIndexByHash *lru2.Cache[BlockHash, *BlockNode] + blockIndexByHeight *lru2.Cache[uint64, []*BlockNode] maxHeightSeen uint64 } func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { + blockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + blockIndexByHeight, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) return &BlockIndex{ db: db, snapshot: snapshot, - blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? - blockIndexByHeight: lru2.NewMap[uint64, []*BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + blockIndexByHash: blockIndexByHash, // TODO: parameterize this? + blockIndexByHeight: blockIndexByHeight, // TODO: parameterize this? maxHeightSeen: 0, } } func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { - newHashToBlockNodeMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) - newHeightToBlockNodeMap := lru2.NewMap[uint64, []*BlockNode](MaxBlockIndexNodes) + newHashToBlockNodeMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + newHeightToBlockNodeMap, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) maxHeight := uint64(0) for key, val := range input { - newHashToBlockNodeMap.Put(key, val) + newHashToBlockNodeMap.Add(key, val) blocksAtHeight, exists := newHeightToBlockNodeMap.Get(uint64(val.Height)) if !exists { blocksAtHeight = []*BlockNode{} } blocksAtHeight = append(blocksAtHeight, val) - newHeightToBlockNodeMap.Put(uint64(val.Height), append(blocksAtHeight, val)) + newHeightToBlockNodeMap.Add(uint64(val.Height), append(blocksAtHeight, val)) if uint64(val.Height) > maxHeight { maxHeight = uint64(val.Height) } @@ -583,12 +585,12 @@ func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { } func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { - bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) + bi.blockIndexByHash.Add(*blockNode.Hash, blockNode) blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) if !exists { blocksAtHeight = []*BlockNode{} } - bi.blockIndexByHeight.Put(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) + bi.blockIndexByHeight.Add(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) if uint64(blockNode.Height) > bi.maxHeightSeen { bi.maxHeightSeen = uint64(blockNode.Height) } @@ -615,6 +617,9 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { + if height > bi.maxHeightSeen { + return nil, false + } val, exists := bi.blockIndexByHash.Get(*blockHash) if exists { return val, true @@ -622,9 +627,6 @@ func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height u if height > math.MaxUint32 { glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) } - if height > bi.maxHeightSeen { - return nil, false - } bn := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false @@ -665,24 +667,25 @@ type BestChain struct { snapshot *Snapshot IsHeaderChain bool Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. - ChainMap *lru2.Map[BlockHash, *BlockNode] + ChainMap *lru2.Cache[BlockHash, *BlockNode] params *DeSoParams } func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { + chainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) return &BestChain{ db: db, snapshot: snapshot, IsHeaderChain: isHeaderChain, Chain: []*BlockNode{}, - ChainMap: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), + ChainMap: chainMap, params: params, } } func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { bestChain.Chain = append(bestChain.Chain, tipNode) - bestChain.ChainMap.Put(*tipNode.Hash, tipNode) + bestChain.ChainMap.Add(*tipNode.Hash, tipNode) } func (bestChain *BestChain) GetTip() *BlockNode { @@ -770,7 +773,7 @@ func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height if bn == nil { return nil, false } - bestChain.ChainMap.Put(*blockHash, bn) + bestChain.ChainMap.Add(*blockHash, bn) return bn, true } @@ -797,7 +800,7 @@ func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bo if !bn.IsCommitted() { return nil, false, nil } - bestChain.ChainMap.Put(*blockHash, bn) + bestChain.ChainMap.Add(*blockHash, bn) return bn, true, nil } @@ -983,16 +986,16 @@ func getCheckpointBlockInfoFromProviderHelper(provider string) *CheckpointBlockI } func (bc *Blockchain) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { - bc.blockIndex.blockIndexByHash.Put(*blockNode.Hash, blockNode) + bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) } func (bc *Blockchain) CopyBlockIndexes() ( - _blockIndexByHash *lru2.Map[BlockHash, *BlockNode], + _blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], ) { - newBlockIndexByHash := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBlockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, key := range bc.blockIndex.blockIndexByHash.Keys() { val, _ := bc.blockIndex.blockIndexByHash.Get(key) - newBlockIndexByHash.Put(key, val) + newBlockIndexByHash.Add(key, val) } return newBlockIndexByHash } @@ -1012,25 +1015,25 @@ func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { return len(blockNodes) > 0 } -func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Map[BlockHash, *BlockNode]) { +func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { newBestChain := []*BlockNode{} - newBestChainMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) newBestChain = append(newBestChain, bc.bestChain.Chain...) for _, key := range bc.bestChain.ChainMap.Keys() { val, _ := bc.bestChain.ChainMap.Get(key) - newBestChainMap.Put(key, val) + newBestChainMap.Add(key, val) } return newBestChain, newBestChainMap } -func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Map[BlockHash, *BlockNode]) { +func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { newBestChain := []*BlockNode{} - newBestChainMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) for _, key := range bc.bestHeaderChain.ChainMap.Keys() { val, _ := bc.bestHeaderChain.ChainMap.Get(key) - newBestChainMap.Put(key, val) + newBestChainMap.Add(key, val) } return newBestChain, newBestChainMap @@ -1162,7 +1165,7 @@ func (bc *Blockchain) _initChain() error { return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") } for _, bestChainNode := range bc.bestChain.Chain { - bc.bestChain.ChainMap.Put(*bestChainNode.Hash, bestChainNode) + bc.bestChain.ChainMap.Add(*bestChainNode.Hash, bestChainNode) } } @@ -1175,7 +1178,7 @@ func (bc *Blockchain) _initChain() error { return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") } for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { - bc.bestHeaderChain.ChainMap.Put(*bestHeaderChainNode.Hash, bestHeaderChainNode) + bc.bestHeaderChain.ChainMap.Add(*bestHeaderChainNode.Hash, bestHeaderChainNode) } } @@ -2023,8 +2026,8 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { func (bc *Blockchain) SetBestChainMap( bestChain []*BlockNode, - bestChainMap *lru2.Map[BlockHash, *BlockNode], - blockIndexByHash *lru2.Map[BlockHash, *BlockNode], + bestChainMap *lru2.Cache[BlockHash, *BlockNode], + blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], ) { bc.bestChain.Chain = bestChain bc.bestChain.ChainMap = bestChainMap @@ -2350,14 +2353,14 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( return commonAncestor, detachBlocks, attachBlocks } -func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Map[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( - chainList []*BlockNode, chainMap *lru2.Map[BlockHash, *BlockNode]) { +func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Cache[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( + chainList []*BlockNode, chainMap *lru2.Cache[BlockHash, *BlockNode]) { // Remove the nodes we detached from the end of the best chain node list. tipIndex := len(mainChainList) - 1 for blockOffset := 0; blockOffset < len(detachBlocks); blockOffset++ { blockIndex := tipIndex - blockOffset - mainChainMap.Delete(*mainChainList[blockIndex].Hash) + mainChainMap.Remove(*mainChainList[blockIndex].Hash) } mainChainList = mainChainList[:len(mainChainList)-len(detachBlocks)] @@ -2366,7 +2369,7 @@ func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Map[ // first, with the new tip at the end. for _, attachNode := range attachBlocks { mainChainList = append(mainChainList, attachNode) - mainChainMap.Put(*attachNode.Hash, attachNode) + mainChainMap.Add(*attachNode.Hash, attachNode) } return mainChainList, mainChainMap @@ -2676,14 +2679,19 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.End("Blockchain.ProcessBlock: Initial") bc.timer.Start("Blockchain.ProcessBlock: BlockNode") + now := time.Now() // See if a node for the block exists in our node index. // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set // the header validation status for it. + glog.V(0).Infof("ProcessBlock: time to check if node exists: %v", time.Since(now)) + now = time.Now() if !nodeExists { _, isOrphan, err := bc.processHeaderPoW(blockHeader, blockHash) + glog.V(0).Infof("ProcessBlock: time to process header: %v", time.Since(now)) + now = time.Now() if err != nil { // If an error occurred processing the header, then the header // should be marked as invalid, which should be sufficient. @@ -2701,6 +2709,9 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // block index. // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) + + glog.V(0).Infof("ProcessBlock: time to get node after processing header: %v", time.Since(now)) + now = time.Now() } // At this point if the node still doesn't exist or if the header's validation // failed then we should return an error for the block. Note that at this point diff --git a/lib/miner.go b/lib/miner.go index 5bc1e45d1..fab9cb1d8 100644 --- a/lib/miner.go +++ b/lib/miner.go @@ -113,6 +113,7 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B // // TODO(miner): Replace with a call to GetBlockTemplate publicKey := desoMiner._getRandomPublicKey() + now := time.Now() blockID, headerBytes, extraNonces, diffTarget, err := desoMiner.BlockProducer.GetHeadersAndExtraDatas( publicKey, 1 /*numHeaders*/, CurrentHeaderVersion) if err != nil { @@ -121,6 +122,8 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B time.Sleep(1 * time.Second) continue } + glog.V(0).Infof("Time to get headers and extra data: %v", time.Since(now)) + now = time.Now() header := &MsgDeSoHeader{} if err := header.FromBytes(headerBytes[0]); err != nil { glog.Errorf("DeSoMiner._startThread: Error parsing header to " + @@ -138,6 +141,8 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B glog.Error(errors.Wrapf(err, "DeSoMiner._startThread: Problem while mining: ")) break } + glog.V(0).Infof("Time to header to bytes and find lowest hash: %v", time.Since(now)) + now = time.Now() if atomic.LoadInt32(&desoMiner.stopping) == 1 { glog.V(1).Infof("DeSoMiner._startThread: Stopping thread %d", threadIndex) @@ -161,6 +166,8 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B time.Sleep(1 * time.Second) continue } + glog.V(0).Infof("Time to get copy of recent block: %v", time.Since(now)) + now = time.Now() // Swap in the public key and extraNonce. This should make the block consistent with // the header we were just mining on. @@ -178,6 +185,8 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B // Use the nonce we computed blockToMine.Header.Nonce = bestNonce + glog.V(0).Infof("Time to recompute rewards and whatnot: %v", time.Since(now)) + now = time.Now() return diffTarget, blockToMine } @@ -187,16 +196,21 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoolToUpdate *DeSoMempool) (_block *MsgDeSoBlock, _err error) { // Add a call to update the BlockProducer. + now := time.Now() // TODO(performance): We shouldn't have to do this, it just makes tests pass right now. if err := desoMiner.BlockProducer.UpdateLatestBlockTemplate(); err != nil { // Error if we can't update the template but don't stop the show. glog.Error(err) } + glog.V(0).Infof("Time to update block template: %v", time.Since(now)) + now = time.Now() diffTarget, blockToMine := desoMiner._mineSingleBlock(threadIndex) if blockToMine == nil { return nil, fmt.Errorf("DeSoMiner._startThread: _mineSingleBlock returned nil; should only happen if we're stopping") } + glog.V(0).Infof("Time to update mine single block: %v", time.Since(now)) + now = time.Now() // Log information on the block we just mined. bestHash, _ := blockToMine.Hash() @@ -208,6 +222,8 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo desoMiner.BlockProducer.chain.blockTip().Header) scs := spew.ConfigState{DisableMethods: true, Indent: " ", DisablePointerAddresses: true} glog.V(1).Infof(scs.Sdump(blockToMine)) + //glog.V(0).Infof("Time to print a bunch of stuff: %v", time.Since(now)) + now = time.Now() // Sanitize the block for the comparison we're about to do. We need to do // this because the comparison function below will think they're different // if one has nil and one has an empty list. Annoying, but this solves the @@ -242,6 +258,8 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if err := desoMiner.BlockProducer.SignBlock(blockToMine); err != nil { return nil, fmt.Errorf("Error signing block: %v", err) } + //glog.V(0).Infof("Time to sign block: %v", time.Since(now)) + now = time.Now() // Process the block. If the block is connected and/or accepted, the Server // will be informed about it. This will cause it to be relayed appropriately. @@ -259,6 +277,8 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo return blockToMine, fmt.Errorf("ERROR calling ProcessBlock: isMainChain=(%v), isOrphan=(%v), err=(%v)", isMainChain, isOrphan, err) } + glog.V(0).Infof("Time to process block: %v", time.Since(now)) + now = time.Now() // If a mempool object is passed then update it. Normally this isn't necessary because // ProcessBlock will trigger it because the backendServer will be set on the blockchain @@ -266,6 +286,8 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if mempoolToUpdate != nil { mempoolToUpdate.UpdateAfterConnectBlock(blockToMine) } + //glog.V(0).Infof("Time to update mempool after connect block: %v", time.Since(now)) + now = time.Now() decimalPlaces := int64(1000) diffTargetBaseline, _ := hex.DecodeString(desoMiner.params.MinDifficultyTargetHex) @@ -278,6 +300,7 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if atomic.LoadInt32(&desoMiner.stopping) == 1 { return nil, fmt.Errorf("DeSoMiner._startThread: Stopping thread %d", threadIndex) } + //glog.V(0).Infof("Time to update everything else: %v", time.Since(now)) return blockToMine, nil } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 1d455be28..134505eab 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1652,7 +1652,7 @@ func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { func (bc *Blockchain) removeTipBlockFromBestChain() *BlockNode { // Remove the last block from the best chain. lastBlock := bc.bestChain.Chain[len(bc.bestChain.Chain)-1] - bc.bestChain.ChainMap.Delete(*lastBlock.Hash) + bc.bestChain.ChainMap.Remove(*lastBlock.Hash) bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] return lastBlock } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index fa50800ce..8dddc5213 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -252,7 +252,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) bc.bestChain.Chain = []*BlockNode{genesisBlock} - bc.blockIndex.blockIndexByHash.Put(*genesisBlock.Hash, genesisBlock) + bc.blockIndex.blockIndexByHash.Add(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) randomBLSPrivateKey := _generateRandomBLSPrivateKey(t) @@ -845,7 +845,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) bc.bestChain.Chain = append(bc.bestChain.Chain, block2) - bc.blockIndex.blockIndexByHash.Put(*hash2, block2) + bc.blockIndex.blockIndexByHash.Add(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) require.Equal(t, err, RuleErrorDoesNotExtendCommittedTip) @@ -1317,9 +1317,9 @@ func TestTryApplyNewTip(t *testing.T) { bc.addTipBlockToBestChain(bn1) bc.addTipBlockToBestChain(bn2) bc.addTipBlockToBestChain(bn3) - bc.blockIndex.blockIndexByHash.Put(*hash1, bn1) - bc.blockIndex.blockIndexByHash.Put(*hash2, bn2) - bc.blockIndex.blockIndexByHash.Put(*hash3, bn3) + bc.blockIndex.blockIndexByHash.Add(*hash1, bn1) + bc.blockIndex.blockIndexByHash.Add(*hash2, bn2) + bc.blockIndex.blockIndexByHash.Add(*hash3, bn3) // Simple reorg. Just replacing the uncommitted tip. newBlock := &MsgDeSoBlock{ @@ -1372,7 +1372,7 @@ func TestTryApplyNewTip(t *testing.T) { // Remove newBlock from the best chain and block index to reset the state. bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] - bc.bestChain.ChainMap.Delete(*newBlockHash) + bc.bestChain.ChainMap.Remove(*newBlockHash) // Add block 3 back bc.addTipBlockToBestChain(bn3) @@ -1401,8 +1401,8 @@ func TestTryApplyNewTip(t *testing.T) { Height: 6, }, } - bc.blockIndex.blockIndexByHash.Put(*hash4, bn4) - bc.blockIndex.blockIndexByHash.Put(*hash5, bn5) + bc.blockIndex.blockIndexByHash.Add(*hash4, bn4) + bc.blockIndex.blockIndexByHash.Add(*hash5, bn5) // Set new block's parent to hash5 newBlockNode.Header.PrevBlockHash = hash5 @@ -1441,9 +1441,9 @@ func TestTryApplyNewTip(t *testing.T) { require.Len(t, disconnectedBlockHashes, 2) // Reset the state of the best chain. - bc.bestChain.ChainMap.Delete(*hash4) - bc.bestChain.ChainMap.Delete(*hash5) - bc.bestChain.ChainMap.Delete(*newBlockHash) + bc.bestChain.ChainMap.Remove(*hash4) + bc.bestChain.ChainMap.Remove(*hash5) + bc.bestChain.ChainMap.Remove(*newBlockHash) bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] // Add block 2 and 3 back. @@ -1514,8 +1514,8 @@ func TestCanCommitGrandparent(t *testing.T) { PrevBlockHash: hash1, }, } - bc.bestChain.ChainMap.Put(*hash1, bn1) - bc.bestChain.ChainMap.Put(*hash2, bn2) + bc.bestChain.ChainMap.Add(*hash1, bn1) + bc.bestChain.ChainMap.Add(*hash2, bn2) // define incoming block hash3 := NewBlockHash(RandomBytes(32)) diff --git a/lib/postgres.go b/lib/postgres.go index 0c5991756..08d078006 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - lru2 "github.com/decred/dcrd/container/lru" + lru2 "github.com/hashicorp/golang-lru/v2" "net/url" "regexp" "strings" @@ -1305,16 +1305,16 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { } // GetBlockIndex gets all the PGBlocks and creates a map of BlockHash to BlockNode as needed by blockchain.go -func (postgres *Postgres) GetBlockIndex() (*lru2.Map[BlockHash, *BlockNode], error) { +func (postgres *Postgres) GetBlockIndex() (*lru2.Cache[BlockHash, *BlockNode], error) { var blocks []PGBlock err := postgres.db.Model(&blocks).Select() if err != nil { return nil, err } - blockMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + blockMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, block := range blocks { - blockMap.Put(*block.Hash, &BlockNode{ + blockMap.Add(*block.Hash, &BlockNode{ Hash: block.Hash, Height: uint32(block.Height), DifficultyTarget: block.DifficultyTarget, diff --git a/lib/txindex.go b/lib/txindex.go index b7643ef7a..904211819 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -372,8 +372,8 @@ func (txi *TXIndex) Update() error { newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() newBestChain, newBestChainMap := txi.TXIndexChain.CopyBestChain() newBestChain = newBestChain[:len(newBestChain)-1] - newBestChainMap.Delete(*blockToDetach.Hash) - newBlockIndex.Delete(*blockToDetach.Hash) + newBestChainMap.Remove(*blockToDetach.Hash) + newBlockIndex.Remove(*blockToDetach.Hash) txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndex) diff --git a/test.Dockerfile b/test.Dockerfile index 21f1dc087..34182c4c7 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -28,4 +28,4 @@ COPY main.go . # build backend RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/core main.go -ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "60m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] +ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "15m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] From 4f7ff12632e26e13d9c8eae6c726f1c076fabbbb Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 12:47:20 -0400 Subject: [PATCH 13/62] Fix up some more tests and remove logging --- lib/blockchain.go | 48 ++++++++++++-------------------------- lib/blockchain_test.go | 28 ++++++++++++---------- lib/miner.go | 24 +------------------ lib/pos_blockchain_test.go | 13 +++++++---- lib/pos_consensus_test.go | 2 +- 5 files changed, 42 insertions(+), 73 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 89988dec4..56bb03ea6 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -548,7 +548,7 @@ type BlockIndex struct { snapshot *Snapshot blockIndexByHash *lru2.Cache[BlockHash, *BlockNode] blockIndexByHeight *lru2.Cache[uint64, []*BlockNode] - maxHeightSeen uint64 + //maxHeightSeen uint64 } func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { @@ -559,29 +559,18 @@ func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { snapshot: snapshot, blockIndexByHash: blockIndexByHash, // TODO: parameterize this? blockIndexByHeight: blockIndexByHeight, // TODO: parameterize this? - maxHeightSeen: 0, + //maxHeightSeen: 0, } } func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { newHashToBlockNodeMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) newHeightToBlockNodeMap, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) - maxHeight := uint64(0) - for key, val := range input { - newHashToBlockNodeMap.Add(key, val) - blocksAtHeight, exists := newHeightToBlockNodeMap.Get(uint64(val.Height)) - if !exists { - blocksAtHeight = []*BlockNode{} - } - blocksAtHeight = append(blocksAtHeight, val) - newHeightToBlockNodeMap.Add(uint64(val.Height), append(blocksAtHeight, val)) - if uint64(val.Height) > maxHeight { - maxHeight = uint64(val.Height) - } - } bi.blockIndexByHash = newHashToBlockNodeMap bi.blockIndexByHeight = newHeightToBlockNodeMap - bi.maxHeightSeen = maxHeight + for _, val := range input { + bi.addNewBlockNodeToBlockIndex(val) + } } func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { @@ -589,11 +578,15 @@ func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) if !exists { blocksAtHeight = []*BlockNode{} + } else { + // Make sure we don't add the same block node twice. + for _, blockAtHeight := range blocksAtHeight { + if blockAtHeight.Hash.IsEqual(blockNode.Hash) { + return + } + } } bi.blockIndexByHeight.Add(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) - if uint64(blockNode.Height) > bi.maxHeightSeen { - bi.maxHeightSeen = uint64(blockNode.Height) - } } func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, bool, error) { @@ -617,9 +610,6 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { - if height > bi.maxHeightSeen { - return nil, false - } val, exists := bi.blockIndexByHash.Get(*blockHash) if exists { return val, true @@ -639,9 +629,9 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { if height > math.MaxUint32 { glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) } - if height > bi.maxHeightSeen { - return []*BlockNode{} - } + //if height > bi.maxHeightSeen { + // return []*BlockNode{} + //} blockNodesAtHeight, exists := bi.blockIndexByHeight.Get(height) if exists { return blockNodesAtHeight @@ -2679,19 +2669,14 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.End("Blockchain.ProcessBlock: Initial") bc.timer.Start("Blockchain.ProcessBlock: BlockNode") - now := time.Now() // See if a node for the block exists in our node index. // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set // the header validation status for it. - glog.V(0).Infof("ProcessBlock: time to check if node exists: %v", time.Since(now)) - now = time.Now() if !nodeExists { _, isOrphan, err := bc.processHeaderPoW(blockHeader, blockHash) - glog.V(0).Infof("ProcessBlock: time to process header: %v", time.Since(now)) - now = time.Now() if err != nil { // If an error occurred processing the header, then the header // should be marked as invalid, which should be sufficient. @@ -2709,9 +2694,6 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // block index. // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) - - glog.V(0).Infof("ProcessBlock: time to get node after processing header: %v", time.Since(now)) - now = time.Now() } // At this point if the node still doesn't exist or if the header's validation // failed then we should return an error for the block. Note that at this point diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 18c136607..adcf3f429 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1244,7 +1244,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1254,6 +1254,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusNone, )) + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1281,7 +1282,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1291,6 +1292,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusNone, )) + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1344,7 +1346,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1354,6 +1356,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusNone, )) + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1381,7 +1384,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1391,6 +1394,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusNone, )) + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1444,7 +1448,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1454,7 +1458,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { }, StatusNone, )) - + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1502,7 +1506,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1512,7 +1516,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { }, StatusNone, )) - + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1561,7 +1565,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1571,7 +1575,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusNone, )) - + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1598,7 +1602,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1608,7 +1612,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusNone, )) - + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } diff --git a/lib/miner.go b/lib/miner.go index fab9cb1d8..0df444912 100644 --- a/lib/miner.go +++ b/lib/miner.go @@ -113,7 +113,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B // // TODO(miner): Replace with a call to GetBlockTemplate publicKey := desoMiner._getRandomPublicKey() - now := time.Now() blockID, headerBytes, extraNonces, diffTarget, err := desoMiner.BlockProducer.GetHeadersAndExtraDatas( publicKey, 1 /*numHeaders*/, CurrentHeaderVersion) if err != nil { @@ -122,8 +121,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B time.Sleep(1 * time.Second) continue } - glog.V(0).Infof("Time to get headers and extra data: %v", time.Since(now)) - now = time.Now() header := &MsgDeSoHeader{} if err := header.FromBytes(headerBytes[0]); err != nil { glog.Errorf("DeSoMiner._startThread: Error parsing header to " + @@ -141,8 +138,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B glog.Error(errors.Wrapf(err, "DeSoMiner._startThread: Problem while mining: ")) break } - glog.V(0).Infof("Time to header to bytes and find lowest hash: %v", time.Since(now)) - now = time.Now() if atomic.LoadInt32(&desoMiner.stopping) == 1 { glog.V(1).Infof("DeSoMiner._startThread: Stopping thread %d", threadIndex) @@ -166,8 +161,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B time.Sleep(1 * time.Second) continue } - glog.V(0).Infof("Time to get copy of recent block: %v", time.Since(now)) - now = time.Now() // Swap in the public key and extraNonce. This should make the block consistent with // the header we were just mining on. @@ -185,8 +178,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B // Use the nonce we computed blockToMine.Header.Nonce = bestNonce - glog.V(0).Infof("Time to recompute rewards and whatnot: %v", time.Since(now)) - now = time.Now() return diffTarget, blockToMine } @@ -196,21 +187,16 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoolToUpdate *DeSoMempool) (_block *MsgDeSoBlock, _err error) { // Add a call to update the BlockProducer. - now := time.Now() // TODO(performance): We shouldn't have to do this, it just makes tests pass right now. if err := desoMiner.BlockProducer.UpdateLatestBlockTemplate(); err != nil { // Error if we can't update the template but don't stop the show. glog.Error(err) } - glog.V(0).Infof("Time to update block template: %v", time.Since(now)) - now = time.Now() diffTarget, blockToMine := desoMiner._mineSingleBlock(threadIndex) if blockToMine == nil { return nil, fmt.Errorf("DeSoMiner._startThread: _mineSingleBlock returned nil; should only happen if we're stopping") } - glog.V(0).Infof("Time to update mine single block: %v", time.Since(now)) - now = time.Now() // Log information on the block we just mined. bestHash, _ := blockToMine.Hash() @@ -222,8 +208,7 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo desoMiner.BlockProducer.chain.blockTip().Header) scs := spew.ConfigState{DisableMethods: true, Indent: " ", DisablePointerAddresses: true} glog.V(1).Infof(scs.Sdump(blockToMine)) - //glog.V(0).Infof("Time to print a bunch of stuff: %v", time.Since(now)) - now = time.Now() + // Sanitize the block for the comparison we're about to do. We need to do // this because the comparison function below will think they're different // if one has nil and one has an empty list. Annoying, but this solves the @@ -258,8 +243,6 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if err := desoMiner.BlockProducer.SignBlock(blockToMine); err != nil { return nil, fmt.Errorf("Error signing block: %v", err) } - //glog.V(0).Infof("Time to sign block: %v", time.Since(now)) - now = time.Now() // Process the block. If the block is connected and/or accepted, the Server // will be informed about it. This will cause it to be relayed appropriately. @@ -277,8 +260,6 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo return blockToMine, fmt.Errorf("ERROR calling ProcessBlock: isMainChain=(%v), isOrphan=(%v), err=(%v)", isMainChain, isOrphan, err) } - glog.V(0).Infof("Time to process block: %v", time.Since(now)) - now = time.Now() // If a mempool object is passed then update it. Normally this isn't necessary because // ProcessBlock will trigger it because the backendServer will be set on the blockchain @@ -286,8 +267,6 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if mempoolToUpdate != nil { mempoolToUpdate.UpdateAfterConnectBlock(blockToMine) } - //glog.V(0).Infof("Time to update mempool after connect block: %v", time.Since(now)) - now = time.Now() decimalPlaces := int64(1000) diffTargetBaseline, _ := hex.DecodeString(desoMiner.params.MinDifficultyTargetHex) @@ -300,7 +279,6 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if atomic.LoadInt32(&desoMiner.stopping) == 1 { return nil, fmt.Errorf("DeSoMiner._startThread: Stopping thread %d", threadIndex) } - //glog.V(0).Infof("Time to update everything else: %v", time.Since(now)) return blockToMine, nil } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 8dddc5213..47d1147c1 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -317,14 +317,14 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { t.Cleanup(resetGlobalDeSoParams) hash1 := NewBlockHash(RandomBytes(32)) hash2 := NewBlockHash(RandomBytes(32)) - genesisNode := NewBlockNode(nil, hash1, 1, nil, nil, &MsgDeSoHeader{ + genesisNode := NewBlockNode(nil, hash1, 0, nil, nil, &MsgDeSoHeader{ Version: 2, - Height: 1, + Height: 0, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated) - block2 := NewBlockNode(genesisNode, hash2, 2, nil, nil, &MsgDeSoHeader{ + block2 := NewBlockNode(genesisNode, hash2, 1, nil, nil, &MsgDeSoHeader{ Version: 2, - Height: 2, + Height: 1, ProposedInView: 2, ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, @@ -1880,6 +1880,11 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { timeoutBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(timeoutBlockHash, timeoutBlockHeight) require.True(t, exists) require.False(t, timeoutBlockNode.IsCommitted()) + + // The reorg block hash should be in the block index now. + reorgBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(reorgBlockHash, reorgBlock.Header.Height) + require.True(t, exists) + require.True(t, reorgBlockNode.IsStored()) } var dummyParentBlockHash, orphanBlockHash *BlockHash { diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index 81e785c5b..74ab70dca 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -104,7 +104,7 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { blockIndex := NewBlockIndex(nil, nil) blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ - *blockHash: {Header: blockHeader}, + *blockHash: {Header: blockHeader, Height: uint32(blockHeader.Height), Hash: blockHash}, }) // Create a mock consensus From 722dcc0afbadef707f5071879de1edcad5d680f7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 16:12:27 -0400 Subject: [PATCH 14/62] fix string slice workaround, fix calc next difficulty --- cmd/config.go | 5 +++-- lib/blockchain.go | 10 ++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 77ce94606..fe04cbac7 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -94,7 +94,8 @@ type Config struct { func GetStringSliceWorkaround(flagName string) []string { value := viper.GetString(flagName) if value == "" || value == " " { - return []string{} + values := viper.GetStringSlice(flagName) + return values } return strings.Split(value, ",") } @@ -147,7 +148,6 @@ func LoadConfig() *Config { // Peers config.ConnectIPs = GetStringSliceWorkaround("connect-ips") - glog.V(2).Infof("Connect IPs read in: %v", config.ConnectIPs) config.AddIPs = GetStringSliceWorkaround("add-ips") config.AddSeeds = GetStringSliceWorkaround("add-seeds") config.TargetOutboundPeers = viper.GetUint32("target-outbound-peers") @@ -266,6 +266,7 @@ func (config *Config) Print() { glog.Infof("MaxSyncBlockHeight: %v", config.MaxSyncBlockHeight) } + glog.Infof("Connect IPs: %s", config.ConnectIPs) if len(config.ConnectIPs) > 0 { glog.Infof("Connect IPs: %s", config.ConnectIPs) } diff --git a/lib/blockchain.go b/lib/blockchain.go index 56bb03ea6..9315999b3 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -475,14 +475,8 @@ func (bc *Blockchain) CalcNextDifficultyTarget( maxRetargetTimeSecs := targetSecs * bc.params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget - // TODO: this needs to be replaced with a call to GetBlockNodeByHeight - firstNode, exists, err := bc.bestChain.GetBlockByHeight(uint64(firstNodeHeight)) - if err != nil { - return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem getting block at "+ - "beginning of retarget interval at height %d during retarget from height %d", - firstNodeHeight, lastNode.Height) - } - if !exists || firstNode == nil { + firstNode := lastNode.Ancestor(firstNodeHeight, bc.blockIndex) + if firstNode == nil { return nil, fmt.Errorf("CalcNextDifficultyTarget: Problem getting block at "+ "beginning of retarget interval at height %d during retarget from height %d", firstNodeHeight, lastNode.Height) From 1941487365703543c6639382476f7f6f52b6eae7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 16:22:31 -0400 Subject: [PATCH 15/62] use best header chain instead of best chain for calc next difficulty target --- lib/blockchain.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 9315999b3..6c58827c4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -475,14 +475,19 @@ func (bc *Blockchain) CalcNextDifficultyTarget( maxRetargetTimeSecs := targetSecs * bc.params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget - firstNode := lastNode.Ancestor(firstNodeHeight, bc.blockIndex) - if firstNode == nil { + firstNode, exists, err := bc.bestHeaderChain.GetBlockByHeight(uint64(firstNodeHeight)) + if err != nil { + return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem getting block at "+ + "beginning of retarget interval at height %d during retarget from height %d", + firstNodeHeight, lastNode.Height) + } + if firstNode == nil || !exists { return nil, fmt.Errorf("CalcNextDifficultyTarget: Problem getting block at "+ "beginning of retarget interval at height %d during retarget from height %d", firstNodeHeight, lastNode.Height) } - actualTimeDiffSecs := int64(lastNode.Header.GetTstampSecs() - firstNode.Header.GetTstampSecs()) + actualTimeDiffSecs := lastNode.Header.GetTstampSecs() - firstNode.Header.GetTstampSecs() clippedTimeDiffSecs := actualTimeDiffSecs if actualTimeDiffSecs < minRetargetTimeSecs { clippedTimeDiffSecs = minRetargetTimeSecs From 627e882cc0da5853c669fdca58cfc40b8d7cbbf5 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 23:22:39 -0400 Subject: [PATCH 16/62] fix tests --- lib/blockchain_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index adcf3f429..224a762f0 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1254,7 +1254,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1292,7 +1292,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1356,7 +1356,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1394,7 +1394,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1458,7 +1458,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1516,7 +1516,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1575,7 +1575,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1612,7 +1612,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } From a469c3a1a436b56988147a25493573d5f84212aa Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 3 Oct 2024 00:07:47 -0400 Subject: [PATCH 17/62] refactor to just use a block hash for best block chain and best header chain representation --- lib/block_view_lockups_test.go | 5 +- lib/block_view_test.go | 10 +- lib/blockchain.go | 727 ++++++++++++++++++++------------- lib/blockchain_test.go | 32 +- lib/db_utils.go | 2 +- lib/pos_blockchain.go | 96 +++-- lib/pos_blockchain_test.go | 99 +++-- lib/pos_consensus_test.go | 2 +- lib/server.go | 143 +++---- lib/state_change_syncer.go | 6 +- lib/txindex.go | 210 +++++----- 11 files changed, 765 insertions(+), 567 deletions(-) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index faaa6a3e3..c4f127892 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2461,8 +2461,9 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, err) require.NoError(t, utxoView.FlushToDb(blk2.Header.Height)) + // TODO: make sure this works? // Update the tip - testMeta.chain.bestChain.Chain = testMeta.chain.bestChain.Chain[:len(testMeta.chain.bestChain.Chain)-1] + testMeta.chain.blockIndex.tip = testMeta.chain.blockIndex.tip.Parent // Validate the state update utxoView = NewUtxoView( @@ -2517,7 +2518,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk1.Header.Height)) // Update the tip - testMeta.chain.bestChain.Chain = testMeta.chain.bestChain.Chain[:len(testMeta.chain.bestChain.Chain)-1] + testMeta.chain.blockIndex.SetTip(testMeta.chain.blockIndex.tip.Parent) // Verify we return back to the initial state utxoView = NewUtxoView( diff --git a/lib/block_view_test.go b/lib/block_view_test.go index b69307dbf..f02d06e36 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -702,8 +702,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te require.NoError(err) // sanity-check that the last block hash is the same as the last header hash. require.Equal(true, bytes.Equal( - tm.chain.bestChain.GetTip().Hash.ToBytes(), - tm.chain.bestHeaderChain.GetTip().Hash.ToBytes())) + tm.chain.blockIndex.GetTip().Hash.ToBytes(), + tm.chain.blockIndex.GetHeaderTip().Hash.ToBytes())) // Last block shouldn't be nil, and the number of expectedTxns should be the same as in the testVectorBlock + 1, // because of the additional block reward. require.NotNil(lastBlock) @@ -791,10 +791,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // TODO: if ever needed we can call tm.chain.eventManager.blockDisconnected() here. // Update the block and header metadata chains. - tm.chain.bestChain.Chain = tm.chain.bestChain.Chain[:len(tm.chain.bestChain.Chain)-1] - tm.chain.bestHeaderChain.Chain = tm.chain.bestHeaderChain.Chain[:len(tm.chain.bestHeaderChain.Chain)-1] - tm.chain.bestChain.ChainMap.Remove(*lastBlockHash) - tm.chain.bestHeaderChain.ChainMap.Remove(*lastBlockHash) + tm.chain.blockIndex.SetTip(tm.chain.BlockTip().GetParent(tm.chain.blockIndex)) + tm.chain.blockIndex.SetHeaderTip(tm.chain.HeaderTip().GetParent(tm.chain.blockIndex)) // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. diff --git a/lib/blockchain.go b/lib/blockchain.go index 6c58827c4..12ab58072 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -125,7 +125,12 @@ func (nn *BlockNode) IsValidateFailed() bool { // IsCommitted returns true if a BlockNode has passed all validations, and it has been committed to // the Blockchain according to the Fast HotStuff commit rule. func (nn *BlockNode) IsCommitted() bool { - return nn.Status&StatusBlockCommitted != 0 || !blockNodeProofOfStakeCutoverMigrationTriggered(nn.Height) + //return nn.Status&StatusBlockCommitted != 0 || !blockNodeProofOfStakeCutoverMigrationTriggered(nn.Height) + return nn.Status&StatusBlockCommitted != 0 +} + +func (nn *BlockNode) ClearCommittedStatus() { + nn.Status &= BlockStatus(^uint32(StatusBlockCommitted)) } // IsFullyProcessed determines if the BlockStatus corresponds to a fully processed and stored block. @@ -238,6 +243,20 @@ func (nn *BlockNode) GetEncoderType() EncoderType { return EncoderTypeBlockNode } +func (nn *BlockNode) GetParent(blockIndex *BlockIndex) *BlockNode { + if nn.Parent != nil { + return nn.Parent + } + // If we don't have a parent, try to get it from the block index. + parentNode, exists := blockIndex.GetBlockNodeByHashAndHeight(nn.Header.PrevBlockHash, uint64(nn.Height-1)) + if !exists { + return nil + } + + nn.Parent = parentNode + return parentNode +} + // Append DeSo Encoder Metadata bytes to BlockNode bytes. func AddEncoderMetadataToBlockNodeBytes(blockNodeBytes []byte, blockHeight uint64) []byte { var blockData []byte @@ -475,7 +494,10 @@ func (bc *Blockchain) CalcNextDifficultyTarget( maxRetargetTimeSecs := targetSecs * bc.params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget - firstNode, exists, err := bc.bestHeaderChain.GetBlockByHeight(uint64(firstNodeHeight)) + // TODO: we need to write the migration to only have committed blocks from PoW. + // This code is dead for PoS. + // TODO: do we need to do something if we need to get this from the header chain? + firstNode, exists, err := bc.GetBlockFromBestChainByHeight(uint64(firstNodeHeight), true) if err != nil { return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem getting block at "+ "beginning of retarget interval at height %d during retarget from height %d", @@ -547,18 +569,19 @@ type BlockIndex struct { snapshot *Snapshot blockIndexByHash *lru2.Cache[BlockHash, *BlockNode] blockIndexByHeight *lru2.Cache[uint64, []*BlockNode] - //maxHeightSeen uint64 + tip *BlockNode + headerTip *BlockNode } -func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { - blockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - blockIndexByHeight, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) +func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { + blockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + blockIndexByHeight, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? return &BlockIndex{ db: db, snapshot: snapshot, - blockIndexByHash: blockIndexByHash, // TODO: parameterize this? - blockIndexByHeight: blockIndexByHeight, // TODO: parameterize this? - //maxHeightSeen: 0, + blockIndexByHash: blockIndexByHash, + blockIndexByHeight: blockIndexByHeight, + tip: tipNode, } } @@ -569,9 +592,28 @@ func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { bi.blockIndexByHeight = newHeightToBlockNodeMap for _, val := range input { bi.addNewBlockNodeToBlockIndex(val) + // This function is always used for tests. + // We assume that the tip is just the highest block in the block index. + if bi.tip == nil { + bi.tip = val + } else if val.Height > bi.tip.Height { + bi.tip = val + } } } +func (bi *BlockIndex) SetHeaderTip(tip *BlockNode) { + // Just to be safe, we also add it to the block index. + bi.addNewBlockNodeToBlockIndex(tip) + bi.headerTip = tip +} + +func (bi *BlockIndex) SetTip(tip *BlockNode) { + // Just to be safe, we also add it to the block index. + bi.addNewBlockNodeToBlockIndex(tip) + bi.tip = tip +} + func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { bi.blockIndexByHash.Add(*blockNode.Hash, blockNode) blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) @@ -579,9 +621,10 @@ func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { blocksAtHeight = []*BlockNode{} } else { // Make sure we don't add the same block node twice. - for _, blockAtHeight := range blocksAtHeight { + for ii, blockAtHeight := range blocksAtHeight { if blockAtHeight.Hash.IsEqual(blockNode.Hash) { - return + blocksAtHeight[ii] = blockNode + break } } } @@ -651,147 +694,155 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { return blockNodes } -type BestChain struct { - db *badger.DB - snapshot *Snapshot - IsHeaderChain bool - Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. - ChainMap *lru2.Cache[BlockHash, *BlockNode] - params *DeSoParams -} - -func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { - chainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - return &BestChain{ - db: db, - snapshot: snapshot, - IsHeaderChain: isHeaderChain, - Chain: []*BlockNode{}, - ChainMap: chainMap, - params: params, - } -} - -func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { - bestChain.Chain = append(bestChain.Chain, tipNode) - bestChain.ChainMap.Add(*tipNode.Hash, tipNode) +func (bi *BlockIndex) GetTip() *BlockNode { + return bi.tip } -func (bestChain *BestChain) GetTip() *BlockNode { - if len(bestChain.Chain) == 0 { - return nil - } - return bestChain.Chain[len(bestChain.Chain)-1] +func (bi *BlockIndex) GetHeaderTip() *BlockNode { + return bi.headerTip } -func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, error) { - if height > math.MaxUint32 { - return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) - } - - currTip := bestChain.GetTip() - if currTip != nil && height > uint64(currTip.Height) { - return nil, false, nil - } - - if currTip != nil { - currTipHeight := currTip.Height - delta := currTipHeight - uint32(height) - if delta < uint32(len(bestChain.Chain)) { - targetNode := bestChain.Chain[uint32(len(bestChain.Chain)-1)-delta] - if uint64(targetNode.Height) == height { - return targetNode, true, nil - } - } - //targetIndex := (len(bestChain.Chain)-1) - //// We can probably do some binary search thing here instead. - //currNode := &BlockNode{} - //*currNode = *currTip - //if currNode.Height == uint32(height) { - // return currNode, true, nil - //} - //// During syncing, we don't write to the DB, so we need to iterate backwards through the best chain. - //for currNode != nil && !currNode.IsStored() { - // if currNode.Height < uint32(height) { - // break - // } - // if currNode.Height == uint32(height) { - // return currNode, true, nil - // } - // var currNodeExists bool - // currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( - // currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) - // if !currNodeExists { - // break - // } - //} - } - - prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) - _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) - if len(valsFound) == 0 { - return nil, false, nil - } - for _, val := range valsFound { - blockNode, err := DeserializeBlockNode(val) - if err != nil { - glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) - continue - } - if blockNode.IsCommitted() { - return blockNode, true, nil - } - } - // TODO: how to return uncommitted blocks by height. We probably just need to iterate backwards through the - // best chain. - return nil, false, nil -} - -func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { - val, exists := bestChain.ChainMap.Get(*blockHash) - if exists { - return val, true - } - if height > math.MaxUint32 { - glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) - } - if height > uint64(bestChain.GetTip().Height) { - return nil, false - } - bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) - if bn == nil { - return nil, false - } - bestChain.ChainMap.Add(*blockHash, bn) - return bn, true -} - -func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bool, error) { - val, exists := bestChain.ChainMap.Get(*blockHash) - if exists { - return val, true, nil - } - - height, err := GetHeightForHash(bestChain.db, bestChain.snapshot, blockHash) - if err != nil { - if errors.Is(err, badger.ErrKeyNotFound) { - return nil, false, nil - } - return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") - } - if height > uint64(bestChain.GetTip().Height) { - return nil, false, nil - } - bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) - if bn == nil { - return nil, false, nil - } - if !bn.IsCommitted() { - return nil, false, nil - } - bestChain.ChainMap.Add(*blockHash, bn) - return bn, true, nil -} +//type BestChain struct { +// db *badger.DB +// snapshot *Snapshot +// IsHeaderChain bool +// Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. +// ChainMap *lru2.Cache[BlockHash, *BlockNode] +// params *DeSoParams +//} +// +//func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { +// chainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) +// return &BestChain{ +// db: db, +// snapshot: snapshot, +// IsHeaderChain: isHeaderChain, +// Chain: []*BlockNode{}, +// ChainMap: chainMap, +// params: params, +// } +//} +// +//func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { +// bestChain.Chain = append(bestChain.Chain, tipNode) +// bestChain.ChainMap.Add(*tipNode.Hash, tipNode) +//} +// +//func (bestChain *BestChain) GetTip() *BlockNode { +// if len(bestChain.Chain) == 0 { +// return nil +// } +// return bestChain.Chain[len(bestChain.Chain)-1] +//} +// +//func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, error) { +// if height > math.MaxUint32 { +// return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) +// } +// +// currTip := bestChain.GetTip() +// if currTip != nil && height > uint64(currTip.Height) { +// return nil, false, nil +// } +// +// if currTip != nil { +// currTipHeight := currTip.Height +// delta := currTipHeight - uint32(height) +// if delta < uint32(len(bestChain.Chain)) { +// targetNode := bestChain.Chain[uint32(len(bestChain.Chain)-1)-delta] +// if uint64(targetNode.Height) == height { +// return targetNode, true, nil +// } +// } +// //targetIndex := (len(bestChain.Chain)-1) +// //// We can probably do some binary search thing here instead. +// //currNode := &BlockNode{} +// //*currNode = *currTip +// //if currNode.Height == uint32(height) { +// // return currNode, true, nil +// //} +// //// During syncing, we don't write to the DB, so we need to iterate backwards through the best chain. +// //for currNode != nil && !currNode.IsStored() { +// // if currNode.Height < uint32(height) { +// // break +// // } +// // if currNode.Height == uint32(height) { +// // return currNode, true, nil +// // } +// // var currNodeExists bool +// // currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( +// // currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) +// // if !currNodeExists { +// // break +// // } +// //} +// } +// +// prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) +// _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) +// if len(valsFound) == 0 { +// return nil, false, nil +// } +// for _, val := range valsFound { +// blockNode, err := DeserializeBlockNode(val) +// if err != nil { +// glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) +// continue +// } +// if blockNode.IsCommitted() { +// return blockNode, true, nil +// } +// } +// // TODO: how to return uncommitted blocks by height. We probably just need to iterate backwards through the +// // best chain. +// return nil, false, nil +//} +// +//func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { +// val, exists := bestChain.ChainMap.Get(*blockHash) +// if exists { +// return val, true +// } +// if height > math.MaxUint32 { +// glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) +// } +// if height > uint64(bestChain.GetTip().Height) { +// return nil, false +// } +// bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) +// if bn == nil { +// return nil, false +// } +// bestChain.ChainMap.Add(*blockHash, bn) +// return bn, true +//} +// +//func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bool, error) { +// val, exists := bestChain.ChainMap.Get(*blockHash) +// if exists { +// return val, true, nil +// } +// +// height, err := GetHeightForHash(bestChain.db, bestChain.snapshot, blockHash) +// if err != nil { +// if errors.Is(err, badger.ErrKeyNotFound) { +// return nil, false, nil +// } +// return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") +// } +// if height > uint64(bestChain.GetTip().Height) { +// return nil, false, nil +// } +// bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) +// if bn == nil { +// return nil, false, nil +// } +// if !bn.IsCommitted() { +// return nil, false, nil +// } +// bestChain.ChainMap.Add(*blockHash, bn) +// return bn, true, nil +//} type Blockchain struct { db *badger.DB @@ -834,9 +885,9 @@ type Blockchain struct { //bestHeaderChain []*BlockNode //bestHeaderChainMap map[BlockHash]*BlockNode - blockIndex *BlockIndex - bestChain *BestChain - bestHeaderChain *BestChain + blockIndex *BlockIndex + //bestChain *BestChain + //bestHeaderChain *BestChain // We keep track of orphan blocks with the following data structures. Orphans // are not written to disk and are only cached in memory. Moreover we only keep @@ -1004,29 +1055,29 @@ func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { return len(blockNodes) > 0 } -func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { - newBestChain := []*BlockNode{} - newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - newBestChain = append(newBestChain, bc.bestChain.Chain...) - for _, key := range bc.bestChain.ChainMap.Keys() { - val, _ := bc.bestChain.ChainMap.Get(key) - newBestChainMap.Add(key, val) - } - - return newBestChain, newBestChainMap -} - -func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { - newBestChain := []*BlockNode{} - newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) - for _, key := range bc.bestHeaderChain.ChainMap.Keys() { - val, _ := bc.bestHeaderChain.ChainMap.Get(key) - newBestChainMap.Add(key, val) - } - - return newBestChain, newBestChainMap -} +//func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { +// newBestChain := []*BlockNode{} +// newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) +// newBestChain = append(newBestChain, bc.bestChain.Chain...) +// for _, key := range bc.bestChain.ChainMap.Keys() { +// val, _ := bc.bestChain.ChainMap.Get(key) +// newBestChainMap.Add(key, val) +// } +// +// return newBestChain, newBestChainMap +//} + +//func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { +// newBestChain := []*BlockNode{} +// newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) +// newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) +// for _, key := range bc.bestHeaderChain.ChainMap.Keys() { +// val, _ := bc.bestHeaderChain.ChainMap.Get(key) +// newBestChainMap.Add(key, val) +// } +// +// return newBestChain, newBestChainMap +//} // IsFullyStored determines if there are block nodes that haven't been fully stored or processed in the best block chain. func (bc *Blockchain) IsFullyStored() bool { @@ -1034,11 +1085,13 @@ func (bc *Blockchain) IsFullyStored() bool { chainState := bc.ChainState() if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && bc.headerTip().Height-bc.blockTip().Height < 10) { - for _, blockNode := range bc.bestChain.Chain { - if !blockNode.Status.IsFullyProcessed() { - return false - } - } + // TODO: rewrite to work w/ block index. + panic("IsFullyStored: Need to implement this.") + //for _, blockNode := range bc.bestChain.Chain { + // if !blockNode.Status.IsFullyProcessed() { + // return false + // } + //} return true } return false @@ -1142,6 +1195,10 @@ func (bc *Blockchain) _initChain() error { return errors.Wrapf(err, "_initChain: Problem loading block index from db") } + // We start by simply setting the chain tip and header tip to the tip node. + bc.blockIndex.tip = tipNode + bc.blockIndex.headerTip = tipNode + } // At this point the blockIndexByHash should contain a full node tree with all @@ -1149,26 +1206,26 @@ func (bc *Blockchain) _initChain() error { { // Walk back from the best node to the genesis block and store them all // in bestChain. - bc.bestChain.Chain, err = GetBestChain(tipNode) - if err != nil { - return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") - } - for _, bestChainNode := range bc.bestChain.Chain { - bc.bestChain.ChainMap.Add(*bestChainNode.Hash, bestChainNode) - } + //bc.bestChain.Chain, err = GetBestChain(tipNode) + //if err != nil { + // return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") + //} + //for _, bestChainNode := range bc.bestChain.Chain { + // bc.bestChain.ChainMap.Add(*bestChainNode.Hash, bestChainNode) + //} } // TODO: This code is a bit repetitive but this seemed clearer than factoring it out. { // Walk back from the best node to the genesis block and store them all // in bestChain. - bc.bestHeaderChain.Chain, err = GetBestChain(tipNode) - if err != nil { - return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") - } - for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { - bc.bestHeaderChain.ChainMap.Add(*bestHeaderChainNode.Hash, bestHeaderChainNode) - } + //bc.bestHeaderChain.Chain, err = GetBestChain(tipNode) + //if err != nil { + // return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") + //} + //for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { + // bc.bestHeaderChain.ChainMap.Add(*bestHeaderChainNode.Hash, bestHeaderChainNode) + //} } bc.isInitialized = true @@ -1216,15 +1273,17 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { return errors.Wrapf(err, "_applyUncommittedBlocksToBestChain: ") } + bc.blockIndex.SetTip(uncommittedTipBlockNode) + ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// - currentHeaderTip := bc.headerTip() - _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) - bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - bc.bestHeaderChain.Chain, - bc.bestHeaderChain.ChainMap, - blocksToDetach, - blocksToAttach, - ) + //currentHeaderTip := bc.headerTip() + //_, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) + //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + // bc.bestHeaderChain.Chain, + // bc.bestHeaderChain.ChainMap, + // blocksToDetach, + // blocksToAttach, + //) return nil } @@ -1273,15 +1332,15 @@ func NewBlockchain( eventManager: eventManager, archivalMode: archivalMode, - blockIndex: NewBlockIndex(db, snapshot), + blockIndex: NewBlockIndex(db, snapshot, nil), // TODO: replace with actual tip. //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), //bestChainMap: make(map[BlockHash]*BlockNode), - bestChain: NewBestChain(db, snapshot, false, params), + //bestChain: NewBestChain(db, snapshot, false, params), //bestHeaderChainMap: make(map[BlockHash]*BlockNode), - bestHeaderChain: NewBestChain(db, snapshot, true, params), - blockViewCache: lru.NewKVCache(100), // TODO: parameterize - snapshotCache: NewSnapshotCache(), + //bestHeaderChain: NewBestChain(db, snapshot, true, params), + blockViewCache: lru.NewKVCache(100), // TODO: parameterize + snapshotCache: NewSnapshotCache(), checkpointSyncingProviders: checkpointSyncingProviders, @@ -1346,12 +1405,12 @@ func fastLog2Floor(n uint32) uint8 { // functions. // // This function MUST be called with the chain state lock held (for reads). -func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32, - blockIndex *BlockIndex, bestChain *BestChain) (*BlockNode, uint32) { +// TODO: this function needs a whole bunch of work. +func (bc *Blockchain) locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32) (*BlockNode, uint32) { // There are no block locators so a specific block is being requested // as identified by the stop hash. - stopNode, stopNodeExists, stopNodeError := blockIndex.GetBlockNodeByHashOnly(stopHash) + stopNode, stopNodeExists, stopNodeError := bc.GetBlockFromBestChainByHash(stopHash, true) if len(locator) == 0 { if stopNodeError != nil || !stopNodeExists || stopNode == nil { // TODO: what should we really do here? @@ -1366,7 +1425,7 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // Find the most recent locator block hash in the main chain. In the // case none of the hashes in the locator are in the main chain, fall // back to the genesis block. - startNode, startNodeExists, err := bestChain.GetBlockByHeight(0) + startNode, startNodeExists, err := bc.GetBlockFromBestChainByHeight(0, true) if err != nil { glog.Errorf("locateInventory: Problem getting block by height: %v", err) return nil, 0 @@ -1377,12 +1436,8 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 } for _, hash := range locator { // TODO: replace w/ read-through cache call. - node, bestChainContainsNode, err := bestChain.GetBlockByHash(hash) - if err != nil { - glog.Errorf("locateInventory: Problem getting block by hash: %v", err) - continue - } - if bestChainContainsNode { + node := bc.GetBlockNodeWithHash(hash) + if node != nil { startNode = node break } @@ -1392,7 +1447,7 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // is no next block it means the most recently known block is the tip of // the best chain, so there is nothing more to do. nextNodeHeight := uint32(startNode.Header.Height) + 1 - startNode, startNodeExists, err = bestChain.GetBlockByHeight(uint64(nextNodeHeight)) + startNode, startNodeExists, err = bc.GetBlockFromBestChainByHeight(uint64(nextNodeHeight), true) if err != nil { glog.Errorf("locateInventory: Problem getting block by height: %v", err) return nil, 0 @@ -1402,11 +1457,11 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 } // Calculate how many entries are needed. - total := (bestChain.GetTip().Height - startNode.Height) + 1 + total := (bc.blockIndex.GetTip().Height - startNode.Height) + 1 if stopNodeError != nil && stopNodeExists && stopNode != nil && stopNode.Header.Height >= startNode.Header.Height { - _, bestChainContainsStopNode, err := bestChain.GetBlockByHash(stopNode.Hash) + _, bestChainContainsStopNode, err := bc.blockIndex.GetBlockNodeByHashOnly(stopNode.Hash) if err != nil { glog.Errorf("locateInventory: Problem getting block by hash: %v", err) return nil, 0 @@ -1429,14 +1484,12 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // See the comment on the exported function for more details on special cases. // // This function MUST be called with the ChainLock held (for reads). -func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, - blockIndex *BlockIndex, bestChain *BestChain) []*MsgDeSoHeader { +func (bc *Blockchain) locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32) []*MsgDeSoHeader { // Find the node after the first known block in the locator and the // total number of nodes after it needed while respecting the stop hash // and max entries. - node, total := locateInventory(locator, stopHash, maxHeaders, - blockIndex, bestChain) + node, total := bc.locateInventory(locator, stopHash, maxHeaders) if total == 0 { return nil } @@ -1452,7 +1505,7 @@ func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, break } var nodeExists bool - node, nodeExists, err = bestChain.GetBlockByHeight(node.Header.Height + 1) + node, nodeExists, err = bc.GetBlockFromBestChainByHeight(node.Header.Height+1, true) if err != nil { glog.Errorf("locateHeaders: Problem getting block by height: %v", err) break @@ -1487,8 +1540,7 @@ func (bc *Blockchain) LocateBestBlockChainHeaders( // TODO: Shouldn't we hold a ChainLock here? I think it's fine though because the place // where it's currently called is single-threaded via a channel in server.go. Going to // avoid messing with it for now. - headers := locateHeaders(locator, stopHash, maxHeaders, - bc.blockIndex, bc.bestChain) + headers := bc.locateHeaders(locator, stopHash, maxHeaders) return headers } @@ -1550,14 +1602,14 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { // ancestors must be too, so use a much faster O(1) lookup in // that case. Otherwise, fall back to walking backwards through // the nodes of the other chain to the correct ancestor. - _, exists, err := bc.bestHeaderChain.GetBlockByHash(tip.Hash) + _, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(tip.Hash) if err != nil { glog.Errorf("LatestLocator: Problem getting block by hash: %v", err) exists = false } if exists { var innerExists bool - tip, innerExists, err = bc.bestHeaderChain.GetBlockByHeight(uint64(height)) + tip, innerExists, err = bc.GetBlockFromBestChainByHeight(uint64(height), true) if err != nil { glog.Errorf("LatestLocator: Problem getting block by height: %v", err) break @@ -1614,7 +1666,11 @@ func (bc *Blockchain) GetBlockNodesToFetch( // If the tip of the best block chain is in the main header chain, make that // the start point for our fetch. - headerNodeStart, blockTipExistsInBestHeaderChain := bc.bestHeaderChain.GetBlockByHashAndHeight(bestBlockTip.Hash, uint64(bestBlockTip.Height)) + headerNodeStart, blockTipExistsInBestHeaderChain, err := bc.GetBlockFromBestChainByHeight(uint64(bestBlockTip.Height), true) + if err != nil { + glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) + return nil + } if !blockTipExistsInBestHeaderChain { // If the hash of the tip of the best blockchain is not in the best header chain, then // this is a case where the header chain has forked off from the best block @@ -1635,7 +1691,7 @@ func (bc *Blockchain) GetBlockNodesToFetch( "backward through best header chain; using genesis block") var err error var genesisBlockExists bool - headerNodeStart, genesisBlockExists, err = bc.bestHeaderChain.GetBlockByHeight(0) + headerNodeStart, genesisBlockExists, err = bc.GetBlockFromBestChainByHeight(0, true) if err != nil { glog.Errorf("GetBlockToFetch: Problem getting genesis block: %v", err) return nil @@ -1653,14 +1709,14 @@ func (bc *Blockchain) GetBlockNodesToFetch( currentHeight := headerNodeStart.Height + 1 blockNodesToFetch := []*BlockNode{} heightLimit := maxHeight - if heightLimit >= bc.bestHeaderChain.GetTip().Height { - heightLimit = bc.bestHeaderChain.GetTip().Height - 1 + if heightLimit >= bc.blockIndex.GetHeaderTip().Height { + heightLimit = bc.blockIndex.GetHeaderTip().Height - 1 } for currentHeight <= heightLimit && len(blockNodesToFetch) < numBlocks { // Get the current hash and increment the height. - currentNode, currentNodeExists, err := bc.bestHeaderChain.GetBlockByHeight(uint64(currentHeight)) + currentNode, currentNodeExists, err := bc.GetBlockFromBestChainByHeight(uint64(currentHeight), true) if err != nil { glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) return nil @@ -1697,10 +1753,10 @@ func (bc *Blockchain) HasHeaderByHashAndHeight(headerHash *BlockHash, height uin // TODO: delete me? func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) (*BlockNode, bool, error) { - if blockHeight >= bc.bestChain.GetTip().Height { + if blockHeight >= bc.blockIndex.GetHeaderTip().Height { return nil, false, nil } - return bc.bestChain.GetBlockByHeight(uint64(blockHeight)) + return bc.GetBlockFromBestChainByHeight(uint64(blockHeight), true) } func (bc *Blockchain) HasBlockInBlockIndex(blockHash *BlockHash) (bool, error) { @@ -1722,8 +1778,8 @@ func (bc *Blockchain) GetBlock(blockHash *BlockHash) *MsgDeSoBlock { return blk } -func (bc *Blockchain) GetBlockAtHeight(height uint32) (*MsgDeSoBlock, error) { - bn, bnExists, err := bc.bestChain.GetBlockByHeight(uint64(height)) +func (bc *Blockchain) GetBlockAtHeight(height uint32, isHeaderChain bool) (*MsgDeSoBlock, error) { + bn, bnExists, err := bc.GetBlockFromBestChainByHeight(uint64(height), isHeaderChain) if !bnExists || err != nil { glog.Errorf("Blockchain.GetBlockAtHeight: Problem getting block by height: %v", err) return nil, err @@ -1737,7 +1793,7 @@ func (bc *Blockchain) GetBlockNodeWithHash(hash *BlockHash) *BlockNode { if hash == nil { return nil } - bn, bnExists, err := bc.bestChain.GetBlockByHash(hash) + bn, bnExists, err := bc.blockIndex.GetBlockNodeByHashOnly(hash) if !bnExists || err != nil { return nil } @@ -1904,24 +1960,26 @@ func (bc *Blockchain) checkArchivalMode() bool { } firstSnapshotHeight := bc.snapshot.CurrentEpochSnapshotMetadata.FirstSnapshotBlockHeight + _ = firstSnapshotHeight // @diamondhands - can we spot check just a few blocks such as firstSnapshotHeight - 1, // firstSnapshotHeight / 2 - 1, and firstSnapshotHeight / 4 - 1 to see if they are stored? // TODO: figure out how to iterate over best chain to checkArchivalMode // when we only have a portion of best chain in memory. - for _, blockNode := range bc.bestChain.Chain { - if uint64(blockNode.Height) > firstSnapshotHeight { - return false - } - - // Check if we have blocks that have been processed and validated but not stored. This would indicate that there - // are historical blocks that we are yet to download. - if (blockNode.Status&StatusBlockProcessed) == 1 && - (blockNode.Status&StatusBlockValidated) == 1 && - (blockNode.Status&StatusBlockStored) == 0 { - - return true - } - } + panic("NOT IMPLEMENTED: checkArchivalMode") + //for _, blockNode := range bc.bestChain.Chain { + // if uint64(blockNode.Height) > firstSnapshotHeight { + // return false + // } + // + // // Check if we have blocks that have been processed and validated but not stored. This would indicate that there + // // are historical blocks that we are yet to download. + // if (blockNode.Status&StatusBlockProcessed) == 1 && + // (blockNode.Status&StatusBlockValidated) == 1 && + // (blockNode.Status&StatusBlockStored) == 0 { + // + // return true + // } + //} // If we get here, it means that all blocks have been processed and stored, so there is nothing to do. return false @@ -1958,7 +2016,7 @@ func (bc *Blockchain) isHyperSyncCondition() bool { // main chain for blocks, which is why separate functions are required for // each of them. func (bc *Blockchain) headerTip() *BlockNode { - return bc.bestHeaderChain.GetTip() + return bc.blockIndex.GetHeaderTip() } func (bc *Blockchain) HeaderTip() *BlockNode { @@ -1990,37 +2048,114 @@ func (bc *Blockchain) Snapshot() *Snapshot { // invalidate and chop off the headers corresponding to those blocks and // their ancestors so the two generally stay in sync. func (bc *Blockchain) blockTip() *BlockNode { - return bc.bestChain.GetTip() + return bc.blockIndex.GetTip() } func (bc *Blockchain) BlockTip() *BlockNode { return bc.blockTip() } +// TODO: this won't work for now. Need to figure out how to handle this. func (bc *Blockchain) BestChain() []*BlockNode { - return bc.bestChain.Chain + panic("BestChain not supported.") } -func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash) (*BlockNode, bool, error) { - return bc.bestChain.GetBlockByHash(blockHash) +func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeaderChain bool) (*BlockNode, bool, error) { + bn, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) + if err != nil { + return nil, false, err + } + if !exists { + return nil, false, nil + } + if bn.IsCommitted() { + return bn, true, nil // TODO: what do we do about header chain? they're not committed so we're going to + // have to get a bunch of parents in order to be sure it is part of the best header chain. I guess we could + // have a map, but kinda defeats the purpose of this refactor. + } + blockTip := bc.BlockTip() + if useHeaderChain { + blockTip = bc.HeaderTip() + } + if blockTip == nil { + return nil, false, fmt.Errorf("GetBlockFromBestChainByHash: Block tip not found: use header chain: %v", useHeaderChain) + } + committedTip, exists := bc.GetCommittedTip() + if !exists { + return nil, false, errors.New("GetBlockFromBestChainByHash: Committed tip not found") + } + if uint64(bn.Height) > uint64(blockTip.Height) { + return nil, false, nil + } + currNode := &BlockNode{} + *currNode = *blockTip + for currNode != nil && !currNode.Hash.IsEqual(blockHash) && currNode.Height > committedTip.Height { + currNode = currNode.GetParent(bc.blockIndex) + } + if currNode != nil && currNode.Hash.IsEqual(blockHash) { + return currNode, true, nil + } + return nil, false, nil } -func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64) (*BlockNode, bool, error) { - return bc.bestChain.GetBlockByHeight(height) +func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChain bool) (*BlockNode, bool, error) { + if !useHeaderChain { + committedTip, exists := bc.GetCommittedTip() + if !exists { + return nil, false, nil + } + if height >= uint64(committedTip.Height) { + // For this, we can just loop back from the tip block. + currentNode := bc.blockIndex.GetTip() + if useHeaderChain { + currentNode = bc.blockIndex.GetHeaderTip() + } + for currentNode != nil { + if uint64(currentNode.Height) == height { + return currentNode, true, nil + } + if currentNode.Height < committedTip.Height { + break + } + currentNode = currentNode.GetParent(bc.blockIndex) + } + return nil, false, nil + } + } + blockNodes := bc.blockIndex.GetBlockNodesByHeight(height) + if len(blockNodes) == 0 { + return nil, false, nil + } + for _, blockNode := range blockNodes { + if !useHeaderChain && blockNode.IsCommitted() { + return blockNode, true, nil + } + // TODO: this is crude and incorrect. + if useHeaderChain && blockNode.IsHeaderValidated() { + return blockNode, true, nil + } + } + return nil, false, nil } +// TODO: need to figure out how to handle this for exchange api tests. func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { - bc.bestChain.Chain = bestChain + for _, blockNode := range bestChain { + bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) + if bc.blockIndex.GetTip() == nil { + bc.blockIndex.SetTip(blockNode) + } else if bc.blockIndex.GetTip().Height < blockNode.Height { + bc.blockIndex.SetTip(blockNode) + } + } } func (bc *Blockchain) SetBestChainMap( - bestChain []*BlockNode, - bestChainMap *lru2.Cache[BlockHash, *BlockNode], blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], + tipNode *BlockNode, ) { - bc.bestChain.Chain = bestChain - bc.bestChain.ChainMap = bestChainMap bc.blockIndex.blockIndexByHash = blockIndexByHash + bc.blockIndex.SetTip(tipNode) } func (bc *Blockchain) _validateOrphanBlockPoW(desoBlock *MsgDeSoBlock) error { @@ -2293,6 +2428,9 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( "block (%v) at height (%d) to block (%v) at height of (%d)", numBlocks, tip, tip.Height, newNode, newNode.Height) } + } else { + glog.Fatal("GetReorgBlocks: Tip is nil") + return } // Get the blocks to detach. Start at the tip and work backwards to the @@ -2537,10 +2675,10 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B headerTip := bc.headerTip() if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - - _, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode) - bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) + bc.blockIndex.SetHeaderTip(newNode) + //_, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode) + //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + // bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) // Note that we don't store the best header hash here and so this is an // in-memory-only adjustment. See the comment above on preventing attacks. @@ -2963,6 +3101,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // update our data structures to actually make this connection. Do this // in a transaction so that it is atomic. if bc.postgres != nil { + // TODO: pg support for setting committed on block node. if err = bc.postgres.UpsertBlockAndTransactions(nodeToValidate, desoBlock); err != nil { return false, false, errors.Wrapf(err, "ProcessBlock: Problem upserting block and transactions") } @@ -2984,6 +3123,10 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures err = bc.db.Update(func(txn *badger.Txn) error { // This will update the node's status. bc.timer.Start("Blockchain.ProcessBlock: Transactions Db height & hash") + if !nodeToValidate.IsCommitted() { + nodeToValidate.Status |= StatusBlockCommitted + bc.blockIndex.addNewBlockNodeToBlockIndex(nodeToValidate) + } if innerErr := PutHeightHashToNodeInfoWithTxn(txn, bc.snapshot, nodeToValidate, false /*bitcoinNodes*/, bc.eventManager); innerErr != nil { return errors.Wrapf( innerErr, "ProcessBlock: Problem calling PutHeightHashToNodeInfo after validation") @@ -3030,8 +3173,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Now that we've set the best chain in the db, update our in-memory data // structure to reflect this. Do a quick check first to make sure it's consistent. - lastIndex := len(bc.bestChain.Chain) - 1 - bestChainHash := bc.bestChain.Chain[lastIndex].Hash + bestChainTip := bc.blockIndex.GetTip() + if bestChainTip == nil { + return false, false, fmt.Errorf("ProcessBlock: Best chain tip is nil") + } + bestChainHash := bestChainTip.Hash if !bestChainHash.IsEqual(nodeToValidate.Header.PrevBlockHash) { return false, false, fmt.Errorf("ProcessBlock: Last block in bestChain "+ @@ -3042,11 +3188,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If we're syncing there's no risk of concurrency issues. Otherwise, we // need to make a copy in order to be save. if bc.isSyncing() { - bc.bestChain.PushNewTip(nodeToValidate) + bc.blockIndex.SetTip(nodeToValidate) } else { - newBestChain, newBestChainMap := bc.CopyBestChain() - bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap - bc.bestChain.PushNewTip(nodeToValidate) + //newBestChain, newBestChainMap := bc.CopyBestChain() + //bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap + bc.blockIndex.SetTip(nodeToValidate) } // This node is on the main chain so set this variable. @@ -3271,6 +3417,14 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures if err := PutBestHashWithTxn(txn, bc.snapshot, newTipNode.Hash, ChainTypeDeSoBlock, bc.eventManager); err != nil { return err } + if !newTipNode.IsCommitted() { + newTipNode.Status |= StatusBlockCommitted + // update the block index to be safe. + bc.addNewBlockNodeToBlockIndex(newTipNode) + if err := PutHeightHashToNodeInfoWithTxn(txn, bc.snapshot, newTipNode, false, bc.eventManager); err != nil { + return err + } + } for _, detachNode := range detachBlocks { // Delete the utxo operations for the blocks we're detaching since we don't need @@ -3279,6 +3433,16 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures return errors.Wrapf(err, "ProcessBlock: Problem deleting utxo operations for block") } + // We also need to revert the committed state if applicable. + if detachNode.IsCommitted() { + detachNode.ClearCommittedStatus() + // update the block index to be safe. + bc.addNewBlockNodeToBlockIndex(detachNode) + if err = PutHeightHashToNodeInfoWithTxn(txn, bc.snapshot, detachNode, false, bc.eventManager); err != nil { + return errors.Wrapf(err, "ProcessBlock: Problem putting height hash to node info for detach node that is not committed.") + } + } + // Note we could be even more aggressive here by deleting the nodes and // corresponding blocks from the db here (i.e. not storing any side chain // data on the db). But this seems like a minor optimization that comes at @@ -3291,6 +3455,15 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures if err := PutUtxoOperationsForBlockWithTxn(txn, bc.snapshot, blockHeight, attachNode.Hash, utxoOpsForAttachBlocks[ii], bc.eventManager); err != nil { return errors.Wrapf(err, "ProcessBlock: Problem putting utxo operations for block") } + + if !attachNode.IsCommitted() { + attachNode.Status |= StatusBlockCommitted + // update the block index to be safe. + bc.addNewBlockNodeToBlockIndex(attachNode) + if err = PutHeightHashToNodeInfoWithTxn(txn, bc.snapshot, attachNode, false, bc.eventManager); err != nil { + return errors.Wrapf(err, "ProcessBlock: Problem putting height hash to node info for detach node that is not committed.") + } + } } // Write the modified utxo set to the view. @@ -3307,10 +3480,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Now the db has been updated, update our in-memory best chain. Note that there // is no need to update the node index because it was updated as we went along. - newBestChain, newBestChainMap := bc.CopyBestChain() - newBestChain, newBestChainMap = updateBestChainInMemory( - newBestChain, newBestChainMap, detachBlocks, attachBlocks) - bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap + bc.blockIndex.SetTip(newTipNode) + bc.blockIndex.SetHeaderTip(newTipNode) // If we made it here then this block is on the main chain. isMainChain = true diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 224a762f0..2bf1cab3d 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1252,9 +1252,9 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { // Blocks generating every 1 second, which is 2x too fast. TstampNanoSecs: SecondsToNanoSeconds(int64(ii)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1290,9 +1290,9 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { // Blocks generating every 4 second, which is 2x too slow. TstampNanoSecs: SecondsToNanoSeconds(int64(ii * 4)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1354,9 +1354,9 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { // Blocks generating every 1 second, which is 2x too fast. TstampNanoSecs: SecondsToNanoSeconds(int64(ii)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1392,9 +1392,9 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { // Blocks generating every 8 second, which is >2x too slow. TstampNanoSecs: SecondsToNanoSeconds(int64(ii * 4)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1456,9 +1456,9 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { // Blocks generating all at once. TstampNanoSecs: SecondsToNanoSeconds(0), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1514,9 +1514,9 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { // Blocks generating every 2 second, which is under the limit. TstampNanoSecs: SecondsToNanoSeconds(int64(ii * 2)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1573,9 +1573,9 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { // Blocks generating every 1 second, which is 2x too fast. TstampNanoSecs: SecondsToNanoSeconds(int64(ii)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1610,9 +1610,9 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { // Blocks generating every 3 seconds, which is slow but under the limit. TstampNanoSecs: SecondsToNanoSeconds(int64(ii) * 3), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } diff --git a/lib/db_utils.go b/lib/db_utils.go index cc3e4faee..de0de1bf8 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5360,7 +5360,7 @@ func InitDbWithDeSoGenesisBlock(params *DeSoParams, handle *badger.DB, diffTarget, BytesToBigint(ExpectedWorkForBlockHash(diffTarget)[:]), // CumWork genesisBlock.Header, // Header - StatusHeaderValidated|StatusBlockProcessed|StatusBlockStored|StatusBlockValidated, // Status + StatusHeaderValidated|StatusBlockProcessed|StatusBlockStored|StatusBlockValidated|StatusBlockCommitted, // Status ) // Set the fields in the db to reflect the current state of our chain. diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 134505eab..7b5dd2c86 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -58,7 +58,13 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. - if _, isInBestHeaderChain := bc.bestHeaderChain.ChainMap.Get(*headerHash); isInBestHeaderChain { + // TODO: make sure this is ok or do we need to explicitly check the block index's cache? + var isInBestHeaderChain bool + _, isInBestHeaderChain, err = bc.GetBlockFromBestChainByHash(headerHash, true) + if err != nil { + return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") + } + if isInBestHeaderChain { return true, false, nil } @@ -91,15 +97,17 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b return false, false, nil } + bc.blockIndex.SetHeaderTip(blockNode) + // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. - _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode) - bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - bc.bestHeaderChain.Chain, - bc.bestHeaderChain.ChainMap, - blocksToDetach, - blocksToAttach, - ) + //_, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode) + //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + // bc.bestHeaderChain.Chain, + // bc.bestHeaderChain.ChainMap, + // blocksToDetach, + // blocksToAttach, + //) // Success. The header is at the tip of the best header chain. return true, false, nil @@ -1320,8 +1328,8 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( _missingBlockHashes []*BlockHash, _err error, ) { - highestCommittedBlock, idx := bc.GetCommittedTip() - if idx == -1 || highestCommittedBlock == nil { + highestCommittedBlock, exists := bc.GetCommittedTip() + if !exists || highestCommittedBlock == nil { return nil, nil, errors.New("getStoredLineageFromCommittedTip: No committed blocks found") } currentHash := header.PrevBlockHash.NewBlockHash() @@ -1642,7 +1650,7 @@ func (bc *Blockchain) shouldReorg(blockNode *BlockNode, currentView uint64) bool // addTipBlockToBestChain adds the block as the new tip of the best chain. func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { - bc.bestChain.PushNewTip(blockNode) + bc.blockIndex.SetTip(blockNode) } // removeTipBlockFromBestChain removes the current tip from the best chain. It @@ -1651,9 +1659,9 @@ func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { // the bestChain slice and bestChainMap map. func (bc *Blockchain) removeTipBlockFromBestChain() *BlockNode { // Remove the last block from the best chain. - lastBlock := bc.bestChain.Chain[len(bc.bestChain.Chain)-1] - bc.bestChain.ChainMap.Remove(*lastBlock.Hash) - bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] + lastBlock := bc.blockIndex.GetTip() + // Uhhh what happens if we don't have the parent set up!? + bc.blockIndex.SetTip(lastBlock.GetParent(bc.blockIndex)) return lastBlock } @@ -1664,24 +1672,25 @@ func (bc *Blockchain) runCommitRuleOnBestChain(verifySignatures bool) error { currentBlock := bc.BlockTip() // If we can commit the grandparent, commit it. // Otherwise, we can't commit it and return nil. - blockToCommit, canCommit := bc.canCommitGrandparent(currentBlock) + blockNodeToCommit, canCommit := bc.canCommitGrandparent(currentBlock) if !canCommit { return nil } // Find all uncommitted ancestors of block to commit - _, idx := bc.GetCommittedTip() - if idx == -1 { + committedTip, exists := bc.GetCommittedTip() + if !exists { // This is an edge case we'll never hit in practice since all the PoW blocks // are committed. return errors.New("runCommitRuleOnBestChain: No committed blocks found") } uncommittedAncestors := []*BlockNode{} - for ii := idx + 1; ii < len(bc.bestChain.Chain); ii++ { - uncommittedAncestors = append(uncommittedAncestors, bc.bestChain.Chain[ii]) - if bc.bestChain.Chain[ii].Hash.IsEqual(blockToCommit) { - break - } + currentNode := &BlockNode{} + *currentNode = *blockNodeToCommit + for currentNode.Height > committedTip.Height { + uncommittedAncestors = append(uncommittedAncestors, currentNode) + currentNode = currentNode.GetParent(bc.blockIndex) } + uncommittedAncestors = collections.Reverse(uncommittedAncestors) for ii := 0; ii < len(uncommittedAncestors); ii++ { if err := bc.commitBlockPoS(uncommittedAncestors[ii].Hash, uint64(uncommittedAncestors[ii].Height), verifySignatures); err != nil { return errors.Wrapf(err, @@ -1696,16 +1705,18 @@ func (bc *Blockchain) runCommitRuleOnBestChain(verifySignatures bool) error { // between the grandparent and parent of the new block, meaning the grandparent and parent // are proposed in consecutive views, and the "parent" is an ancestor of the incoming block // (not necessarily consecutive views). Additionally, the grandparent must not already be committed. -func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparentBlockHash *BlockHash, _canCommit bool, +func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) ( + _grandparentBlockNode *BlockNode, + _canCommit bool, ) { // TODO: Is it sufficient that the current block's header points to the parent // or does it need to have something to do with the QC? - parent, exists := bc.bestChain.GetBlockByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + parent, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) if !exists { glog.Errorf("canCommitGrandparent: Parent block %v not found in best chain map", currentBlock.Header.PrevBlockHash.String()) return nil, false } - grandParent, exists := bc.bestChain.GetBlockByHashAndHeight(parent.Header.PrevBlockHash, uint64(parent.Height-1)) + grandParent, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(parent.Header.PrevBlockHash, uint64(parent.Height-1)) if !exists { glog.Errorf("canCommitGrandparent: Grandparent block %v not found in best chain map", parent.Header.PrevBlockHash.String()) return nil, false @@ -1715,7 +1726,7 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen } if grandParent.Header.ProposedInView+1 == parent.Header.ProposedInView { // Then we can run the commit rule up to the grandparent! - return grandParent.Hash, true + return grandParent, true } return nil, false } @@ -1725,7 +1736,7 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen // to the DB and updates relevant badger indexes with info about the block. func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, blockHeight uint64, verifySignatures bool) error { // block must be in the best chain. we grab the block node from there. - blockNode, exists := bc.bestChain.GetBlockByHashAndHeight(blockHash, blockHeight) + blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeight) if !exists { return errors.Errorf("commitBlockPoS: Block %v not found in best chain map", blockHash.String()) } @@ -1854,7 +1865,7 @@ func (bc *Blockchain) GetUncommittedBlocks(tipHash *BlockHash) ([]*BlockNode, er } bc.ChainLock.RLock() defer bc.ChainLock.RUnlock() - tipBlock, exists, err := bc.bestChain.GetBlockByHash(tipHash) + tipBlock, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(tipHash) if err != nil { return nil, errors.Wrapf(err, "GetUncommittedBlocks: Problem getting block %v", tipHash.String()) } @@ -1946,8 +1957,9 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: Block %v not found in block index", blockHash) } - highestCommittedBlock, _ := bc.GetCommittedTip() - if highestCommittedBlock == nil { + highestCommittedBlock, exists := bc.GetCommittedTip() + glog.Infof("Highest committed block: %v", highestCommittedBlock) + if !exists || highestCommittedBlock == nil { return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: No committed blocks found") } // If the provided block is committed, we need to make sure it's the committed tip. @@ -1972,10 +1984,6 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc return nil, errors.Errorf( "getUtxoViewAndUtxoOpsAtBlockHash: extends from a committed block that isn't the committed tip") } - if currentBlock.IsCommitted() && !currentBlock.Hash.IsEqual(highestCommittedBlock.Hash) { - return nil, errors.Errorf( - "getUtxoViewAndUtxoOpsAtBlockHash: extends from a committed block that isn't the committed tip") - } } viewAndUtxoOpsAtHash, err, exists := bc.getCachedBlockViewAndUtxoOps(blockHash) if err != nil { @@ -1994,6 +2002,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc var utxoOps [][]*UtxoOperation var fullBlock *MsgDeSoBlock for ii := len(uncommittedAncestors) - 1; ii >= 0; ii-- { + glog.Infof("Connecting block %v", uncommittedAncestors[ii]) // We need to get these blocks from badger fullBlock, err = GetBlock(uncommittedAncestors[ii].Hash, bc.db, bc.snapshot) if err != nil { @@ -2027,13 +2036,16 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc } // GetCommittedTip returns the highest committed block and its index in the best chain. -func (bc *Blockchain) GetCommittedTip() (*BlockNode, int) { - for ii := len(bc.bestChain.Chain) - 1; ii >= 0; ii-- { - if bc.bestChain.Chain[ii].IsCommitted() { - return bc.bestChain.Chain[ii], ii +func (bc *Blockchain) GetCommittedTip() (*BlockNode, bool) { + // iterate backwards from the tip node + currentNode := bc.blockIndex.GetTip() + for currentNode != nil { + if currentNode.IsCommitted() { + return currentNode, true } + currentNode = currentNode.GetParent(bc.blockIndex) } - return nil, -1 + return nil, false } // GetSafeBlocks returns all headers of blocks from which the chain can safely extend. @@ -2058,9 +2070,9 @@ func (bc *Blockchain) GetSafeBlocks() ([]*MsgDeSoHeader, error) { func (bc *Blockchain) getSafeBlockNodes() ([]*BlockNode, error) { // First get committed tip. - committedTip, idx := bc.GetCommittedTip() - if idx == -1 || committedTip == nil { - return nil, errors.New("getSafeBlockNodes: No committed blocks found") + committedTip, exists := bc.GetCommittedTip() + if !exists || committedTip == nil { + return []*BlockNode{}, nil } // Now get all blocks from the committed tip to the best chain tip. safeBlocks := []*BlockNode{committedTip} diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 47d1147c1..888ee1f9b 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -2,6 +2,7 @@ package lib import ( "bytes" + "encoding/hex" "fmt" "math" "math/rand" @@ -251,7 +252,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.bestChain.Chain = []*BlockNode{genesisBlock} + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{*genesisBlock.Hash: genesisBlock}) bc.blockIndex.blockIndexByHash.Add(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) @@ -301,7 +302,8 @@ func TestHasValidBlockHeight(t *testing.T) { require.Equal(t, err, RuleErrorInvalidPoSBlockHeight) block.Header.Height = 2 - bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot) + // TODO: make sure setting to genesis block works. + bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot, genesisBlock) err = bc.hasValidBlockHeightPoS(block.Header) require.Equal(t, err, RuleErrorMissingParentBlock) } @@ -471,10 +473,6 @@ func TestHasValidBlockViewPoS(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.bestChain.Chain = []*BlockNode{ - genesisNode, - block2, - } bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, @@ -807,7 +805,6 @@ func TestGetLineageFromCommittedTip(t *testing.T) { Height: 1, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.bestChain.Chain = []*BlockNode{genesisNode} bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) @@ -844,7 +841,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 2, PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.bestChain.Chain = append(bc.bestChain.Chain, block2) + bc.blockIndex.SetTip(block2) bc.blockIndex.blockIndexByHash.Add(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) @@ -1235,21 +1232,27 @@ func TestShouldReorg(t *testing.T) { hash1 := NewBlockHash(RandomBytes(32)) hash2 := NewBlockHash(RandomBytes(32)) hash3 := NewBlockHash(RandomBytes(32)) - bc.bestChain.Chain = []*BlockNode{ + chain := []*BlockNode{ { Hash: hash1, Status: StatusBlockStored | StatusBlockValidated | StatusBlockCommitted, + Height: 0, }, { Hash: hash3, Status: StatusBlockStored | StatusBlockValidated, + Height: 1, }, } + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + *hash1: chain[0], + *hash3: chain[1], + }) newBlock := &BlockNode{ Header: &MsgDeSoHeader{ ProposedInView: 2, - PrevBlockHash: bc.bestChain.Chain[1].Hash, + PrevBlockHash: chain[1].Hash, }, } @@ -1335,44 +1338,47 @@ func TestTryApplyNewTip(t *testing.T) { ancestors, _, err := bc.getStoredLineageFromCommittedTip(newBlock.Header) require.NoError(t, err) checkBestChainForHash := func(hash *BlockHash) bool { - return collections.Any(bc.bestChain.Chain, func(bn *BlockNode) bool { - return bn.Hash.IsEqual(hash) - }) + _, exists, err := bc.GetBlockFromBestChainByHash(hash, false) + require.NoError(t, err) + return exists } // Try to apply newBlock as tip. This should succeed. newBlockNode := &BlockNode{ Header: newBlock.Header, Hash: newBlockHash, + Height: 4, } appliedNewTip, connectedBlockHashes, disconnectedBlockHashes, err := bc.tryApplyNewTip(newBlockNode, 9, ancestors) require.NoError(t, err) require.True(t, appliedNewTip) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash3) + _, hash3ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash3, false) + require.NoError(t, err) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) require.Len(t, connectedBlockHashes, 1) require.Len(t, disconnectedBlockHashes, 1) // newBlock should be in the best chain and the best chain map and should be the tip. - _, newBlockExistsInBestChainMap := bc.bestChain.ChainMap.Get(*newBlockHash) + _, newBlockExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(newBlockHash, false) + require.NoError(t, err) require.True(t, newBlockExistsInBestChainMap) require.True(t, checkBestChainForHash(newBlockHash)) require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // Make sure block 2 and block 1 are still in the best chain. - _, hash2ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash2) + _, hash2ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash2, false) + require.NoError(t, err) require.True(t, hash2ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash2)) - _, hash1ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash1) + _, hash1ExistsInBestChainMap := bc.blockIndex.blockIndexByHash.Get(*hash1) require.True(t, hash1ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash1)) // Remove newBlock from the best chain and block index to reset the state. - bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] - bc.bestChain.ChainMap.Remove(*newBlockHash) + bc.blockIndex.SetTip(bc.blockIndex.GetTip().GetParent(bc.blockIndex)) // Add block 3 back bc.addTipBlockToBestChain(bn3) @@ -1409,6 +1415,8 @@ func TestTryApplyNewTip(t *testing.T) { newBlockNode.Header.ProposedInView = 7 newBlockNode.Header.Height = 7 newBlockNode.Height = 7 + // Clear parent out for safety. + newBlockNode.Parent = nil require.NoError(t, err) ancestors, _, err = bc.getStoredLineageFromCommittedTip(newBlock.Header) require.NoError(t, err) @@ -1420,19 +1428,23 @@ func TestTryApplyNewTip(t *testing.T) { // newBlockHash should be tip. require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap = bc.bestChain.ChainMap.Get(*hash3) + _, hash3ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHash(hash3, false) + require.NoError(t, err) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) // hash 2 should no longer be in the best chain or best chain map - _, hash2ExistsInBestChainMap = bc.bestChain.ChainMap.Get(*hash2) + _, hash2ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHash(hash2, false) + require.NoError(t, err) require.False(t, hash2ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash2)) // hash 4 should be in the best chain and the best chain map - _, hash4ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash4) + _, hash4ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash4, false) + require.NoError(t, err) require.True(t, hash4ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash4)) // hash 5 should be in the best chain and the best chain map - _, hash5ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash5) + _, hash5ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash5, false) + require.NoError(t, err) require.True(t, hash5ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash5)) @@ -1440,11 +1452,12 @@ func TestTryApplyNewTip(t *testing.T) { require.Len(t, connectedBlockHashes, 3) require.Len(t, disconnectedBlockHashes, 2) - // Reset the state of the best chain. - bc.bestChain.ChainMap.Remove(*hash4) - bc.bestChain.ChainMap.Remove(*hash5) - bc.bestChain.ChainMap.Remove(*newBlockHash) - bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] + // Reset the state of the best chain - parent of newBlock + //bc.bestChain.ChainMap.Remove(*hash4) + //bc.bestChain.ChainMap.Remove(*hash5) + //bc.bestChain.ChainMap.Remove(*newBlockHash) + //bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] + bc.blockIndex.SetTip(newBlockNode.GetParent(bc.blockIndex)) // Add block 2 and 3 back. bc.addTipBlockToBestChain(bn2) @@ -1514,8 +1527,8 @@ func TestCanCommitGrandparent(t *testing.T) { PrevBlockHash: hash1, }, } - bc.bestChain.ChainMap.Add(*hash1, bn1) - bc.bestChain.ChainMap.Add(*hash2, bn2) + bc.blockIndex.addNewBlockNodeToBlockIndex(bn1) + bc.blockIndex.addNewBlockNodeToBlockIndex(bn2) // define incoming block hash3 := NewBlockHash(RandomBytes(32)) @@ -1534,7 +1547,7 @@ func TestCanCommitGrandparent(t *testing.T) { // (meaning they are in consecutive views). So we should be able // to commit bn1. grandparentHash, canCommit := bc.canCommitGrandparent(bn3) - require.True(t, hash1.IsEqual(grandparentHash)) + require.True(t, hash1.IsEqual(grandparentHash.Hash)) require.True(t, canCommit) // Update bn1 to be committed. We no longer can run the commit since bn1 is already @@ -1609,7 +1622,7 @@ func TestRunCommitRuleOnBestChain(t *testing.T) { // Add one more block to the best chain, but have the view be further in the future. // this should trigger a commit on block 2. - blockTemplate4 := _generateBlockAndAddToBestChain(testMeta, 14, 20, 429) + blockTemplate4 := _generateBlockAndAddToBestChain(testMeta, 15, 20, 429) err = testMeta.chain.runCommitRuleOnBestChain(true) require.NoError(t, err) @@ -1621,7 +1634,7 @@ func TestRunCommitRuleOnBestChain(t *testing.T) { // Okay so add block 5 to the best chain. This should NOT trigger a commit on block 3 // as block 4 is not a direct child of block 3 based on its view. - blockTemplate5 := _generateBlockAndAddToBestChain(testMeta, 15, 21, 654) + blockTemplate5 := _generateBlockAndAddToBestChain(testMeta, 16, 21, 654) err = testMeta.chain.runCommitRuleOnBestChain(true) require.NoError(t, err) @@ -1632,9 +1645,9 @@ func TestRunCommitRuleOnBestChain(t *testing.T) { _verifyCommitRuleHelper(testMeta, []*BlockHash{blockHash1, blockHash2}, []*BlockHash{blockHash3, blockHash4, blockHash5}, blockHash2) // If we now add a block that is a descendent of block 5, we should be able to commit - // blocks 3 and 4 as block 4 and 5 possess a direct parent child relationship and + // blocks 3 and 4 as block 4 and 5 possess a direct parent child relationship, and // we have a descendent of block 5. - blockTemplate6 := _generateBlockAndAddToBestChain(testMeta, 16, 22, 912) + blockTemplate6 := _generateBlockAndAddToBestChain(testMeta, 17, 22, 912) require.NoError(t, err) err = testMeta.chain.runCommitRuleOnBestChain(true) require.NoError(t, err) @@ -1657,7 +1670,8 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, committedHash := range committedBlocks { // Okay so let's make sure the block is committed. - blockNode, exists := testMeta.chain.bestChain.ChainMap.Get(*committedHash) + blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHash(committedHash, false) + require.NoError(testMeta.t, err) require.True(testMeta.t, exists) require.True(testMeta.t, blockNode.IsCommitted()) @@ -1682,7 +1696,8 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, uncommittedBlockHash := range uncommittedBlocks { // Okay so let's make sure the block is uncommitted. - blockNode, exists := testMeta.chain.bestChain.ChainMap.Get(*uncommittedBlockHash) + blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHash(uncommittedBlockHash, false) + require.NoError(testMeta.t, err) require.True(testMeta.t, exists) require.False(testMeta.t, blockNode.IsCommitted()) // TODO: Verify DB results?? Kinda silly to make sure everything is missing. @@ -1874,7 +1889,8 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { // Timeout block will no longer be in best chain, and will still be in an uncommitted state in the block index _verifyCommitRuleHelper(testMeta, []*BlockHash{blockHash1, blockHash2}, []*BlockHash{blockHash3, reorgBlockHash}, blockHash2) _verifyRandomSeedHashHelper(testMeta, reorgBlock) - _, exists := testMeta.chain.bestChain.ChainMap.Get(*timeoutBlockHash) + _, exists, err := testMeta.chain.GetBlockFromBestChainByHash(timeoutBlockHash, false) + require.NoError(t, err) require.False(t, exists) timeoutBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(timeoutBlockHash, timeoutBlockHeight) @@ -1987,6 +2003,7 @@ func TestGetSafeBlocks(t *testing.T) { testMeta := NewTestPoSBlockchainWithValidators(t) committedHash := testMeta.chain.BlockTip().Hash var block1 *MsgDeSoBlock + fmt.Println("HEX STUFF: ", hex.EncodeToString(committedHash[:])) block1 = _generateRealBlock(testMeta, uint64(testMeta.savedHeight), uint64(testMeta.savedHeight), 1723, committedHash, false) block1Hash, err := block1.Hash() require.NoError(t, err) @@ -2542,7 +2559,10 @@ func _generateDummyBlock(testMeta *TestMeta, blockHeight uint64, view uint64, se // _generateBlockAndAddToBestChain generates a BlockTemplate by calling _generateRealBlock and then adds it to the // best chain. Finally it updates the PosMempool's latest block view. func _generateBlockAndAddToBestChain(testMeta *TestMeta, blockHeight uint64, view uint64, seed int64) *MsgDeSoBlock { - blockTemplate := _generateRealBlock(testMeta, blockHeight, view, seed, testMeta.chain.BlockTip().Hash, false) + prevBlockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHeight(blockHeight-1, false) + require.NoError(testMeta.t, err) + require.True(testMeta.t, exists) + blockTemplate := _generateRealBlock(testMeta, blockHeight, view, seed, prevBlockNode.Hash, false) var msgDesoBlock *MsgDeSoBlock msgDesoBlock = blockTemplate newBlockHash, err := msgDesoBlock.Hash() @@ -2824,6 +2844,7 @@ func NewTestPoSBlockchainWithValidators(t *testing.T) *TestMeta { for ii := 0; ii < 10; ii++ { _, err := miner.MineAndProcessSingleBlock(0 /*threadIndex*/, oldPool) require.NoError(t, err) + fmt.Println("CHAIN TIP: ", chain.BlockTip().Hash) } m0PubBytes, _, _ := Base58CheckDecode(m0Pub) diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index 74ab70dca..9d471b2a5 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -102,7 +102,7 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { currentView := blockHeader.ValidatorsVoteQC.GetView() + 1 nextView := currentView + 1 - blockIndex := NewBlockIndex(nil, nil) + blockIndex := NewBlockIndex(nil, nil, nil) blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *blockHash: {Header: blockHeader, Height: uint32(blockHeader.Height), Hash: blockHash}, }) diff --git a/lib/server.go b/lib/server.go index 71f003c8b..997f63785 100644 --- a/lib/server.go +++ b/lib/server.go @@ -885,58 +885,58 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { // Go through the block nodes in the blockchain and download the blocks if they're not stored. // TODO: need to figure out a way to get all the blocks in the best chain so we can download historical blocks. - for _, blockNode := range srv.blockchain.bestChain.Chain { - // We find the first block that's not stored and get ready to download blocks starting from this block onwards. - if blockNode.Status&StatusBlockStored == 0 { - maxBlocksInFlight := MaxBlocksInFlight - if pp.NegotiatedProtocolVersion >= ProtocolVersion2 && - (srv.params.IsPoSBlockHeight(uint64(blockNode.Height)) || - srv.params.NetworkType == NetworkType_TESTNET) { - - maxBlocksInFlight = MaxBlocksInFlightPoS - } - numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) - currentHeight := int(blockNode.Height) - blockNodesToFetch := []*BlockNode{} - // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. - var heightLimit int - for heightLimit = len(srv.blockchain.bestChain.Chain) - 1; heightLimit >= 0; heightLimit-- { - if !srv.blockchain.bestChain.Chain[heightLimit].Status.IsFullyProcessed() { - break - } - } - - // Find the blocks that we should download. - for currentHeight <= heightLimit && - len(blockNodesToFetch) < numBlocksToFetch { - - // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to - // the array index. - currentNode := srv.blockchain.bestChain.Chain[currentHeight] - currentHeight++ - - // If we've already requested this block then we don't request it again. - if _, exists := pp.requestedBlocks[*currentNode.Hash]; exists { - continue - } - - blockNodesToFetch = append(blockNodesToFetch, currentNode) - } - - var hashList []*BlockHash - for _, node := range blockNodesToFetch { - hashList = append(hashList, node.Hash) - pp.requestedBlocks[*node.Hash] = true - } - pp.AddDeSoMessage(&MsgDeSoGetBlocks{ - HashList: hashList, - }, false) - - glog.V(1).Infof("GetBlocksToStore: Downloading blocks to store for header %v from peer %v", - blockNode.Header, pp) - return - } - } + //for _, blockNode := range srv.blockchain.bestChain.Chain { + // // We find the first block that's not stored and get ready to download blocks starting from this block onwards. + // if blockNode.Status&StatusBlockStored == 0 { + // maxBlocksInFlight := MaxBlocksInFlight + // if pp.NegotiatedProtocolVersion >= ProtocolVersion2 && + // (srv.params.IsPoSBlockHeight(uint64(blockNode.Height)) || + // srv.params.NetworkType == NetworkType_TESTNET) { + // + // maxBlocksInFlight = MaxBlocksInFlightPoS + // } + // numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) + // currentHeight := int(blockNode.Height) + // blockNodesToFetch := []*BlockNode{} + // // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. + // var heightLimit int + // for heightLimit = len(srv.blockchain.bestChain.Chain) - 1; heightLimit >= 0; heightLimit-- { + // if !srv.blockchain.bestChain.Chain[heightLimit].Status.IsFullyProcessed() { + // break + // } + // } + // + // // Find the blocks that we should download. + // for currentHeight <= heightLimit && + // len(blockNodesToFetch) < numBlocksToFetch { + // + // // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to + // // the array index. + // currentNode := srv.blockchain.bestChain.Chain[currentHeight] + // currentHeight++ + // + // // If we've already requested this block then we don't request it again. + // if _, exists := pp.requestedBlocks[*currentNode.Hash]; exists { + // continue + // } + // + // blockNodesToFetch = append(blockNodesToFetch, currentNode) + // } + // + // var hashList []*BlockHash + // for _, node := range blockNodesToFetch { + // hashList = append(hashList, node.Hash) + // pp.requestedBlocks[*node.Hash] = true + // } + // pp.AddDeSoMessage(&MsgDeSoGetBlocks{ + // HashList: hashList, + // }, false) + // + // glog.V(1).Infof("GetBlocksToStore: Downloading blocks to store for header %v from peer %v", + // blockNode.Header, pp) + // return + // } + //} // If we get here then it means that we've downloaded all blocks so we can update srv.blockchain.downloadingHistoricalBlocks = false @@ -1038,12 +1038,16 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b } var hasSeenCheckpointBlockHash bool var checkpointBlockNode *BlockNode + var err error if isHeaderChain { - checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestHeaderChain.GetBlockByHashAndHeight( - checkpointBlockInfo.Hash, checkpointBlockInfo.Height) + checkpointBlockNode, hasSeenCheckpointBlockHash, err = srv.blockchain.GetBlockFromBestChainByHash( + checkpointBlockInfo.Hash, true) } else { - checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestChain.GetBlockByHashAndHeight( - checkpointBlockInfo.Hash, checkpointBlockInfo.Height) + checkpointBlockNode, hasSeenCheckpointBlockHash, err = srv.blockchain.GetBlockFromBestChainByHash( + checkpointBlockInfo.Hash, false) + } + if err != nil { + glog.Fatalf("shouldVerifySignatures: Problem getting checkpoint block node from best chain: %v", err) } // If we haven't seen the checkpoint block hash yet, we skip signature verification. if !hasSeenCheckpointBlockHash { @@ -1071,13 +1075,11 @@ func (srv *Server) getCheckpointSyncingStatus(isHeaders bool) string { if checkpointBlockInfo == nil { return "" } - hasSeenCheckPointBlockHash := false - if isHeaders { - _, hasSeenCheckPointBlockHash = srv.blockchain.bestHeaderChain.GetBlockByHashAndHeight( - checkpointBlockInfo.Hash, checkpointBlockInfo.Height) - } else { - _, hasSeenCheckPointBlockHash = srv.blockchain.bestChain.GetBlockByHashAndHeight( - checkpointBlockInfo.Hash, checkpointBlockInfo.Height) + _, hasSeenCheckPointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHash( + checkpointBlockInfo.Hash, isHeaders) + + if err != nil { + glog.Fatalf("getCheckpointSyncingStatus: Problem getting checkpoint block node from best chain: %v", err) } if !hasSeenCheckPointBlockHash { return fmt.Sprintf("", checkpointBlockInfo.String()) @@ -1258,7 +1260,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // snapshot we receive from the peer is up-to-date. // TODO: error handle if the hash doesn't exist for some reason. expectedSnapshotHeightBlock, expectedSnapshotHeightblockExists, err := - srv.blockchain.bestHeaderChain.GetBlockByHeight(expectedSnapshotHeight) + srv.blockchain.GetBlockFromBestChainByHeight(expectedSnapshotHeight, true) if err != nil { glog.Errorf("Server._handleHeaderBundle: Problem getting expected snapshot height block, error (%v)", err) return @@ -1404,8 +1406,9 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { glog.V(1).Infof("Server._handleHeaderBundle: *Syncing* headers for blocks starting at "+ "header tip %v out of %d from peer %v", headerTip.Header, msg.TipHeight, pp) - glog.V(0).Infof("Server._handleHeaderBundle: Num Headers in header chain: (chain map: %v) - (chain: %v) ", - srv.blockchain.bestHeaderChain.ChainMap.Len(), len(srv.blockchain.bestHeaderChain.Chain)) + // TODO: this may be wrong? + glog.V(0).Infof("Server._handleHeaderBundle: Num Headers in header chain: (header tip height: %v) ", + srv.blockchain.blockIndex.GetHeaderTip()) } func (srv *Server) _handleGetBlocks(pp *Peer, msg *MsgDeSoGetBlocks) { @@ -1677,7 +1680,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { srv.snapshot.PrintChecksum("Finished hyper sync. Checksum is:") glog.Infof(CLog(Magenta, fmt.Sprintf("Metadata checksum: (%v)", srv.HyperSyncProgress.SnapshotMetadata.CurrentEpochChecksumBytes))) - blockNode, exists, err := srv.blockchain.bestHeaderChain.GetBlockByHeight(msg.SnapshotMetadata.SnapshotBlockHeight) + blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(msg.SnapshotMetadata.SnapshotBlockHeight, true) if err != nil { glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error (%v)", err) return @@ -1687,7 +1690,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { //return } else { glog.Infof(CLog(Yellow, fmt.Sprintf("Best header chain %v best block chain %v", - blockNode, srv.blockchain.bestChain.Chain))) + blockNode, srv.blockchain.blockIndex.GetTip()))) } // Verify that the state checksum matches the one in HyperSyncProgress snapshot metadata. // If the checksums don't match, it means that we've been interacting with a peer that was misbehaving. @@ -1730,7 +1733,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { // being too large and possibly causing an error in badger. var blockNodeBatch []*BlockNode for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { - currentNode, currentNodeExists, err := srv.blockchain.bestHeaderChain.GetBlockByHeight(ii) + currentNode, currentNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(ii, true) if err != nil { glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) break @@ -1744,7 +1747,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { currentNode.Status |= StatusBlockValidated currentNode.Status |= StatusBlockCommitted srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) - srv.blockchain.bestChain.PushNewTip(currentNode) + srv.blockchain.blockIndex.SetTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) if len(blockNodeBatch) < 10000 { continue diff --git a/lib/state_change_syncer.go b/lib/state_change_syncer.go index 0172e30f9..47832000a 100644 --- a/lib/state_change_syncer.go +++ b/lib/state_change_syncer.go @@ -777,7 +777,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser return true, nil } - blockHeight := uint64(server.blockchain.bestChain.GetTip().Height) + blockHeight := uint64(server.blockchain.blockIndex.GetTip().Height) stateChangeSyncer.MempoolFlushId = originalCommittedFlushId @@ -804,7 +804,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser mempoolUtxoView.Snapshot = nil server.blockchain.ChainLock.RLock() - mempoolUtxoView.TipHash = server.blockchain.bestChain.GetTip().Hash + mempoolUtxoView.TipHash = server.blockchain.blockIndex.GetTip().Hash server.blockchain.ChainLock.RUnlock() // A new transaction is created so that we can simulate writes to the db without actually writing to the db. @@ -815,7 +815,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser defer txn.Discard() glog.V(2).Infof("Time since mempool sync start: %v", time.Since(startTime)) startTime = time.Now() - err = mempoolUtxoView.FlushToDbWithTxn(txn, uint64(server.blockchain.bestChain.GetTip().Height)) + err = mempoolUtxoView.FlushToDbWithTxn(txn, uint64(server.blockchain.blockIndex.GetTip().Height)) if err != nil { mempoolUtxoView.EventManager.stateSyncerFlushed(&StateSyncerFlushedEvent{ FlushId: originalCommittedFlushId, diff --git a/lib/txindex.go b/lib/txindex.go index 904211819..d3794d935 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -150,8 +150,8 @@ func NewTXIndex(coreChain *Blockchain, params *DeSoParams, dataDirectory string) } func (txi *TXIndex) FinishedSyncing() bool { - committedTip, idx := txi.CoreChain.GetCommittedTip() - if idx == -1 { + committedTip, exists := txi.CoreChain.GetCommittedTip() + if !exists { return false } return txi.TXIndexChain.BlockTip().Height == committedTip.Height @@ -208,8 +208,7 @@ func (txi *TXIndex) Stop() { // GetTxindexUpdateBlockNodes ... func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( - _txindexTipNode *BlockNode, _blockTipNode *BlockNode, _commonAncestor *BlockNode, - _detachBlocks []*BlockNode, _attachBlocks []*BlockNode) { + _txindexTipNode *BlockNode, _blockTipNode *BlockNode, _commonAncestor *BlockNode) { // Get the current txindex tip. txindexTipHash := txi.TXIndexChain.BlockTip() @@ -219,7 +218,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // case. glog.Error("Error: TXIndexChain had nil tip; this should never " + "happen and it means the transaction index is broken.") - return nil, nil, nil, nil, nil + return nil, nil, nil } // If the tip of the txindex is no longer stored in the block index, it // means the txindex hit a fork that we are no longer keeping track of. @@ -230,22 +229,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // Get the committed tip. committedTip, _ := txi.CoreChain.GetCommittedTip() - if txindexTipNode == nil { - glog.Info("GetTxindexUpdateBlockNodes: Txindex tip was not found; building txindex starting at genesis block") - - newTxIndexBestChain, _ := txi.TXIndexChain.CopyBestChain() - newBlockchainBestChain, _ := txi.CoreChain.CopyBestChain() - - return txindexTipNode, committedTip, nil, newTxIndexBestChain, newBlockchainBestChain - } - - derefedTxindexTipNode := *txindexTipNode - - // At this point, we know our txindex tip is in our block index so - // there must be a common ancestor between the tip and the block tip. - commonAncestor, detachBlocks, attachBlocks := txi.CoreChain.GetReorgBlocks(&derefedTxindexTipNode, committedTip) - - return txindexTipNode, committedTip, commonAncestor, detachBlocks, attachBlocks + return txindexTipNode, committedTip, txindexTipNode } // Update syncs the transaction index with the blockchain. @@ -265,7 +249,7 @@ func (txi *TXIndex) Update() error { // done with the rest of the function. txi.TXIndexLock.Lock() defer txi.TXIndexLock.Unlock() - txindexTipNode, blockTipNode, commonAncestor, detachBlocks, attachBlocks := txi.GetTxindexUpdateBlockNodes() + txindexTipNode, blockTipNode, commonAncestor := txi.GetTxindexUpdateBlockNodes() // Note that the blockchain's ChainLock does not need to be held at this // point because we're just reading blocks from the db, which never get @@ -294,97 +278,100 @@ func (txi *TXIndex) Update() error { // For each of the blocks we're removing, delete the transactions from // the transaction index. - for _, blockToDetach := range detachBlocks { - if txi.killed { - glog.Infof(CLog(Yellow, "TxIndex: Update: Killed while detaching blocks")) - break - } - // Go through each txn in the block and delete its mappings from our - // txindex. - glog.V(1).Infof("Update: Detaching block (height: %d, hash: %v)", - blockToDetach.Height, blockToDetach.Hash) - blockMsg, err := GetBlock(blockToDetach.Hash, txi.TXIndexChain.DB(), nil) - if err != nil { - return fmt.Errorf("Update: Problem fetching detach block "+ - "with hash %v: %v", blockToDetach.Hash, err) - } - blockHeight := uint64(txi.CoreChain.blockTip().Height) - err = txi.TXIndexChain.DB().Update(func(dbTxn *badger.Txn) error { - for _, txn := range blockMsg.Txns { - if err := DbDeleteTxindexTransactionMappingsWithTxn(dbTxn, nil, - blockHeight, txn, txi.Params, txi.CoreChain.eventManager, true); err != nil { - - return fmt.Errorf("Update: Problem deleting "+ - "transaction mappings for transaction %v: %v", txn.Hash(), err) - } - } - return nil - }) - if err != nil { - return err - } - - // Now that all the transactions have been deleted from our txindex, - // it's safe to disconnect the block from our txindex chain. - utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) - utxoOps, err := GetUtxoOperationsForBlock( - txi.TXIndexChain.DB(), nil, blockToDetach.Hash) - if err != nil { - return fmt.Errorf( - "Update: Error getting UtxoOps for block %v: %v", blockToDetach, err) - } - // Compute the hashes for all the transactions. - txHashes, err := ComputeTransactionHashes(blockMsg.Txns) - if err != nil { - return fmt.Errorf( - "Update: Error computing tx hashes for block %v: %v", - blockToDetach, err) - } - if err := utxoView.DisconnectBlock(blockMsg, txHashes, utxoOps, blockHeight); err != nil { - return fmt.Errorf("Update: Error detaching block "+ - "%v from UtxoView: %v", blockToDetach, err) - } - if err := utxoView.FlushToDb(blockHeight); err != nil { - return fmt.Errorf("Update: Error flushing view to db for block "+ - "%v: %v", blockToDetach, err) - } - // We have to flush a couple of extra things that the view doesn't flush... - if err := PutBestHash(txi.TXIndexChain.DB(), nil, utxoView.TipHash, ChainTypeDeSoBlock, txi.CoreChain.eventManager); err != nil { - return fmt.Errorf("Update: Error putting best hash for block "+ - "%v: %v", blockToDetach, err) - } - err = txi.TXIndexChain.DB().Update(func(txn *badger.Txn) error { - if err := DeleteUtxoOperationsForBlockWithTxn(txn, nil, blockToDetach.Hash, txi.TXIndexChain.eventManager, true); err != nil { - return fmt.Errorf("Update: Error deleting UtxoOperations 1 for block %v, %v", blockToDetach.Hash, err) - } - if err := txn.Delete(BlockHashToBlockKey(blockToDetach.Hash)); err != nil { - return fmt.Errorf("Update: Error deleting UtxoOperations 2 for block %v %v", blockToDetach.Hash, err) - } - return nil - }) - - if err != nil { - return fmt.Errorf("Update: Error updating badgger: %v", err) - } - // Delete this block from the chain db so we don't get duplicate block errors. - - // Remove this block from our bestChain data structures. - newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() - newBestChain, newBestChainMap := txi.TXIndexChain.CopyBestChain() - newBestChain = newBestChain[:len(newBestChain)-1] - newBestChainMap.Remove(*blockToDetach.Hash) - newBlockIndex.Remove(*blockToDetach.Hash) - - txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndex) - - // At this point the entries for the block should have been removed - // from both our Txindex chain and our transaction index mappings. - } + // TODO: delete - we're simplifying the txindex logic to only use committed state. + //for _, blockToDetach := range detachBlocks { + // if txi.killed { + // glog.Infof(CLog(Yellow, "TxIndex: Update: Killed while detaching blocks")) + // break + // } + // // Go through each txn in the block and delete its mappings from our + // // txindex. + // glog.V(1).Infof("Update: Detaching block (height: %d, hash: %v)", + // blockToDetach.Height, blockToDetach.Hash) + // blockMsg, err := GetBlock(blockToDetach.Hash, txi.TXIndexChain.DB(), nil) + // if err != nil { + // return fmt.Errorf("Update: Problem fetching detach block "+ + // "with hash %v: %v", blockToDetach.Hash, err) + // } + // blockHeight := uint64(txi.CoreChain.blockTip().Height) + // err = txi.TXIndexChain.DB().Update(func(dbTxn *badger.Txn) error { + // for _, txn := range blockMsg.Txns { + // if err := DbDeleteTxindexTransactionMappingsWithTxn(dbTxn, nil, + // blockHeight, txn, txi.Params, txi.CoreChain.eventManager, true); err != nil { + // + // return fmt.Errorf("Update: Problem deleting "+ + // "transaction mappings for transaction %v: %v", txn.Hash(), err) + // } + // } + // return nil + // }) + // if err != nil { + // return err + // } + // + // // Now that all the transactions have been deleted from our txindex, + // // it's safe to disconnect the block from our txindex chain. + // utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) + // utxoOps, err := GetUtxoOperationsForBlock( + // txi.TXIndexChain.DB(), nil, blockToDetach.Hash) + // if err != nil { + // return fmt.Errorf( + // "Update: Error getting UtxoOps for block %v: %v", blockToDetach, err) + // } + // // Compute the hashes for all the transactions. + // txHashes, err := ComputeTransactionHashes(blockMsg.Txns) + // if err != nil { + // return fmt.Errorf( + // "Update: Error computing tx hashes for block %v: %v", + // blockToDetach, err) + // } + // if err := utxoView.DisconnectBlock(blockMsg, txHashes, utxoOps, blockHeight); err != nil { + // return fmt.Errorf("Update: Error detaching block "+ + // "%v from UtxoView: %v", blockToDetach, err) + // } + // if err := utxoView.FlushToDb(blockHeight); err != nil { + // return fmt.Errorf("Update: Error flushing view to db for block "+ + // "%v: %v", blockToDetach, err) + // } + // // We have to flush a couple of extra things that the view doesn't flush... + // if err := PutBestHash(txi.TXIndexChain.DB(), nil, utxoView.TipHash, ChainTypeDeSoBlock, txi.CoreChain.eventManager); err != nil { + // return fmt.Errorf("Update: Error putting best hash for block "+ + // "%v: %v", blockToDetach, err) + // } + // err = txi.TXIndexChain.DB().Update(func(txn *badger.Txn) error { + // if err := DeleteUtxoOperationsForBlockWithTxn(txn, nil, blockToDetach.Hash, txi.TXIndexChain.eventManager, true); err != nil { + // return fmt.Errorf("Update: Error deleting UtxoOperations 1 for block %v, %v", blockToDetach.Hash, err) + // } + // if err := txn.Delete(BlockHashToBlockKey(blockToDetach.Hash)); err != nil { + // return fmt.Errorf("Update: Error deleting UtxoOperations 2 for block %v %v", blockToDetach.Hash, err) + // } + // return nil + // }) + // + // if err != nil { + // return fmt.Errorf("Update: Error updating badgger: %v", err) + // } + // // Delete this block from the chain db so we don't get duplicate block errors. + // + // // Remove this block from our bestChain data structures. + // newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() + // newTip := blockToDetach.GetParent(txi.TXIndexChain.blockIndex) + // if newTip == nil { + // return fmt.Errorf("Update: Error getting parent of block %v", blockToDetach) + // } + // + // txi.TXIndexChain.SetBestChainMap(newBlockIndex, newTip) + // + // // At this point the entries for the block should have been removed + // // from both our Txindex chain and our transaction index mappings. + //} // For each of the blocks we're adding, process them on our txindex chain // and add their mappings to our txn index. Compute any metadata that might // be useful. - for _, blockToAttach := range attachBlocks { + blockToAttach := &BlockNode{} + *blockToAttach = *txindexTipNode + for !blockToAttach.Hash.IsEqual(blockTipNode.Hash) { if txi.killed { glog.Infof(CLog(Yellow, "TxIndex: Update: Killed while attaching blocks")) break @@ -454,6 +441,11 @@ func (txi *TXIndex) Update() error { return fmt.Errorf("Update: Problem attaching block %v: %v", blockToAttach, err) } + var exists bool + blockToAttach, exists, err = txi.CoreChain.GetBlockFromBestChainByHeight(uint64(blockToAttach.Height+1), false) + if !exists || err != nil { + return fmt.Errorf("Update: Problem getting block at height %d: %v", blockToAttach.Height+1, err) + } } glog.Infof("Update: Txindex update complete. New tip: (height: %d, hash: %v)", From c6ad24a409ff2829f997cd41ff9156ea21599c6d Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 3 Oct 2024 00:34:53 -0400 Subject: [PATCH 18/62] move block index migration to NewBlockchain instead of NewServer, add setting committed status for PoW nodes, remove timeout adjustment in test.Dockerfile --- lib/blockchain.go | 3 +++ lib/db_utils.go | 46 ++++++++++++++++++++++++++++++++++++++++++++-- lib/server.go | 11 +++-------- test.Dockerfile | 2 +- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 12ab58072..0620b545b 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1306,6 +1306,9 @@ func NewBlockchain( archivalMode bool, checkpointSyncingProviders []string, ) (*Blockchain, error) { + if err := RunBlockIndexMigrationOnce(db, params); err != nil { + return nil, errors.Wrapf(err, "NewBlockchain: Problem running block index migration") + } trustedBlockProducerPublicKeys := make(map[PkMapKey]bool) for _, keyStr := range trustedBlockProducerPublicKeyStrs { diff --git a/lib/db_utils.go b/lib/db_utils.go index de0de1bf8..f6fe875fe 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5659,8 +5659,8 @@ func (bi *BlockIndex) LoadBlockIndexFromHeight(height uint32, params *DeSoParams }) } -func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager *EventManager) error { - return handle.View(func(txn *badger.Txn) error { +func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager *EventManager, params *DeSoParams) error { + return handle.Update(func(txn *badger.Txn) error { prefix := _heightHashToNodeIndexPrefix(false) opts := badger.DefaultIteratorOptions opts.Prefix = prefix @@ -5692,6 +5692,48 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") } } + // TODO: get best chain up to PoS Cutover height and set all blocks in that chain to committed. + firstPoSBlockHeight := params.GetFirstPoSBlockHeight() + // Look up blocks at cutover height. + prefixKey := _heightHashToNodePrefixByHeight(uint32(firstPoSBlockHeight), false) + _, valsFound, err := _enumerateKeysForPrefixWithTxn(txn, prefixKey, false) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem enumerating keys for prefix") + } + if len(valsFound) == 0 { + return fmt.Errorf("RunBlockIndexMigration: No blocks found at PoS cutover height") + } + if len(valsFound) > 1 { + return fmt.Errorf("RunBlockIndexMigration: More than one block found at PoS cutover height") + } + blockNode, err := DeserializeBlockNode(valsFound[0]) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem deserializing block node for pos cutover") + } + var blockNodeBatch []*BlockNode + for blockNode != nil { + if !blockNode.IsCommitted() { + blockNode.Status |= StatusBlockCommitted + } + // TODO: make sure I don't need a copy. + blockNodeBatch = append(blockNodeBatch, blockNode) + if len(blockNodeBatch) < 10000 { + continue + } + err = PutHeightHashToNodeInfoBatch(handle, snapshot, blockNodeBatch, false /*bitcoinNodes*/, eventManager) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem putting block node batch") + } + parentBlockNode := GetHeightHashToNodeInfoWithTxn(txn, snapshot, blockNode.Height, blockNode.Hash, false /*bitcoinNodes*/) + if blockNode.Height > 0 && parentBlockNode == nil { + return errors.New("RunBlockIndexMigration: Parent block node not found") + } + blockNode = parentBlockNode + } + err = PutHeightHashToNodeInfoBatch(handle, snapshot, blockNodeBatch, false /*bitcoinNodes*/, eventManager) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem putting block node batch") + } return nil }) } diff --git a/lib/server.go b/lib/server.go index 997f63785..57a7bc625 100644 --- a/lib/server.go +++ b/lib/server.go @@ -364,8 +364,8 @@ func ValidateHyperSyncFlags(isHypersync bool, syncType NodeSyncType) { } } -func RunBlockIndexMigrationOnce(db *badger.DB, dataDir string) error { - blockIndexMigrationFileName := filepath.Join(dataDir, BlockIndexMigrationFileName) +func RunBlockIndexMigrationOnce(db *badger.DB, params *DeSoParams) error { + blockIndexMigrationFileName := filepath.Join(db.Opts().Dir, BlockIndexMigrationFileName) glog.V(0).Info("FileName: ", blockIndexMigrationFileName) hasRunMigration, err := ReadBoolFromFile(blockIndexMigrationFileName) if err == nil && hasRunMigration { @@ -373,7 +373,7 @@ func RunBlockIndexMigrationOnce(db *badger.DB, dataDir string) error { return nil } glog.V(0).Info("Running block index migration") - if err = RunBlockIndexMigration(db, nil, nil); err != nil { + if err = RunBlockIndexMigration(db, nil, nil, params); err != nil { return errors.Wrapf(err, "Problem running block index migration") } if err = SaveBoolToFile(blockIndexMigrationFileName, true); err != nil { @@ -457,11 +457,6 @@ func NewServer( _err error, _shouldRestart bool, ) { - - if err := RunBlockIndexMigrationOnce(_db, _dataDir); err != nil { - return nil, errors.Wrapf(err, "NewServer: Problem running block index migration"), true - } - var err error // Only initialize state change syncer if the directories are defined. diff --git a/test.Dockerfile b/test.Dockerfile index 34182c4c7..3eb8c7501 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -28,4 +28,4 @@ COPY main.go . # build backend RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/core main.go -ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "15m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] +ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] From 216fd31274cda0ee8aa6a3c40ab0ce89f387c8c2 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 3 Oct 2024 00:39:07 -0400 Subject: [PATCH 19/62] merge main into branch --- cmd/node.go | 2 +- go.mod | 69 +++++++++------- go.sum | 199 +++++++++++++++++++++++++++++---------------- lib/pos_mempool.go | 10 +++ lib/server.go | 2 +- 5 files changed, 181 insertions(+), 101 deletions(-) diff --git a/cmd/node.go b/cmd/node.go index f494a79f7..1041da734 100644 --- a/cmd/node.go +++ b/cmd/node.go @@ -11,7 +11,7 @@ import ( "syscall" "time" - "github.com/DataDog/datadog-go/statsd" + "github.com/DataDog/datadog-go/v5/statsd" "github.com/btcsuite/btcd/addrmgr" "github.com/btcsuite/btcd/wire" "github.com/davecgh/go-spew/spew" diff --git a/go.mod b/go.mod index ddf84244b..02c87cd5f 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/deso-protocol/core go 1.20 require ( - github.com/DataDog/datadog-go v4.5.0+incompatible + github.com/DataDog/datadog-go/v5 v5.5.0 github.com/brianvoe/gofakeit v3.18.0+incompatible github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 github.com/bxcodec/faker v2.0.1+incompatible github.com/cloudflare/circl v1.1.0 - github.com/davecgh/go-spew v1.1.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 github.com/decred/dcrd/lru v1.1.3 github.com/deso-protocol/go-deadlock v1.0.0 @@ -17,29 +17,29 @@ require ( github.com/dgraph-io/badger/v3 v3.2103.5 github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.9.25 - github.com/fatih/color v1.13.0 + github.com/fatih/color v1.16.0 github.com/gernest/mention v2.0.0+incompatible - github.com/go-pg/pg/v10 v10.10.0 + github.com/go-pg/pg/v10 v10.11.1 github.com/golang/glog v1.0.0 - github.com/google/uuid v1.2.0 + github.com/google/uuid v1.5.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/holiman/uint256 v1.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/oleiade/lane v1.0.1 github.com/onflow/crypto v0.25.0 github.com/pkg/errors v0.9.1 - github.com/pmezard/go-difflib v1.0.0 + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/robinjoseph08/go-pg-migrations/v3 v3.0.0 github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.0.2 github.com/unrolled/secure v1.0.8 golang.org/x/crypto v0.27.0 golang.org/x/sync v0.8.0 - gopkg.in/DataDog/dd-trace-go.v1 v1.29.0 + gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 ) require ( @@ -49,7 +49,14 @@ require ( ) require ( - github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/DataDog/appsec-internal-go v1.7.0 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 // indirect + github.com/DataDog/go-libddwaf/v3 v3.2.1 // indirect + github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect + github.com/DataDog/gostackparse v0.7.0 // indirect + github.com/DataDog/sketches-go v1.4.5 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect github.com/bwesterb/go-ristretto v1.2.0 // indirect @@ -58,62 +65,66 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.6.0-alpha.5 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/git-chglog/git-chglog v0.0.0-20200414013904-db796966b373 // indirect github.com/go-pg/zerochecker v0.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.0+incompatible // indirect - github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.8 // indirect + github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect + github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.12.3 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/klauspost/compress v1.17.1 // indirect github.com/magiconair/properties v1.8.1 // indirect - github.com/mattn/go-colorable v0.1.9 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/goveralls v0.0.6 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect - github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/onsi/ginkgo v1.15.0 // indirect github.com/onsi/gomega v1.10.5 // indirect + github.com/outcaste-io/ristretto v0.2.3 // indirect github.com/pelletier/go-toml v1.7.0 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect - github.com/philhofer/fwd v1.1.1 // indirect + github.com/philhofer/fwd v1.1.2 // indirect + github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.1.2 // indirect github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/tinylib/msgp v1.1.2 // indirect + github.com/tinylib/msgp v1.1.8 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df // indirect github.com/urfave/cli v1.22.1 // indirect github.com/vmihailenco/bufpool v0.1.11 // indirect - github.com/vmihailenco/msgpack/v5 v5.3.1 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - go.opencensus.io v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.uber.org/atomic v1.11.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect - golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.25.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gonum.org/v1/gonum v0.6.1 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/AlecAivazis/survey.v1 v1.8.7 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.51.0 // indirect gopkg.in/kyokomi/emoji.v1 v1.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - mellium.im/sasl v0.2.1 // indirect + mellium.im/sasl v0.3.1 // indirect ) diff --git a/go.sum b/go.sum index c159e2333..5536e1ec2 100644 --- a/go.sum +++ b/go.sum @@ -27,11 +27,25 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/datadog-go v4.5.0+incompatible h1:MyyuIz5LVAI3Im+0F/tfo64ETyH4sNVynZ29yOiHm50= -github.com/DataDog/datadog-go v4.5.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/DataDog/appsec-internal-go v1.7.0 h1:iKRNLih83dJeVya3IoUfK+6HLD/hQsIbyBlfvLmAeb0= +github.com/DataDog/appsec-internal-go v1.7.0/go.mod h1:wW0cRfWBo4C044jHGwYiyh5moQV2x0AhnwqMuiX7O/g= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ= +github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= +github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= +github.com/DataDog/go-libddwaf/v3 v3.2.1 h1:lZPc6UxCOwioHc++nsldKR50FpIrRh1uGnGLuryqnE8= +github.com/DataDog/go-libddwaf/v3 v3.2.1/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY= +github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I= +github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= +github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= +github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= +github.com/DataDog/sketches-go v1.4.5 h1:ki7VfeNz7IcNafq7yI/j5U/YCkO3LJiMDtXz9OMQbyE= +github.com/DataDog/sketches-go v1.4.5/go.mod h1:7Y8GN8Jf66DLyDhc94zuWA3uHEt/7ttt8jHOBWWrSOg= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -86,9 +100,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= @@ -106,11 +117,11 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= @@ -134,9 +145,14 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 h1:8EXxF+tCLqaVk8AOC29zl2mnhQjwyLxxOTuhUazWRsg= +github.com/ebitengine/purego v0.6.0-alpha.5 h1:EYID3JOAdmQ4SNZYJHu9V6IqOeRQDBYxqKAg9PyoHFY= +github.com/ebitengine/purego v0.6.0-alpha.5/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= @@ -148,8 +164,8 @@ github.com/ethereum/go-ethereum v1.9.25 h1:mMiw/zOOtCLdGLWfcekua0qPrJTe7FVIiHJ4I github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fergusstrange/embedded-postgres v1.19.0 h1:NqDufJHeA03U7biULlPHZ0pZ10/mDOMKPILEpT50Fyk= github.com/fergusstrange/embedded-postgres v1.19.0/go.mod h1:0B+3bPsMvcNgR9nN+bdM2x9YaNYDnf3ksUqYp1OAub0= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -169,8 +185,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-pg/pg/v10 v10.5.0/go.mod h1:BfgPoQnD2wXNd986RYEHzikqv9iE875PrFaZ9vXvtNM= -github.com/go-pg/pg/v10 v10.10.0 h1:xc5zWYQ/55XI8pk5NkK+ixXqbJh1vnOun3VODPmbYfY= -github.com/go-pg/pg/v10 v10.10.0/go.mod h1:EmoJGYErc+stNN/1Jf+o4csXuprjxcRztBnn6cHe38E= +github.com/go-pg/pg/v10 v10.11.1 h1:vYwbFpqoMpTDphnzIPshPPepdy3VpzD8qo29OFKp4vo= +github.com/go-pg/pg/v10 v10.11.1/go.mod h1:ExJWndhDNNftBdw1Ow83xqpSf4WMSJK8urmXD5VXS1I= github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= @@ -191,6 +207,7 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -204,8 +221,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -226,17 +243,18 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 h1:zIaiqGYDQwa4HVx5wGRTXbx38Pqxjemn4BP98wpzpXo= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -255,7 +273,10 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -265,8 +286,9 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= +github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= @@ -278,9 +300,9 @@ github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= @@ -307,19 +329,17 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -329,16 +349,16 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/goveralls v0.0.6 h1:cr8Y0VMo/MnEZBjxNN/vh6G90SZ7IMb6lms1dzMoO+Y= @@ -355,8 +375,9 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -386,8 +407,10 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= +github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -396,14 +419,15 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= -github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= -github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -416,6 +440,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVOB87y175cLJC/mbsgKmoDOjrBldtXvioEy96WY= +github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3/go.mod h1:vl5+MqJ1nBINuSsUI2mGgH79UweUT/B5Fy8857PqyyI= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/robinjoseph08/go-pg-migrations/v3 v3.0.0 h1:0/H63lDsoNYVn5YmP6VLDEnnKkoVYiHx7udTWCK4BUI= github.com/robinjoseph08/go-pg-migrations/v3 v3.0.0/go.mod h1:nOkSFfwwDUBFnDDQqMRC2p4PDE7GZb/KSVqILVB3bmw= @@ -427,15 +453,18 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= +github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 h1:Xuk8ma/ibJ1fOy4Ee11vHhUFHQNpHhrBneOCNHVXS5w= github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0/go.mod h1:7AwjWCpdPhkSmNAgUv5C7EJ4AbmjEB3r047r3DXWu3Y= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -465,8 +494,9 @@ github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -475,13 +505,15 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= -github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= @@ -502,8 +534,9 @@ github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5 github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.8/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= -github.com/vmihailenco/msgpack/v5 v5.3.1 h1:0i85a4dsZh8mC//wmyyTEzidDLPQfQAxZIOLtafGbFY= -github.com/vmihailenco/msgpack/v5 v5.3.1/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= @@ -517,16 +550,20 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -536,6 +573,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -545,8 +583,8 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -557,8 +595,8 @@ golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -579,7 +617,9 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -610,6 +650,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -623,6 +666,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -643,12 +688,9 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -658,26 +700,38 @@ golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201026173827-119d4633e4d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -704,14 +758,18 @@ golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.1 h1:/LSrTrgZtpbXyAR6+0e152SROCkJJSh7goYWVmdPFGc= gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -757,19 +815,18 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/AlecAivazis/survey.v1 v1.8.7 h1:oBJqtgsyBLg9K5FK9twNUbcPnbCPoh+R9a+7nag3qJM= gopkg.in/AlecAivazis/survey.v1 v1.8.7/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA= -gopkg.in/DataDog/dd-trace-go.v1 v1.29.0 h1:3C1EEjgFTPqrnS2SXuSqkBbZGacIOPJ7ScGJk4nrP9s= -gopkg.in/DataDog/dd-trace-go.v1 v1.29.0/go.mod h1:FLwUDeuH0z5hkvgvd04/M3MHQN4AF5pQDnedeWRWvok= +gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 h1:Ne7kzWr/br/jwhUJR7CnqPl/mUpNxa6LfgZs0S4htZM= +gopkg.in/DataDog/dd-trace-go.v1 v1.65.1/go.mod h1:beNFIWd/H04d0k96cfltgiDH2+t0T5sDbyYLF3VTXqk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= @@ -794,13 +851,15 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/gotraceui v0.2.0 h1:dmNsfQ9Vl3GwbiVD7Z8d/osC6WtGGrasyrC2suc4ZIQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w= mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index 7e0ba023e..ae8a2c94a 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -1,6 +1,7 @@ package lib import ( + "bytes" "fmt" "path/filepath" "sync" @@ -944,6 +945,15 @@ func (mp *PosMempool) UpdateLatestBlock(blockView *UtxoView, blockHeight uint64) // new minimum will be removed from the mempool. To safely handle this, this method re-creates the TransactionRegister // with the new global params and re-adds all transactions in the mempool to the new register. func (mp *PosMempool) UpdateGlobalParams(globalParams *GlobalParamsEntry) { + // If the global params haven't changed at all, then we don't need to do anything. + newGlobalParamBytes := globalParams.RawEncodeWithoutMetadata(mp.latestBlockHeight, true) + mp.RLock() + mpGlobalParamBytes := mp.globalParams.RawEncodeWithoutMetadata(mp.latestBlockHeight, true) + mp.RUnlock() + if bytes.Equal(newGlobalParamBytes, mpGlobalParamBytes) { + return + } + mp.Lock() defer mp.Unlock() diff --git a/lib/server.go b/lib/server.go index 71f003c8b..608cccc8b 100644 --- a/lib/server.go +++ b/lib/server.go @@ -18,7 +18,7 @@ import ( "github.com/decred/dcrd/lru" - "github.com/DataDog/datadog-go/statsd" + "github.com/DataDog/datadog-go/v5/statsd" "github.com/btcsuite/btcd/addrmgr" chainlib "github.com/btcsuite/btcd/blockchain" From 8b73b1cba90de39c13ec2eb14b1bc26c44f71c57 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 3 Oct 2024 00:47:49 -0400 Subject: [PATCH 20/62] handle case where we haven't hit PoS cutover yet in block index migration --- lib/db_utils.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/db_utils.go b/lib/db_utils.go index f6fe875fe..6b7ebfc62 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5669,6 +5669,9 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager nodeIterator := txn.NewIterator(opts) defer nodeIterator.Close() hashToHeightMap := make(map[BlockHash]uint32) + // Just in case we need it, get the height of the best hash. + bestHash := DbGetBestHash(handle, snapshot, ChainTypeDeSoBlock) + var bestHashHeight uint32 for nodeIterator.Seek(prefix); nodeIterator.ValidForPrefix(prefix); nodeIterator.Next() { item := nodeIterator.Item().Key() @@ -5677,6 +5680,9 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager hash := BlockHash{} copy(hash[:], item[5:]) hashToHeightMap[hash] = height + if bestHash != nil && bestHash.IsEqual(&hash) { + bestHashHeight = height + } if len(hashToHeightMap) < 10000 { continue } @@ -5692,6 +5698,10 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") } } + // If we don't have a best hash, then we certainly haven't hit the first pos block height. + if bestHash == nil { + return nil + } // TODO: get best chain up to PoS Cutover height and set all blocks in that chain to committed. firstPoSBlockHeight := params.GetFirstPoSBlockHeight() // Look up blocks at cutover height. @@ -5700,15 +5710,21 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager if err != nil { return errors.Wrap(err, "RunBlockIndexMigration: Problem enumerating keys for prefix") } - if len(valsFound) == 0 { - return fmt.Errorf("RunBlockIndexMigration: No blocks found at PoS cutover height") - } if len(valsFound) > 1 { return fmt.Errorf("RunBlockIndexMigration: More than one block found at PoS cutover height") } - blockNode, err := DeserializeBlockNode(valsFound[0]) - if err != nil { - return errors.Wrap(err, "RunBlockIndexMigration: Problem deserializing block node for pos cutover") + var blockNode *BlockNode + // In this case, we need to find pull the best hash from the DB and iterate backwards. + if len(valsFound) == 0 { + blockNode = GetHeightHashToNodeInfoWithTxn(txn, snapshot, bestHashHeight, bestHash, false) + if blockNode == nil { + return fmt.Errorf("RunBlockIndexMigration: block with Best hash (%v) and height (%v) not found", bestHash, bestHashHeight) + } + } else { + blockNode, err = DeserializeBlockNode(valsFound[0]) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem deserializing block node for pos cutover") + } } var blockNodeBatch []*BlockNode for blockNode != nil { From 7ac67bdf52a779a086a7aec7f3332a1ba9bb6c57 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 4 Oct 2024 18:37:19 -0400 Subject: [PATCH 21/62] cleanup --- lib/blockchain.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 0620b545b..20fd99bbb 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1335,14 +1335,8 @@ func NewBlockchain( eventManager: eventManager, archivalMode: archivalMode, - blockIndex: NewBlockIndex(db, snapshot, nil), // TODO: replace with actual tip. - //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), - //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), - //bestChainMap: make(map[BlockHash]*BlockNode), - //bestChain: NewBestChain(db, snapshot, false, params), - //bestHeaderChainMap: make(map[BlockHash]*BlockNode), - //bestHeaderChain: NewBestChain(db, snapshot, true, params), - blockViewCache: lru.NewKVCache(100), // TODO: parameterize + blockIndex: NewBlockIndex(db, snapshot, nil), // TODO: replace with actual tip. + blockViewCache: lru.NewKVCache(100), // TODO: parameterize snapshotCache: NewSnapshotCache(), checkpointSyncingProviders: checkpointSyncingProviders, From 7675765dc6884d1a9aa1c7d7e0950a3d114f18b5 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 17:21:07 -0400 Subject: [PATCH 22/62] do some sampling for is fully stored and check archival mode --- lib/blockchain.go | 87 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 20fd99bbb..591022eb2 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1085,13 +1085,37 @@ func (bc *Blockchain) IsFullyStored() bool { chainState := bc.ChainState() if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && bc.headerTip().Height-bc.blockTip().Height < 10) { - // TODO: rewrite to work w/ block index. - panic("IsFullyStored: Need to implement this.") - //for _, blockNode := range bc.bestChain.Chain { - // if !blockNode.Status.IsFullyProcessed() { - // return false - // } - //} + // Get a sampling of blocks from the best chain and check if they are fully stored. + // We only need to check a few blocks to determine if the chain is fully stored. + blockTipHeight := uint64(bc.BlockTip().Height) + increment := blockTipHeight / 20 + if increment == 0 { + increment = 1 + } + blockHeights := []uint64{} + for ii := uint64(0); ii < blockTipHeight; ii += increment { + blockHeights = append(blockHeights, ii) + } + if blockTipHeight > 100 { + for ii := blockTipHeight - 20; ii < blockTipHeight; ii++ { + blockHeights = append(blockHeights, ii) + } + } + blockHeights = append(blockHeights, blockTipHeight) + blockHeightSet := NewSet(blockHeights) + for _, blockHeight := range blockHeightSet.ToSlice() { + blockNode, exists, err := bc.GetBlockFromBestChainByHeight(blockHeight, false) + if err != nil { + glog.Errorf("IsFullyStored: Problem getting block at height %d: %v", blockHeight, err) + return false + } + if !exists { + return false + } + if !blockNode.Status.IsFullyProcessed() { + return false + } + } return true } return false @@ -1960,23 +1984,38 @@ func (bc *Blockchain) checkArchivalMode() bool { _ = firstSnapshotHeight // @diamondhands - can we spot check just a few blocks such as firstSnapshotHeight - 1, // firstSnapshotHeight / 2 - 1, and firstSnapshotHeight / 4 - 1 to see if they are stored? - // TODO: figure out how to iterate over best chain to checkArchivalMode - // when we only have a portion of best chain in memory. - panic("NOT IMPLEMENTED: checkArchivalMode") - //for _, blockNode := range bc.bestChain.Chain { - // if uint64(blockNode.Height) > firstSnapshotHeight { - // return false - // } - // - // // Check if we have blocks that have been processed and validated but not stored. This would indicate that there - // // are historical blocks that we are yet to download. - // if (blockNode.Status&StatusBlockProcessed) == 1 && - // (blockNode.Status&StatusBlockValidated) == 1 && - // (blockNode.Status&StatusBlockStored) == 0 { - // - // return true - // } - //} + // We take a sampling of blocks to determine if we've downloaded all the blocks up to the first snapshot height. + blockHeights := []uint64{} + increment := firstSnapshotHeight / 10 + for ii := uint64(0); ii < firstSnapshotHeight; ii += increment { + blockHeights = append(blockHeights, ii) + } + for ii := firstSnapshotHeight - 10; ii < firstSnapshotHeight; ii++ { + blockHeights = append(blockHeights, ii) + } + blockHeights = append(blockHeights, firstSnapshotHeight) + for _, height := range blockHeights { + blockNode, exists, err := bc.GetBlockFromBestChainByHeight(height, false) + if err != nil { + glog.Errorf("checkArchivalMode: Problem getting block by height: %v", err) + return false + } + if !exists { + return false + } + if uint64(blockNode.Height) > firstSnapshotHeight { + return false + } + + // Check if we have blocks that have been processed and validated but not stored. This would indicate that there + // are historical blocks that we are yet to download. + if (blockNode.Status&StatusBlockProcessed) == 1 && + (blockNode.Status&StatusBlockValidated) == 1 && + (blockNode.Status&StatusBlockStored) == 0 { + + return true + } + } // If we get here, it means that all blocks have been processed and stored, so there is nothing to do. return false From 148bad3ae471eb14f6097f96b4470f5f0737fb0c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 17:48:41 -0400 Subject: [PATCH 23/62] fix get blocks to store --- lib/blockchain.go | 20 +------- lib/server.go | 120 ++++++++++++++++++++++++++-------------------- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 591022eb2..3d0801d6a 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -870,24 +870,8 @@ type Blockchain struct { ChainLock deadlock.RWMutex // These should only be accessed after acquiring the ChainLock. - // - // An in-memory index of the "tree" of blocks we are currently aware of. - // This index includes forks and side-chains. - //blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode] - //// blockIndexByHeight is an in-memory map of block height to block nodes. This is - //// used to quickly find the safe blocks from which the chain can be extended for PoS - //blockIndexByHeight map[uint64]map[BlockHash]*BlockNode - // An in-memory slice of the blocks on the main chain only. The end of - // this slice is the best known tip that we have at any given time. - //bestChain []*BlockNode - //bestChainMap map[BlockHash]*BlockNode - // - //bestHeaderChain []*BlockNode - //bestHeaderChainMap map[BlockHash]*BlockNode - - blockIndex *BlockIndex - //bestChain *BestChain - //bestHeaderChain *BestChain + blockIndex *BlockIndex + lowestBlockNotStored uint64 // We keep track of orphan blocks with the following data structures. Orphans // are not written to disk and are only cached in memory. Moreover we only keep diff --git a/lib/server.go b/lib/server.go index 9f6fcf36d..6fe1f0766 100644 --- a/lib/server.go +++ b/lib/server.go @@ -879,59 +879,73 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } // Go through the block nodes in the blockchain and download the blocks if they're not stored. - // TODO: need to figure out a way to get all the blocks in the best chain so we can download historical blocks. - //for _, blockNode := range srv.blockchain.bestChain.Chain { - // // We find the first block that's not stored and get ready to download blocks starting from this block onwards. - // if blockNode.Status&StatusBlockStored == 0 { - // maxBlocksInFlight := MaxBlocksInFlight - // if pp.NegotiatedProtocolVersion >= ProtocolVersion2 && - // (srv.params.IsPoSBlockHeight(uint64(blockNode.Height)) || - // srv.params.NetworkType == NetworkType_TESTNET) { - // - // maxBlocksInFlight = MaxBlocksInFlightPoS - // } - // numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) - // currentHeight := int(blockNode.Height) - // blockNodesToFetch := []*BlockNode{} - // // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. - // var heightLimit int - // for heightLimit = len(srv.blockchain.bestChain.Chain) - 1; heightLimit >= 0; heightLimit-- { - // if !srv.blockchain.bestChain.Chain[heightLimit].Status.IsFullyProcessed() { - // break - // } - // } - // - // // Find the blocks that we should download. - // for currentHeight <= heightLimit && - // len(blockNodesToFetch) < numBlocksToFetch { - // - // // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to - // // the array index. - // currentNode := srv.blockchain.bestChain.Chain[currentHeight] - // currentHeight++ - // - // // If we've already requested this block then we don't request it again. - // if _, exists := pp.requestedBlocks[*currentNode.Hash]; exists { - // continue - // } - // - // blockNodesToFetch = append(blockNodesToFetch, currentNode) - // } - // - // var hashList []*BlockHash - // for _, node := range blockNodesToFetch { - // hashList = append(hashList, node.Hash) - // pp.requestedBlocks[*node.Hash] = true - // } - // pp.AddDeSoMessage(&MsgDeSoGetBlocks{ - // HashList: hashList, - // }, false) - // - // glog.V(1).Infof("GetBlocksToStore: Downloading blocks to store for header %v from peer %v", - // blockNode.Header, pp) - // return - // } - //} + for ii := uint32(srv.blockchain.lowestBlockNotStored); ii <= srv.blockchain.blockTip().Height; ii++ { + blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(ii), false) + if err != nil { + glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) + return + } + if !exists { + glog.Errorf("GetBlocksToStore: Block at height %v not found in best chain", ii) + return + } + // We find the first block that's not stored and get ready to download blocks starting from this block onwards. + if blockNode.Status&StatusBlockStored == 0 { + maxBlocksInFlight := MaxBlocksInFlight + if pp.NegotiatedProtocolVersion >= ProtocolVersion2 && + (srv.params.IsPoSBlockHeight(uint64(blockNode.Height)) || + srv.params.NetworkType == NetworkType_TESTNET) { + + maxBlocksInFlight = MaxBlocksInFlightPoS + } + srv.blockchain.lowestBlockNotStored = uint64(blockNode.Height) + numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) + currentHeight := int(blockNode.Height) + blockNodesToFetch := []*BlockNode{} + // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. + // We filter those out in the loop below by checking IsFullyProcessed. + // Find the blocks that we should download. + for len(blockNodesToFetch) < numBlocksToFetch { + + // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to + // the array index. + currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(currentHeight), false) + if err != nil { + glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) + return + } + if !currNodeExists { + glog.Errorf("GetBlocksToStore: Block at height %v not found in best chain", currentHeight) + return + } + currentHeight++ + // If this node is already fully processed, then we don't need to download it. + if currentNode.Status.IsFullyProcessed() { + break + } + + // If we've already requested this block then we don't request it again. + if _, exists = pp.requestedBlocks[*currentNode.Hash]; exists { + continue + } + + blockNodesToFetch = append(blockNodesToFetch, currentNode) + } + + var hashList []*BlockHash + for _, node := range blockNodesToFetch { + hashList = append(hashList, node.Hash) + pp.requestedBlocks[*node.Hash] = true + } + pp.AddDeSoMessage(&MsgDeSoGetBlocks{ + HashList: hashList, + }, false) + + glog.V(1).Infof("GetBlocksToStore: Downloading blocks to store for header %v from peer %v", + blockNode.Header, pp) + return + } + } // If we get here then it means that we've downloaded all blocks so we can update srv.blockchain.downloadingHistoricalBlocks = false From 8aa6a5b404bbe2d9235b6b97f8ef94ba709a390f Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 17:57:02 -0400 Subject: [PATCH 24/62] cleanup --- lib/blockchain.go | 236 +++------------------------------------------- 1 file changed, 11 insertions(+), 225 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 3d0801d6a..79818c28c 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -702,148 +702,6 @@ func (bi *BlockIndex) GetHeaderTip() *BlockNode { return bi.headerTip } -//type BestChain struct { -// db *badger.DB -// snapshot *Snapshot -// IsHeaderChain bool -// Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. -// ChainMap *lru2.Cache[BlockHash, *BlockNode] -// params *DeSoParams -//} -// -//func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { -// chainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) -// return &BestChain{ -// db: db, -// snapshot: snapshot, -// IsHeaderChain: isHeaderChain, -// Chain: []*BlockNode{}, -// ChainMap: chainMap, -// params: params, -// } -//} -// -//func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { -// bestChain.Chain = append(bestChain.Chain, tipNode) -// bestChain.ChainMap.Add(*tipNode.Hash, tipNode) -//} -// -//func (bestChain *BestChain) GetTip() *BlockNode { -// if len(bestChain.Chain) == 0 { -// return nil -// } -// return bestChain.Chain[len(bestChain.Chain)-1] -//} -// -//func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, error) { -// if height > math.MaxUint32 { -// return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) -// } -// -// currTip := bestChain.GetTip() -// if currTip != nil && height > uint64(currTip.Height) { -// return nil, false, nil -// } -// -// if currTip != nil { -// currTipHeight := currTip.Height -// delta := currTipHeight - uint32(height) -// if delta < uint32(len(bestChain.Chain)) { -// targetNode := bestChain.Chain[uint32(len(bestChain.Chain)-1)-delta] -// if uint64(targetNode.Height) == height { -// return targetNode, true, nil -// } -// } -// //targetIndex := (len(bestChain.Chain)-1) -// //// We can probably do some binary search thing here instead. -// //currNode := &BlockNode{} -// //*currNode = *currTip -// //if currNode.Height == uint32(height) { -// // return currNode, true, nil -// //} -// //// During syncing, we don't write to the DB, so we need to iterate backwards through the best chain. -// //for currNode != nil && !currNode.IsStored() { -// // if currNode.Height < uint32(height) { -// // break -// // } -// // if currNode.Height == uint32(height) { -// // return currNode, true, nil -// // } -// // var currNodeExists bool -// // currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( -// // currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) -// // if !currNodeExists { -// // break -// // } -// //} -// } -// -// prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) -// _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) -// if len(valsFound) == 0 { -// return nil, false, nil -// } -// for _, val := range valsFound { -// blockNode, err := DeserializeBlockNode(val) -// if err != nil { -// glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) -// continue -// } -// if blockNode.IsCommitted() { -// return blockNode, true, nil -// } -// } -// // TODO: how to return uncommitted blocks by height. We probably just need to iterate backwards through the -// // best chain. -// return nil, false, nil -//} -// -//func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { -// val, exists := bestChain.ChainMap.Get(*blockHash) -// if exists { -// return val, true -// } -// if height > math.MaxUint32 { -// glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) -// } -// if height > uint64(bestChain.GetTip().Height) { -// return nil, false -// } -// bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) -// if bn == nil { -// return nil, false -// } -// bestChain.ChainMap.Add(*blockHash, bn) -// return bn, true -//} -// -//func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bool, error) { -// val, exists := bestChain.ChainMap.Get(*blockHash) -// if exists { -// return val, true, nil -// } -// -// height, err := GetHeightForHash(bestChain.db, bestChain.snapshot, blockHash) -// if err != nil { -// if errors.Is(err, badger.ErrKeyNotFound) { -// return nil, false, nil -// } -// return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") -// } -// if height > uint64(bestChain.GetTip().Height) { -// return nil, false, nil -// } -// bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) -// if bn == nil { -// return nil, false, nil -// } -// if !bn.IsCommitted() { -// return nil, false, nil -// } -// bestChain.ChainMap.Add(*blockHash, bn) -// return bn, true, nil -//} - type Blockchain struct { db *badger.DB postgres *Postgres @@ -1030,39 +888,10 @@ func (bc *Blockchain) getAllBlockNodesIndexedAtHeight(blockHeight uint64) []*Blo } func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { - //blocksAtHeight, hasNestedMapAtHeight := bc.blockIndexByHeight[blockHeight] - //if !hasNestedMapAtHeight { - // return false - //} - //return len(blocksAtHeight) > 0 blockNodes := bc.blockIndex.GetBlockNodesByHeight(blockHeight) return len(blockNodes) > 0 } -//func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { -// newBestChain := []*BlockNode{} -// newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) -// newBestChain = append(newBestChain, bc.bestChain.Chain...) -// for _, key := range bc.bestChain.ChainMap.Keys() { -// val, _ := bc.bestChain.ChainMap.Get(key) -// newBestChainMap.Add(key, val) -// } -// -// return newBestChain, newBestChainMap -//} - -//func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { -// newBestChain := []*BlockNode{} -// newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) -// newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) -// for _, key := range bc.bestHeaderChain.ChainMap.Keys() { -// val, _ := bc.bestHeaderChain.ChainMap.Get(key) -// newBestChainMap.Add(key, val) -// } -// -// return newBestChain, newBestChainMap -//} - // IsFullyStored determines if there are block nodes that haven't been fully stored or processed in the best block chain. func (bc *Blockchain) IsFullyStored() bool { // TODO: figure out how to iterate over best chain w/o having entire thing in memory. @@ -1192,9 +1021,9 @@ func (bc *Blockchain) _initChain() error { if !tipNodeExists { return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) } - // Walk back the last 100 hours of blocks. + // Walk back the last 24 hours of blocks. currBlockCounter := 1 - for currBlockCounter < 3600*100 && tipNode.Header.PrevBlockHash != nil { + for currBlockCounter < 3600*24 && tipNode.Header.PrevBlockHash != nil { bc.blockIndex.GetBlockNodeByHashAndHeight(tipNode.Header.PrevBlockHash, tipNode.Header.Height-1) currBlockCounter++ } @@ -1204,38 +1033,9 @@ func (bc *Blockchain) _initChain() error { } // We start by simply setting the chain tip and header tip to the tip node. - bc.blockIndex.tip = tipNode - bc.blockIndex.headerTip = tipNode - - } - - // At this point the blockIndexByHash should contain a full node tree with all - // nodes pointing to valid parent nodes. - { - // Walk back from the best node to the genesis block and store them all - // in bestChain. - //bc.bestChain.Chain, err = GetBestChain(tipNode) - //if err != nil { - // return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") - //} - //for _, bestChainNode := range bc.bestChain.Chain { - // bc.bestChain.ChainMap.Add(*bestChainNode.Hash, bestChainNode) - //} - } - - // TODO: This code is a bit repetitive but this seemed clearer than factoring it out. - { - // Walk back from the best node to the genesis block and store them all - // in bestChain. - //bc.bestHeaderChain.Chain, err = GetBestChain(tipNode) - //if err != nil { - // return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") - //} - //for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { - // bc.bestHeaderChain.ChainMap.Add(*bestHeaderChainNode.Hash, bestHeaderChainNode) - //} + bc.blockIndex.SetTip(tipNode) + bc.blockIndex.SetHeaderTip(tipNode) } - bc.isInitialized = true return nil @@ -1277,22 +1077,12 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { } // Add the uncommitted blocks to the in-memory data structures. - if _, _, _, err := bc.tryApplyNewTip(uncommittedTipBlockNode, 0, lineageFromCommittedTip); err != nil { + if _, _, _, err = bc.tryApplyNewTip(uncommittedTipBlockNode, 0, lineageFromCommittedTip); err != nil { return errors.Wrapf(err, "_applyUncommittedBlocksToBestChain: ") } bc.blockIndex.SetTip(uncommittedTipBlockNode) - - ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// - //currentHeaderTip := bc.headerTip() - //_, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) - //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - // bc.bestHeaderChain.Chain, - // bc.bestHeaderChain.ChainMap, - // blocksToDetach, - // blocksToAttach, - //) - + bc.blockIndex.SetHeaderTip(uncommittedTipBlockNode) return nil } @@ -2696,12 +2486,6 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true bc.blockIndex.SetHeaderTip(newNode) - //_, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode) - //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - // bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) - - // Note that we don't store the best header hash here and so this is an - // in-memory-only adjustment. See the comment above on preventing attacks. } return isMainChain, false, nil @@ -3121,7 +2905,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // update our data structures to actually make this connection. Do this // in a transaction so that it is atomic. if bc.postgres != nil { - // TODO: pg support for setting committed on block node. + if !nodeToValidate.IsCommitted() { + nodeToValidate.Status |= StatusBlockCommitted + bc.blockIndex.addNewBlockNodeToBlockIndex(nodeToValidate) + } + if err = bc.postgres.UpsertBlockAndTransactions(nodeToValidate, desoBlock); err != nil { return false, false, errors.Wrapf(err, "ProcessBlock: Problem upserting block and transactions") } @@ -3210,8 +2998,6 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures if bc.isSyncing() { bc.blockIndex.SetTip(nodeToValidate) } else { - //newBestChain, newBestChainMap := bc.CopyBestChain() - //bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap bc.blockIndex.SetTip(nodeToValidate) } From 53b2d298d9589bc4ff11b27f3ac027ce5d27d913 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 19:55:39 -0400 Subject: [PATCH 25/62] make a bunch of methods on block index private and expose block index to backend --- lib/block_view_lockups_test.go | 2 +- lib/block_view_test.go | 4 ++-- lib/blockchain.go | 34 +++++++++++++++++++--------------- lib/blockchain_test.go | 16 ++++++++-------- lib/db_utils.go | 2 ++ lib/pos_blockchain.go | 6 +++--- lib/pos_blockchain_test.go | 16 ++++++++-------- lib/pos_consensus_test.go | 2 +- lib/server.go | 2 +- 9 files changed, 45 insertions(+), 39 deletions(-) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index c4f127892..41e53e74e 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2518,7 +2518,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk1.Header.Height)) // Update the tip - testMeta.chain.blockIndex.SetTip(testMeta.chain.blockIndex.tip.Parent) + testMeta.chain.blockIndex.setTip(testMeta.chain.blockIndex.tip.Parent) // Verify we return back to the initial state utxoView = NewUtxoView( diff --git a/lib/block_view_test.go b/lib/block_view_test.go index f02d06e36..ff17677a4 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -791,8 +791,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // TODO: if ever needed we can call tm.chain.eventManager.blockDisconnected() here. // Update the block and header metadata chains. - tm.chain.blockIndex.SetTip(tm.chain.BlockTip().GetParent(tm.chain.blockIndex)) - tm.chain.blockIndex.SetHeaderTip(tm.chain.HeaderTip().GetParent(tm.chain.blockIndex)) + tm.chain.blockIndex.setTip(tm.chain.BlockTip().GetParent(tm.chain.blockIndex)) + tm.chain.blockIndex.setHeaderTip(tm.chain.HeaderTip().GetParent(tm.chain.blockIndex)) // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. diff --git a/lib/blockchain.go b/lib/blockchain.go index 79818c28c..eae552147 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -585,7 +585,7 @@ func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *Block } } -func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { +func (bi *BlockIndex) setBlockIndexFromMap(input map[BlockHash]*BlockNode) { newHashToBlockNodeMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) newHeightToBlockNodeMap, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) bi.blockIndexByHash = newHashToBlockNodeMap @@ -602,13 +602,13 @@ func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { } } -func (bi *BlockIndex) SetHeaderTip(tip *BlockNode) { +func (bi *BlockIndex) setHeaderTip(tip *BlockNode) { // Just to be safe, we also add it to the block index. bi.addNewBlockNodeToBlockIndex(tip) bi.headerTip = tip } -func (bi *BlockIndex) SetTip(tip *BlockNode) { +func (bi *BlockIndex) setTip(tip *BlockNode) { // Just to be safe, we also add it to the block index. bi.addNewBlockNodeToBlockIndex(tip) bi.tip = tip @@ -882,6 +882,10 @@ func (bc *Blockchain) CopyBlockIndexes() ( return newBlockIndexByHash } +func (bc *Blockchain) GetBlockIndex() *BlockIndex { + return bc.blockIndex +} + // TODO: read through to DB. func (bc *Blockchain) getAllBlockNodesIndexedAtHeight(blockHeight uint64) []*BlockNode { return bc.blockIndex.GetBlockNodesByHeight(blockHeight) @@ -1033,8 +1037,8 @@ func (bc *Blockchain) _initChain() error { } // We start by simply setting the chain tip and header tip to the tip node. - bc.blockIndex.SetTip(tipNode) - bc.blockIndex.SetHeaderTip(tipNode) + bc.blockIndex.setTip(tipNode) + bc.blockIndex.setHeaderTip(tipNode) } bc.isInitialized = true @@ -1081,8 +1085,8 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { return errors.Wrapf(err, "_applyUncommittedBlocksToBestChain: ") } - bc.blockIndex.SetTip(uncommittedTipBlockNode) - bc.blockIndex.SetHeaderTip(uncommittedTipBlockNode) + bc.blockIndex.setTip(uncommittedTipBlockNode) + bc.blockIndex.setHeaderTip(uncommittedTipBlockNode) return nil } @@ -1953,9 +1957,9 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { for _, blockNode := range bestChain { bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) if bc.blockIndex.GetTip() == nil { - bc.blockIndex.SetTip(blockNode) + bc.blockIndex.setTip(blockNode) } else if bc.blockIndex.GetTip().Height < blockNode.Height { - bc.blockIndex.SetTip(blockNode) + bc.blockIndex.setTip(blockNode) } } } @@ -1965,7 +1969,7 @@ func (bc *Blockchain) SetBestChainMap( tipNode *BlockNode, ) { bc.blockIndex.blockIndexByHash = blockIndexByHash - bc.blockIndex.SetTip(tipNode) + bc.blockIndex.setTip(tipNode) } func (bc *Blockchain) _validateOrphanBlockPoW(desoBlock *MsgDeSoBlock) error { @@ -2485,7 +2489,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B headerTip := bc.headerTip() if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - bc.blockIndex.SetHeaderTip(newNode) + bc.blockIndex.setHeaderTip(newNode) } return isMainChain, false, nil @@ -2996,9 +3000,9 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If we're syncing there's no risk of concurrency issues. Otherwise, we // need to make a copy in order to be save. if bc.isSyncing() { - bc.blockIndex.SetTip(nodeToValidate) + bc.blockIndex.setTip(nodeToValidate) } else { - bc.blockIndex.SetTip(nodeToValidate) + bc.blockIndex.setTip(nodeToValidate) } // This node is on the main chain so set this variable. @@ -3286,8 +3290,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Now the db has been updated, update our in-memory best chain. Note that there // is no need to update the node index because it was updated as we went along. - bc.blockIndex.SetTip(newTipNode) - bc.blockIndex.SetHeaderTip(newTipNode) + bc.blockIndex.setTip(newTipNode) + bc.blockIndex.setHeaderTip(newTipNode) // If we made it here then this block is on the main chain. isMainChain = true diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 2bf1cab3d..1d6cfe406 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1254,7 +1254,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1292,7 +1292,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1356,7 +1356,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1394,7 +1394,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1458,7 +1458,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1516,7 +1516,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1575,7 +1575,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1612,7 +1612,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } diff --git a/lib/db_utils.go b/lib/db_utils.go index 6b7ebfc62..1a231b3b4 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5754,6 +5754,8 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager }) } +// TODO: refactor to actually get the whole best chain if that's +// what someone wants. It'll take a while, but whatever. func GetBestChain(tipNode *BlockNode) ([]*BlockNode, error) { reversedBestChain := []*BlockNode{} maxBestChainInitLength := 3600 * 100 // Cache up to 100 hours of blocks. diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 7b5dd2c86..4ae8013ac 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -97,7 +97,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b return false, false, nil } - bc.blockIndex.SetHeaderTip(blockNode) + bc.blockIndex.setHeaderTip(blockNode) // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. @@ -1650,7 +1650,7 @@ func (bc *Blockchain) shouldReorg(blockNode *BlockNode, currentView uint64) bool // addTipBlockToBestChain adds the block as the new tip of the best chain. func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { - bc.blockIndex.SetTip(blockNode) + bc.blockIndex.setTip(blockNode) } // removeTipBlockFromBestChain removes the current tip from the best chain. It @@ -1661,7 +1661,7 @@ func (bc *Blockchain) removeTipBlockFromBestChain() *BlockNode { // Remove the last block from the best chain. lastBlock := bc.blockIndex.GetTip() // Uhhh what happens if we don't have the parent set up!? - bc.blockIndex.SetTip(lastBlock.GetParent(bc.blockIndex)) + bc.blockIndex.setTip(lastBlock.GetParent(bc.blockIndex)) return lastBlock } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 888ee1f9b..291a34eab 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -252,7 +252,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{*genesisBlock.Hash: genesisBlock}) + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{*genesisBlock.Hash: genesisBlock}) bc.blockIndex.blockIndexByHash.Add(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) @@ -331,7 +331,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -473,7 +473,7 @@ func TestHasValidBlockViewPoS(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -805,7 +805,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { Height: 1, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) block := &MsgDeSoBlock{ @@ -841,7 +841,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 2, PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.blockIndex.SetTip(block2) + bc.blockIndex.setTip(block2) bc.blockIndex.blockIndexByHash.Add(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) @@ -1244,7 +1244,7 @@ func TestShouldReorg(t *testing.T) { Height: 1, }, } - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: chain[0], *hash3: chain[1], }) @@ -1378,7 +1378,7 @@ func TestTryApplyNewTip(t *testing.T) { require.True(t, checkBestChainForHash(hash1)) // Remove newBlock from the best chain and block index to reset the state. - bc.blockIndex.SetTip(bc.blockIndex.GetTip().GetParent(bc.blockIndex)) + bc.blockIndex.setTip(bc.blockIndex.GetTip().GetParent(bc.blockIndex)) // Add block 3 back bc.addTipBlockToBestChain(bn3) @@ -1457,7 +1457,7 @@ func TestTryApplyNewTip(t *testing.T) { //bc.bestChain.ChainMap.Remove(*hash5) //bc.bestChain.ChainMap.Remove(*newBlockHash) //bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] - bc.blockIndex.SetTip(newBlockNode.GetParent(bc.blockIndex)) + bc.blockIndex.setTip(newBlockNode.GetParent(bc.blockIndex)) // Add block 2 and 3 back. bc.addTipBlockToBestChain(bn2) diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index 9d471b2a5..3e85457e9 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -103,7 +103,7 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { nextView := currentView + 1 blockIndex := NewBlockIndex(nil, nil, nil) - blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *blockHash: {Header: blockHeader, Height: uint32(blockHeader.Height), Hash: blockHash}, }) diff --git a/lib/server.go b/lib/server.go index 6fe1f0766..28b225a48 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1756,7 +1756,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { currentNode.Status |= StatusBlockValidated currentNode.Status |= StatusBlockCommitted srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) - srv.blockchain.blockIndex.SetTip(currentNode) + srv.blockchain.blockIndex.setTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) if len(blockNodeBatch) < 10000 { continue From 91e22ffc5ebbb15cf39a7a031bd7d1699cfce33c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 20:23:37 -0400 Subject: [PATCH 26/62] don't bother healing orphan pointers if we're syncing --- lib/pos_blockchain.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 4ae8013ac..e9388353f 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -81,9 +81,12 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b return false, false, errors.Wrapf(err, "processHeaderPoS: Problem validating and indexing header: ") } - // Now that we know we have a valid header, we check the block index for it any orphan children for it - // and heal the parent pointers for all of them. - bc.healPointersForOrphanChildren(blockNode) + // Don't worry about healing orphan children when we're syncing. + if !bc.isSyncing() { + // Now that we know we have a valid header, we check the block index for it any orphan children for it + // and heal the parent pointers for all of them. + bc.healPointersForOrphanChildren(blockNode) + } // Exit early if the header is an orphan. if isOrphan { From 9abe23865512255c938fbd74d355e3d65aa6f279 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 21:12:23 -0400 Subject: [PATCH 27/62] refactor to reduce number of times we have to call hash and don't bother healing orphan pointers during syncing --- lib/blockchain.go | 2 +- lib/pos_blockchain.go | 151 +++++++++++++++++++------------------ lib/pos_blockchain_test.go | 34 ++++----- 3 files changed, 96 insertions(+), 91 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index eae552147..3ecdd2b5e 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2508,7 +2508,7 @@ func (bc *Blockchain) ProcessHeader(blockHeader *MsgDeSoHeader, headerHash *Bloc // If the header's height is after the PoS cut-over fork height, then we use the PoS header processing logic. // Otherwise, fall back to the PoW logic. if bc.params.IsPoSBlockHeight(blockHeader.Height) { - return bc.processHeaderPoS(blockHeader, verifySignatures) + return bc.processHeaderPoS(blockHeader, headerHash, verifySignatures) } return bc.processHeaderPoW(blockHeader, headerHash) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index e9388353f..2b0383f95 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -38,7 +38,7 @@ import ( // StatusHeaderValidated or StatusHeaderValidateFailed. // 5. Exit early if the's view is less than the current header chain's tip. // 6. Reorg the best header chain if the header's view is higher than the current tip. -func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures bool) ( +func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockHash, verifySignatures bool) ( _isMainChain bool, _isOrphan bool, _err error, ) { if !bc.params.IsPoSBlockHeight(header.Height) { @@ -48,19 +48,13 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b ) } - headerHash, err := header.Hash() - if err != nil { - return false, false, errors.Wrapf(err, "processHeaderPoS: Problem hashing header") - } - // If the incoming header is already part of the best header chain, then we can exit early. // The header is not part of a fork, and is already an ancestor of the current header chain tip. // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. // TODO: make sure this is ok or do we need to explicitly check the block index's cache? - var isInBestHeaderChain bool - _, isInBestHeaderChain, err = bc.GetBlockFromBestChainByHash(headerHash, true) + _, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) if err != nil { return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") } @@ -206,14 +200,14 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas // is also not valid. if parentBlockNode.IsHeaderValidateFailed() { return nil, false, bc.storeValidateFailedHeaderInBlockIndexWithWrapperError( - header, errors.New("validateAndIndexHeaderPoS: Parent header failed validations"), + header, headerHash, errors.New("validateAndIndexHeaderPoS: Parent header failed validations"), ) } // Verify that the header is properly formed. if err := bc.isValidBlockHeaderPoS(header); err != nil { return nil, false, bc.storeValidateFailedHeaderInBlockIndexWithWrapperError( - header, errors.New("validateAndIndexHeaderPoS: Header failed validations"), + header, headerHash, errors.New("validateAndIndexHeaderPoS: Header failed validations"), ) } @@ -225,13 +219,13 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas } if !isValidRandomSeedSignature { return nil, false, bc.storeValidateFailedHeaderInBlockIndexWithWrapperError( - header, errors.New("validateAndIndexHeaderPoS: Header has invalid random seed signature"), + header, headerHash, errors.New("validateAndIndexHeaderPoS: Header has invalid random seed signature"), ) } } // Store it as HeaderValidated now that it has passed all validations. - blockNode, err = bc.storeValidatedHeaderInBlockIndex(header) + blockNode, err = bc.storeValidatedHeaderInBlockIndex(header, headerHash) if err != nil { return nil, false, errors.Wrapf(err, "validateAndIndexHeaderPoS: Problem adding header to block index: ") } @@ -283,14 +277,15 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v } // If we can't hash the block, we can never store in the block index and we should throw it out immediately. - if _, err := block.Hash(); err != nil { + blockHash, err := block.Hash() + if err != nil { return false, false, nil, errors.Wrapf(err, "processBlockPoS: Problem hashing block") } // In hypersync archival mode, we may receive blocks that have already been processed and committed during state // synchronization. However, we may want to store these blocks in the db for archival purposes. We check if the // block we're dealing with is an archival block. If it is, we store it and return early. - if success, err := bc.checkAndStoreArchivalBlock(block); err != nil { + if success, err := bc.checkAndStoreArchivalBlock(block, blockHash); err != nil { return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem checking and storing archival block") } else if success { return true, false, nil, nil @@ -335,7 +330,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v parentUtxoView := parentUtxoViewAndUtxoOps.UtxoView // First, we perform a validation of the leader and the QC to prevent spam. // If the block fails this check, we throw it away. - passedSpamPreventionCheck, err := bc.validateLeaderAndQC(block, parentUtxoView, verifySignatures) + passedSpamPreventionCheck, err := bc.validateLeaderAndQC(block, blockHash, parentUtxoView, verifySignatures) if err != nil { // If we hit an error, we can't store it since we're not sure if it passed the spam prevention check. return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem validating leader and QC") @@ -346,7 +341,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v } // Validate the block and store it in the block index. The block is guaranteed to not be an orphan. - blockNode, err := bc.validateAndIndexBlockPoS(block, parentUtxoView, verifySignatures) + blockNode, err := bc.validateAndIndexBlockPoS(block, blockHash, parentUtxoView, verifySignatures) if err != nil { return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem validating block: ") @@ -369,7 +364,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // header and applying it to the header chain will result in the two chains being out of // sync. The header chain is less critical and mutations to it are reversible. So we attempt // to mutate it first before attempting to mutate the block chain. - if _, _, err = bc.processHeaderPoS(block.Header, verifySignatures); err != nil { + if _, _, err = bc.processHeaderPoS(block.Header, blockHash, verifySignatures); err != nil { return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem processing header") } @@ -559,31 +554,29 @@ func (bc *Blockchain) processOrphanBlockPoS(block *MsgDeSoBlock) error { // As a spam-prevention measure, we just throw away this block and don't store it. return nil } + + blockHash, err := block.Header.Hash() if err != nil { - return errors.Wrap(err, "processOrphanBlockPoS: Problem getting snapshot global params") + return errors.Wrap(err, "processOrphanBlockPoS: Problem hashing block") } + // All blocks should pass the basic integrity validations, which ensure the block // is not malformed. If the block is malformed, we should store it as ValidateFailed. if err = bc.isProperlyFormedBlockPoS(block); err != nil { - if _, innerErr := bc.storeValidateFailedBlockInBlockIndex(block); innerErr != nil { + if _, innerErr := bc.storeValidateFailedBlockInBlockIndex(block, blockHash); innerErr != nil { return errors.Wrapf(innerErr, "processOrphanBlockPoS: Problem adding validate failed block to block index: %v", err) } return nil } // Add to blockIndexByHash with status STORED only as we are not sure if it's valid yet. - _, err = bc.storeBlockInBlockIndex(block) + _, err = bc.storeBlockInBlockIndex(block, blockHash) return errors.Wrap(err, "processBlockPoS: Problem adding block to block index: ") } // checkAndStoreArchivalBlock is a helper function that takes in a block and checks if it's an archival block. // If it is, it stores the block in the db and returns true. If it's not, it returns false, or false and an error. -func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock) (_success bool, _err error) { - // First, get the block hash and lookup the block index. - blockHash, err := block.Hash() - if err != nil { - return false, errors.Wrap(err, "checkAndStoreArchivalBlock: Problem hashing block") - } +func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock, blockHash *BlockHash) (_success bool, _err error) { blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, block.Header.Height) // If the blockNode doesn't exist, or the block is not committed, or it's already stored, then we're not dealing // with an archival block. Archival blocks must have an existing blockNode, be committed, and not be stored. @@ -593,8 +586,7 @@ func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock) (_success // If we get to this point, we're dealing with an archival block, so we'll attempt to store it. // This means, this block node is already marked as COMMITTED and VALIDATED, and we just need to store it. - _, err = bc.storeBlockInBlockIndex(block) - if err != nil { + if _, err := bc.storeBlockInBlockIndex(block, blockHash); err != nil { return false, errors.Wrap(err, "checkAndStoreArchivalBlock: Problem storing block in block index") } return true, nil @@ -602,9 +594,9 @@ func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock) (_success // storeValidateFailedBlockWithWrappedError is a helper function that takes in a block and an error and // stores the block in the block index with status VALIDATE_FAILED. It returns the resulting BlockNode. -func (bc *Blockchain) storeValidateFailedBlockWithWrappedError(block *MsgDeSoBlock, outerErr error) ( +func (bc *Blockchain) storeValidateFailedBlockWithWrappedError(block *MsgDeSoBlock, hash *BlockHash, outerErr error) ( *BlockNode, error) { - blockNode, innerErr := bc.storeValidateFailedBlockInBlockIndex(block) + blockNode, innerErr := bc.storeValidateFailedBlockInBlockIndex(block, hash) if innerErr != nil { return nil, errors.Wrapf(innerErr, "storeValidateFailedBlockWithWrappedError: Problem adding validate failed block to block index: %v", @@ -615,6 +607,7 @@ func (bc *Blockchain) storeValidateFailedBlockWithWrappedError(block *MsgDeSoBlo func (bc *Blockchain) validateLeaderAndQC( block *MsgDeSoBlock, + blockHash *BlockHash, parentUtxoView *UtxoView, verifySignatures bool, ) (_passedSpamPreventionCheck bool, _err error) { @@ -636,7 +629,7 @@ func (bc *Blockchain) validateLeaderAndQC( "validateLeaderAndQC: Problem getting snapshot epoch number for epoch #%d", currentEpochEntry.EpochNumber) } - isValidPartialSig, err := parentUtxoView.hasValidProposerPartialSignaturePoS(block, snapshotAtEpochNumber) + isValidPartialSig, err := parentUtxoView.hasValidProposerPartialSignaturePoS(block, blockHash, snapshotAtEpochNumber) if err != nil { return false, errors.Wrap(err, "validateLeaderAndQC: Problem validating proposer partial sig") @@ -694,12 +687,12 @@ func (bc *Blockchain) validateLeaderAndQC( // return the new BlockNode. // - Error case: Something goes wrong that doesn't result in the block being marked VALIDATE or VALIDATE_FAILED. In // this case, we will add the block to the block index with status STORED and return the BlockNode. -func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoView *UtxoView, verifySignatures bool) ( - *BlockNode, error) { - blockHash, err := block.Header.Hash() - if err != nil { - return nil, errors.Wrapf(err, "validateAndIndexBlockPoS: Problem hashing block %v", block) - } +func (bc *Blockchain) validateAndIndexBlockPoS( + block *MsgDeSoBlock, + blockHash *BlockHash, + parentUtxoView *UtxoView, + verifySignatures bool, +) (*BlockNode, error) { // Base case - Check if the block is validated or validate failed. If so, we can return early. // TODO: validate height doesn't overflow uint32 @@ -733,13 +726,13 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi // this block as ValidateFailed. If the parent is not ValidateFailed, we ONLY store the block and move on. // We don't want to store it as ValidateFailed because we don't know if it's actually invalid. if parentBlockNode.IsValidateFailed() { - return bc.storeValidateFailedBlockWithWrappedError(block, errors.New("parent block is ValidateFailed")) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, errors.New("parent block is ValidateFailed")) } // If the parent block still has a Stored status, it means that we weren't able to validate it // despite trying. The current block will also be stored as a Stored block. if !parentBlockNode.IsValidated() { - return bc.storeBlockInBlockIndex(block) + return bc.storeBlockInBlockIndex(block, blockHash) } // Validate the block's random seed signature @@ -747,14 +740,14 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi isValidRandomSeedSignature, err := bc.hasValidProposerRandomSeedSignaturePoS(block.Header) if err != nil { var innerErr error - blockNode, innerErr = bc.storeBlockInBlockIndex(block) + blockNode, innerErr = bc.storeBlockInBlockIndex(block, blockHash) if innerErr != nil { return nil, errors.Wrapf(innerErr, "validateAndIndexBlockPoS: Problem adding block to block index: %v", err) } return blockNode, errors.Wrap(err, "validateAndIndexBlockPoS: Problem validating random seed signature") } if !isValidRandomSeedSignature { - return bc.storeValidateFailedBlockWithWrappedError(block, errors.New("invalid random seed signature")) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, errors.New("invalid random seed signature")) } } @@ -762,15 +755,15 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi serializedBlock, err := block.ToBytes(false) if err != nil { return bc.storeValidateFailedBlockWithWrappedError( - block, errors.Wrap(err, "validateAndIndexBlockPoS: Problem serializing block")) + block, blockHash, errors.Wrap(err, "validateAndIndexBlockPoS: Problem serializing block")) } if uint64(len(serializedBlock)) > parentUtxoView.GetCurrentGlobalParamsEntry().MaxBlockSizeBytesPoS { - return bc.storeValidateFailedBlockWithWrappedError(block, RuleErrorBlockTooBig) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, RuleErrorBlockTooBig) } // Check if the block is properly formed and passes all basic validations. if err = bc.isValidBlockPoS(block); err != nil { - return bc.storeValidateFailedBlockWithWrappedError(block, err) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, err) } // Connect this block to the parent block's UtxoView. @@ -781,7 +774,7 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi // If we fail to connect the block, then it means the block is invalid. We should store it as ValidateFailed. if _, err = parentUtxoView.ConnectBlock(block, txHashes, verifySignatures, nil, block.Header.Height); err != nil { // If it doesn't connect, we want to mark it as ValidateFailed. - return bc.storeValidateFailedBlockWithWrappedError(block, err) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, err) } // If the block is too far in the future, we leave it as STORED and return early. @@ -790,11 +783,11 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi return blockNode, errors.Wrap(err, "validateAndIndexBlockPoS: Problem checking block timestamp") } if failsTimestampDriftCheck { - return bc.storeBlockInBlockIndex(block) + return bc.storeBlockInBlockIndex(block, blockHash) } // We can now add this block to the block index since we have performed all basic validations. - blockNode, err = bc.storeValidatedBlockInBlockIndex(block) + blockNode, err = bc.storeValidatedBlockInBlockIndex(block, blockHash) if err != nil { return blockNode, errors.Wrap(err, "validateAndIndexBlockPoS: Problem adding block to block index: ") } @@ -840,7 +833,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( parentUtxoView := parentUtxoViewAndUtxoOps.UtxoView // If the block isn't validated or validate failed, we need to run the anti-spam checks on it. - passedSpamPreventionCheck, err := bc.validateLeaderAndQC(block, parentUtxoView, verifySignatures) + passedSpamPreventionCheck, err := bc.validateLeaderAndQC(block, blockHash, parentUtxoView, verifySignatures) if err != nil { // If we hit an error, that means there was an intermittent issue when trying to // validate the QC or the leader. @@ -849,7 +842,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( if !passedSpamPreventionCheck { // If the QC or Leader check failed, we'll never accept this block, but we've already stored it, // so we need to mark it as ValidateFailed. - blockNode, err = bc.storeValidateFailedBlockInBlockIndex(block) + blockNode, err = bc.storeValidateFailedBlockInBlockIndex(block, blockHash) if err != nil { return nil, errors.Wrap(err, "validatePreviouslyIndexedBlockPoS: Problem adding validate failed block to block index") @@ -858,7 +851,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( } // We run the full validation algorithm on the block. - return bc.validateAndIndexBlockPoS(block, parentUtxoView, verifySignatures) + return bc.validateAndIndexBlockPoS(block, blockHash, parentUtxoView, verifySignatures) } // isValidBlockPoS performs all basic block integrity checks. Any error @@ -1132,8 +1125,20 @@ func (bc *Blockchain) hasValidProposerRandomSeedSignaturePoS(header *MsgDeSoHead return isVerified, nil } -func (bav *UtxoView) hasValidProposerPartialSignaturePoS(block *MsgDeSoBlock, snapshotAtEpochNumber uint64) ( - bool, error) { +func (bav *UtxoView) hasValidProposerPartialSignaturePoS( + block *MsgDeSoBlock, + blockHash *BlockHash, + snapshotAtEpochNumber uint64, +) (bool, error) { + // If we aren't provided a hash, we can just compute it on the fly. + // It's more efficient for us not to recompute the hash though, so we only do it if we have to. + if blockHash == nil { + var err error + blockHash, err = block.Hash() + if err != nil { + return false, errors.Wrapf(err, "hasValidProposerPartialSignaturePoS: Problem hashing block") + } + } votingPublicKey := block.Header.ProposerVotingPublicKey proposerPartialSig := block.Header.ProposerVotePartialSignature // If the proposer partial sig is nil, we can't validate it. That's an error. @@ -1157,11 +1162,6 @@ func (bav *UtxoView) hasValidProposerPartialSignaturePoS(block *MsgDeSoBlock, sn if !snapshotBlockProposerValidatorEntry.VotingPublicKey.Eq(votingPublicKey) { return false, nil } - // Get the block's hash - blockHash, err := block.Header.Hash() - if err != nil { - return false, errors.Wrapf(err, "hasValidProposerPartialSignaturePoS: Problem hashing block") - } // Now that we have the snapshot validator entry and validated that the // voting public key from this block's header matches the snapshotted // voting public key, we can validate the partial sig. @@ -1380,10 +1380,15 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( // getOrCreateBlockNodeFromBlockIndex returns the block node from the block index if it exists. // Otherwise, it creates a new block node and adds it to the blockIndexByHash and blockIndexByHeight. -func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock) (*BlockNode, error) { - hash, err := block.Header.Hash() - if err != nil { - return nil, errors.Wrapf(err, "getOrCreateBlockNodeFromBlockIndex: Problem hashing block %v", block) +func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock, hash *BlockHash) (*BlockNode, error) { + // If we aren't provided a hash, we can just compute it on the fly. + // It's more efficient for us not to recompute the hash though, so we only do it if we have to. + if hash == nil { + var err error + hash, err = block.Hash() + if err != nil { + return nil, errors.Wrapf(err, "storeBlockInBlockIndex: Problem hashing block") + } } blockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(hash, block.Header.Height) prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) @@ -1401,8 +1406,8 @@ func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock) (* // storeBlockInBlockIndex upserts the blocks into the in-memory block index & badger and updates its status to // StatusBlockStored. It also writes the block to the block index in badger -func (bc *Blockchain) storeValidatedHeaderInBlockIndex(header *MsgDeSoHeader) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(&MsgDeSoBlock{Header: header}) +func (bc *Blockchain) storeValidatedHeaderInBlockIndex(header *MsgDeSoHeader, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(&MsgDeSoBlock{Header: header}, hash) if err != nil { return nil, errors.Wrapf(err, "storeValidatedHeaderInBlockIndex: Problem getting or creating block node") } @@ -1420,8 +1425,8 @@ func (bc *Blockchain) storeValidatedHeaderInBlockIndex(header *MsgDeSoHeader) (* return blockNode, nil } -func (bc *Blockchain) storeValidateFailedHeaderInBlockIndexWithWrapperError(header *MsgDeSoHeader, wrapperError error) error { - if _, innerErr := bc.storeValidateFailedHeaderInBlockIndex(header); innerErr != nil { +func (bc *Blockchain) storeValidateFailedHeaderInBlockIndexWithWrapperError(header *MsgDeSoHeader, hash *BlockHash, wrapperError error) error { + if _, innerErr := bc.storeValidateFailedHeaderInBlockIndex(header, hash); innerErr != nil { return errors.Wrapf(innerErr, "%v", wrapperError) } return wrapperError @@ -1429,8 +1434,8 @@ func (bc *Blockchain) storeValidateFailedHeaderInBlockIndexWithWrapperError(head // storeValidateFailedHeaderInBlockIndex stores the header in the block index only and sets its status to // StatusHeaderValidateFailed. It does not write the header to the DB. -func (bc *Blockchain) storeValidateFailedHeaderInBlockIndex(header *MsgDeSoHeader) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(&MsgDeSoBlock{Header: header}) +func (bc *Blockchain) storeValidateFailedHeaderInBlockIndex(header *MsgDeSoHeader, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(&MsgDeSoBlock{Header: header}, hash) if err != nil { return nil, errors.Wrapf(err, "storeValidateFailedHeaderInBlockIndex: Problem getting or creating block node") } @@ -1451,8 +1456,8 @@ func (bc *Blockchain) storeValidateFailedHeaderInBlockIndex(header *MsgDeSoHeade // storeBlockInBlockIndex upserts the blocks into the in-memory block index & badger and updates its status to // StatusBlockStored. It also writes the block to the block index in badger // by calling upsertBlockAndBlockNodeToDB. -func (bc *Blockchain) storeBlockInBlockIndex(block *MsgDeSoBlock) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block) +func (bc *Blockchain) storeBlockInBlockIndex(block *MsgDeSoBlock, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block, hash) if err != nil { return nil, errors.Wrapf(err, "storeBlockInBlockIndex: Problem getting or creating block node") } @@ -1472,8 +1477,8 @@ func (bc *Blockchain) storeBlockInBlockIndex(block *MsgDeSoBlock) (*BlockNode, e // status to StatusBlockValidated. If it does not have the status StatusBlockStored already, we add that as we // will store the block in the DB after updating its status. It also writes the block to the block index in // badger by calling upsertBlockAndBlockNodeToDB. -func (bc *Blockchain) storeValidatedBlockInBlockIndex(block *MsgDeSoBlock) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block) +func (bc *Blockchain) storeValidatedBlockInBlockIndex(block *MsgDeSoBlock, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block, hash) if err != nil { return nil, errors.Wrapf(err, "storeValidatedBlockInBlockIndex: Problem getting or creating block node") } @@ -1502,8 +1507,8 @@ func (bc *Blockchain) storeValidatedBlockInBlockIndex(block *MsgDeSoBlock) (*Blo // status to StatusBlockValidateFailed. If it does not have the status StatusBlockStored already, we add that as we // will store the block in the DB after updating its status. It also writes the block to the block index in badger // by calling upsertBlockAndBlockNodeToDB. -func (bc *Blockchain) storeValidateFailedBlockInBlockIndex(block *MsgDeSoBlock) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block) +func (bc *Blockchain) storeValidateFailedBlockInBlockIndex(block *MsgDeSoBlock, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block, hash) if err != nil { return nil, errors.Wrapf(err, "storeValidateFailedBlockInBlockIndex: Problem getting or creating block node") } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 291a34eab..053d33bac 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -374,10 +374,10 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { }, }, } - blockNode, err := bc.storeBlockInBlockIndex(block) - require.NoError(t, err) newHash, err := block.Hash() require.NoError(t, err) + blockNode, err := bc.storeBlockInBlockIndex(block, newHash) + require.NoError(t, err) // Check the block index by hash blockNodeFromIndex, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(newHash, uint64(blockNode.Height)) require.True(t, exists) @@ -400,7 +400,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { require.NoError(t, err) require.True(t, bytes.Equal(uncommittedBytes, origBlockBytes)) // Okay now we update the status of the block to include validated. - blockNode, err = bc.storeValidatedBlockInBlockIndex(block) + blockNode, err = bc.storeValidatedBlockInBlockIndex(block, newHash) require.NoError(t, err) blockNodeFromIndex, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(newHash, uncommittedBlock.Header.Height) require.True(t, exists) @@ -425,7 +425,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { require.False(t, updatedBlockHash.IsEqual(newHash)) // Okay now put this new block in there. - blockNode, err = bc.storeBlockInBlockIndex(block) + blockNode, err = bc.storeBlockInBlockIndex(block, updatedBlockHash) require.NoError(t, err) // Make sure the blockIndexByHash is correct. updatedBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(updatedBlockHash, uint64(blockNode.Height)) @@ -446,7 +446,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { // If we're missing a field in the header, we should get an error // as we can't compute the hash. block.Header.ProposerVotingPublicKey = nil - _, err = bc.storeBlockInBlockIndex(block) + _, err = bc.storeBlockInBlockIndex(block, nil) require.Error(t, err) } @@ -2008,7 +2008,7 @@ func TestGetSafeBlocks(t *testing.T) { block1Hash, err := block1.Hash() require.NoError(t, err) // Add block 1 w/ stored and validated - bn1, err := testMeta.chain.storeValidatedBlockInBlockIndex(block1) + bn1, err := testMeta.chain.storeValidatedBlockInBlockIndex(block1, nil) require.NoError(t, err) require.True(t, bn1.Hash.IsEqual(block1Hash)) // Create block 2 w/ block 1 as parent and add it to the block index w/ stored & validated @@ -2016,13 +2016,13 @@ func TestGetSafeBlocks(t *testing.T) { block2 = _generateRealBlock(testMeta, uint64(testMeta.savedHeight+1), uint64(testMeta.savedHeight+1), 1293, block1Hash, false) block2Hash, err := block2.Hash() require.NoError(t, err) - bn2, err := testMeta.chain.storeValidatedBlockInBlockIndex(block2) + bn2, err := testMeta.chain.storeValidatedBlockInBlockIndex(block2, nil) require.NoError(t, err) require.True(t, bn2.Hash.IsEqual(block2Hash)) // Add block 3 only as stored and validated var block3 *MsgDeSoBlock block3 = _generateRealBlock(testMeta, uint64(testMeta.savedHeight+2), uint64(testMeta.savedHeight+2), 1372, block2Hash, false) - bn3, err := testMeta.chain.storeValidatedBlockInBlockIndex(block3) + bn3, err := testMeta.chain.storeValidatedBlockInBlockIndex(block3, nil) require.NoError(t, err) block3Hash, err := block3.Hash() require.NoError(t, err) @@ -2030,7 +2030,7 @@ func TestGetSafeBlocks(t *testing.T) { // Add block 3' only as stored var block3Prime *MsgDeSoBlock block3Prime = _generateRealBlock(testMeta, uint64(testMeta.savedHeight+2), uint64(testMeta.savedHeight+3), 137175, block2Hash, false) - bn3Prime, err := testMeta.chain.storeBlockInBlockIndex(block3Prime) + bn3Prime, err := testMeta.chain.storeBlockInBlockIndex(block3Prime, nil) require.NoError(t, err) block3PrimeHash, err := block3Prime.Hash() require.NoError(t, err) @@ -2041,7 +2041,7 @@ func TestGetSafeBlocks(t *testing.T) { block5.Header.Height = uint64(testMeta.savedHeight + 5) block5Hash, err := block5.Hash() require.NoError(t, err) - _, err = testMeta.chain.storeValidatedBlockInBlockIndex(block5) + _, err = testMeta.chain.storeValidatedBlockInBlockIndex(block5, nil) require.NoError(t, err) // Okay let's get the safe blocks. safeBlocks, err := testMeta.chain.GetSafeBlocks() @@ -2062,7 +2062,7 @@ func TestGetSafeBlocks(t *testing.T) { require.False(t, _checkSafeBlocksForBlockHash(block5Hash, safeBlocks)) // Update block 3 prime to be validated and it should now be a safe block. - bn3Prime, err = testMeta.chain.storeValidatedBlockInBlockIndex(block3Prime) + bn3Prime, err = testMeta.chain.storeValidatedBlockInBlockIndex(block3Prime, nil) require.NoError(t, err) require.True(t, bn3Prime.IsValidated()) safeBlocks, err = testMeta.chain.GetSafeBlocks() @@ -2345,7 +2345,7 @@ func TestHasValidProposerPartialSignaturePoS(t *testing.T) { utxoView := _newUtxoView(testMeta) snapshotEpochNumber, err := utxoView.GetCurrentSnapshotEpochNumber() require.NoError(t, err) - isValid, err := utxoView.hasValidProposerPartialSignaturePoS(realBlock, snapshotEpochNumber) + isValid, err := utxoView.hasValidProposerPartialSignaturePoS(realBlock, nil, snapshotEpochNumber) require.NoError(t, err) require.True(t, isValid) @@ -2353,7 +2353,7 @@ func TestHasValidProposerPartialSignaturePoS(t *testing.T) { realVotingPublicKey := realBlock.Header.ProposerVotingPublicKey { realBlock.Header.ProposerVotingPublicKey = _generateRandomBLSPrivateKey(t).PublicKey() - isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, snapshotEpochNumber) + isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, nil, snapshotEpochNumber) require.NoError(t, err) require.False(t, isValid) // Reset the proposer voting public key @@ -2365,7 +2365,7 @@ func TestHasValidProposerPartialSignaturePoS(t *testing.T) { incorrectPayload := consensus.GetVoteSignaturePayload(13, testMeta.chain.BlockTip().Hash) realBlock.Header.ProposerVotePartialSignature, err = testMeta.blsPubKeyToBLSKeyMap[realBlock.Header.ProposerVotingPublicKey.ToString()].Sign(incorrectPayload[:]) - isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, snapshotEpochNumber) + isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, nil, snapshotEpochNumber) require.NoError(t, err) require.False(t, isValid) } @@ -2378,7 +2378,7 @@ func TestHasValidProposerPartialSignaturePoS(t *testing.T) { correctPayload := consensus.GetVoteSignaturePayload(12, realBlockHash) wrongPrivateKey := _generateRandomBLSPrivateKey(t) realBlock.Header.ProposerVotePartialSignature, err = wrongPrivateKey.Sign(correctPayload[:]) - isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, snapshotEpochNumber) + isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, nil, snapshotEpochNumber) require.NoError(t, err) require.False(t, isValid) } @@ -2543,7 +2543,7 @@ func _generateDummyBlock(testMeta *TestMeta, blockHeight uint64, view uint64, se require.NoError(testMeta.t, err) // Add block to block index. - blockNode, err := testMeta.chain.storeBlockInBlockIndex(msgDesoBlock) + blockNode, err := testMeta.chain.storeBlockInBlockIndex(msgDesoBlock, nil) require.NoError(testMeta.t, err) require.True(testMeta.t, blockNode.IsStored()) _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(newBlockHash, msgDesoBlock.Header.Height) @@ -2568,7 +2568,7 @@ func _generateBlockAndAddToBestChain(testMeta *TestMeta, blockHeight uint64, vie newBlockHash, err := msgDesoBlock.Hash() require.NoError(testMeta.t, err) // Add block to block index. - blockNode, err := testMeta.chain.storeValidatedBlockInBlockIndex(msgDesoBlock) + blockNode, err := testMeta.chain.storeValidatedBlockInBlockIndex(msgDesoBlock, nil) require.NoError(testMeta.t, err) require.True(testMeta.t, blockNode.IsStored()) require.True(testMeta.t, blockNode.IsValidated()) From 23991c4c60e132d497de045b7e784ed5de428985 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 22:45:40 -0400 Subject: [PATCH 28/62] upgrade cloudflare/circl --- go.mod | 10 +++++----- go.sum | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 02c87cd5f..eeec46a9b 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 github.com/bxcodec/faker v2.0.1+incompatible - github.com/cloudflare/circl v1.1.0 + github.com/cloudflare/circl v1.4.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 github.com/decred/dcrd/lru v1.1.3 @@ -37,7 +37,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.0.2 github.com/unrolled/secure v1.0.8 - golang.org/x/crypto v0.27.0 + golang.org/x/crypto v0.28.0 golang.org/x/sync v0.8.0 gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 ) @@ -59,7 +59,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect - github.com/bwesterb/go-ristretto v1.2.0 // indirect + github.com/bwesterb/go-ristretto v1.2.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect @@ -114,8 +114,8 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.25.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/go.sum b/go.sum index 5536e1ec2..da57a1eb9 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,7 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bwesterb/go-ristretto v1.2.0 h1:xxWOVbN5m8NNKiSDZXE1jtZvZnC6JSJ9cYFADiZcWtw= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -103,6 +104,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.4.0 h1:BV7h5MgrktNzytKmWjpOtdYrf0lkkbF8YMlBGPhJQrY= +github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -587,6 +590,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -715,6 +720,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -728,6 +735,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= From 8adc62459f8c3420d08da9acf99c91b68de7afa1 Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Mon, 7 Oct 2024 23:05:38 -0400 Subject: [PATCH 29/62] Ln/replace best chain plus replace old lru cache (#1414) * replace decred lru with hashicorp lru * fix usage of delete -> remove * fix calc next difficulty target issue * upgrade cloudflare/circl --- go.mod | 11 +++++------ go.sum | 11 +++++++++-- lib/block_view_test.go | 5 +++-- lib/blockchain.go | 36 +++++++++++++++++------------------- lib/connection_manager.go | 7 ++++--- lib/db_utils.go | 8 ++++---- lib/network_manager.go | 14 +++++++------- lib/peer.go | 14 +++++++------- lib/pos_blockchain.go | 19 +++++-------------- lib/pos_mempool.go | 16 ++++++++-------- lib/postgres.go | 6 +++--- lib/server.go | 23 ++++++++++------------- lib/snapshot.go | 14 ++++++++------ 13 files changed, 90 insertions(+), 94 deletions(-) diff --git a/go.mod b/go.mod index 02c87cd5f..e786aa20c 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,9 @@ require ( github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 github.com/bxcodec/faker v2.0.1+incompatible - github.com/cloudflare/circl v1.1.0 + github.com/cloudflare/circl v1.4.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 - github.com/decred/dcrd/lru v1.1.3 github.com/deso-protocol/go-deadlock v1.0.0 github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/dgraph-io/badger/v3 v3.2103.5 @@ -37,7 +36,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.0.2 github.com/unrolled/secure v1.0.8 - golang.org/x/crypto v0.27.0 + golang.org/x/crypto v0.28.0 golang.org/x/sync v0.8.0 gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 ) @@ -59,7 +58,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect - github.com/bwesterb/go-ristretto v1.2.0 // indirect + github.com/bwesterb/go-ristretto v1.2.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect @@ -114,8 +113,8 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.25.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/go.sum b/go.sum index 5536e1ec2..65e5bdded 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,7 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bwesterb/go-ristretto v1.2.0 h1:xxWOVbN5m8NNKiSDZXE1jtZvZnC6JSJ9cYFADiZcWtw= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -103,6 +104,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.4.0 h1:BV7h5MgrktNzytKmWjpOtdYrf0lkkbF8YMlBGPhJQrY= +github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -128,8 +131,6 @@ github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/decred/dcrd/lru v1.1.3 h1:w9EAbvGLyzm6jTjF83UKuqZEiUtJmvRhQDOCEIvSuE0= -github.com/decred/dcrd/lru v1.1.3/go.mod h1:Tw0i0pJyiLEx/oZdHLe1Wdv/Y7EGzAX+sYftnmxBR4o= github.com/deso-protocol/go-deadlock v1.0.0 h1:mw0pHy/19zgC+JFBStuQt1+1Ehv5OKA5NxXqecnL5ic= github.com/deso-protocol/go-deadlock v1.0.0/go.mod h1:K0Wd2OV2x7ck7SMYDraWerpKjFKUeBqaFcwz21tmkb8= github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+LsH/kRdL5l/Yzvg= @@ -587,6 +588,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -715,6 +718,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -728,6 +733,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= diff --git a/lib/block_view_test.go b/lib/block_view_test.go index ff17677a4..903f4ed4f 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -13,10 +13,10 @@ import ( "github.com/deso-protocol/core/bls" "github.com/btcsuite/btcd/btcec" - "github.com/decred/dcrd/lru" "github.com/dgraph-io/badger/v3" embeddedpostgres "github.com/fergusstrange/embedded-postgres" "github.com/golang/glog" + "github.com/hashicorp/golang-lru/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -797,7 +797,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. if tm.chain.snapshot != nil { - tm.chain.snapshot.DatabaseCache = lru.NewKVCache(DatabaseCacheSize) + tm.chain.snapshot.DatabaseCache, err = lru.New[string, []byte](int(DatabaseCacheSize)) + require.NoError(err) } // Note that unlike connecting test vectors, when disconnecting, we don't need to verify db entries. diff --git a/lib/blockchain.go b/lib/blockchain.go index 3ecdd2b5e..06ef50632 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -16,9 +16,7 @@ import ( "sync" "time" - "github.com/decred/dcrd/lru" - - lru2 "github.com/hashicorp/golang-lru/v2" + "github.com/hashicorp/golang-lru/v2" "github.com/deso-protocol/core/collections" @@ -567,15 +565,15 @@ type CheckpointBlockInfoAndError struct { type BlockIndex struct { db *badger.DB snapshot *Snapshot - blockIndexByHash *lru2.Cache[BlockHash, *BlockNode] - blockIndexByHeight *lru2.Cache[uint64, []*BlockNode] + blockIndexByHash *lru.Cache[BlockHash, *BlockNode] + blockIndexByHeight *lru.Cache[uint64, []*BlockNode] tip *BlockNode headerTip *BlockNode } func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { - blockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? - blockIndexByHeight, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + blockIndexByHash, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + blockIndexByHeight, _ := lru.New[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? return &BlockIndex{ db: db, snapshot: snapshot, @@ -586,8 +584,8 @@ func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *Block } func (bi *BlockIndex) setBlockIndexFromMap(input map[BlockHash]*BlockNode) { - newHashToBlockNodeMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - newHeightToBlockNodeMap, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) + newHashToBlockNodeMap, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + newHeightToBlockNodeMap, _ := lru.New[uint64, []*BlockNode](MaxBlockIndexNodes) bi.blockIndexByHash = newHashToBlockNodeMap bi.blockIndexByHeight = newHeightToBlockNodeMap for _, val := range input { @@ -740,7 +738,7 @@ type Blockchain struct { blockView *UtxoView // cache block view for each block - blockViewCache lru.KVCache + blockViewCache *lru.Cache[BlockHash, *BlockViewAndUtxoOps] // snapshot cache snapshotCache *SnapshotCache @@ -872,9 +870,9 @@ func (bc *Blockchain) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { } func (bc *Blockchain) CopyBlockIndexes() ( - _blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], + _blockIndexByHash *lru.Cache[BlockHash, *BlockNode], ) { - newBlockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBlockIndexByHash, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, key := range bc.blockIndex.blockIndexByHash.Keys() { val, _ := bc.blockIndex.blockIndexByHash.Get(key) newBlockIndexByHash.Add(key, val) @@ -1124,7 +1122,7 @@ func NewBlockchain( timer := &Timer{} timer.Initialize() - + blockViewCache, _ := lru.New[BlockHash, *BlockViewAndUtxoOps](100) // TODO: parameterize bc := &Blockchain{ db: db, postgres: postgres, @@ -1138,7 +1136,7 @@ func NewBlockchain( archivalMode: archivalMode, blockIndex: NewBlockIndex(db, snapshot, nil), // TODO: replace with actual tip. - blockViewCache: lru.NewKVCache(100), // TODO: parameterize + blockViewCache: blockViewCache, snapshotCache: NewSnapshotCache(), checkpointSyncingProviders: checkpointSyncingProviders, @@ -1953,7 +1951,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChai } // TODO: need to figure out how to handle this for exchange api tests. -func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { +func (bc *Blockchain) setBestChain(bestChain []*BlockNode) { for _, blockNode := range bestChain { bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) if bc.blockIndex.GetTip() == nil { @@ -1964,8 +1962,8 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { } } -func (bc *Blockchain) SetBestChainMap( - blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], +func (bc *Blockchain) setBestChainMap( + blockIndexByHash *lru.Cache[BlockHash, *BlockNode], tipNode *BlockNode, ) { bc.blockIndex.blockIndexByHash = blockIndexByHash @@ -2294,8 +2292,8 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( return commonAncestor, detachBlocks, attachBlocks } -func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Cache[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( - chainList []*BlockNode, chainMap *lru2.Cache[BlockHash, *BlockNode]) { +func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru.Cache[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( + chainList []*BlockNode, chainMap *lru.Cache[BlockHash, *BlockNode]) { // Remove the nodes we detached from the end of the best chain node list. tipIndex := len(mainChainList) - 1 diff --git a/lib/connection_manager.go b/lib/connection_manager.go index 40eaf2fc7..d8b060e71 100644 --- a/lib/connection_manager.go +++ b/lib/connection_manager.go @@ -10,8 +10,8 @@ import ( "github.com/btcsuite/btcd/addrmgr" "github.com/btcsuite/btcd/wire" - "github.com/decred/dcrd/lru" "github.com/golang/glog" + "github.com/hashicorp/golang-lru/v2" ) // connection_manager.go contains most of the logic for creating and managing @@ -52,7 +52,7 @@ type ConnectionManager struct { // Keep track of the nonces we've sent in our version messages so // we can prevent connections to ourselves. - sentNonces lru.Cache + sentNonces *lru.Cache[uint64, struct{}] // This section defines the data structures for storing all the // peers we're aware of. @@ -125,13 +125,14 @@ func NewConnectionManager( ValidateHyperSyncFlags(_hyperSync, _syncType) + sentNoncesCache, _ := lru.New[uint64, struct{}](1000) return &ConnectionManager{ srv: _srv, params: _params, listeners: _listeners, // We keep track of the last N nonces we've sent in order to detect // self connections. - sentNonces: lru.NewCache(1000), + sentNonces: sentNoncesCache, //newestBlock: _newestBlock, // Initialize the peer data structures. diff --git a/lib/db_utils.go b/lib/db_utils.go index 1a231b3b4..345737f75 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -1183,8 +1183,8 @@ func DBGetWithTxn(txn *badger.Txn, snap *Snapshot, key []byte) ([]byte, error) { // Lookup the snapshot cache and check if we've already stored a value there. if isState { - if val, exists := snap.DatabaseCache.Lookup(keyString); exists { - return val.([]byte), nil + if val, exists := snap.DatabaseCache.Get(keyString); exists { + return val, nil } } @@ -1241,7 +1241,7 @@ func DBDeleteWithTxn(txn *badger.Txn, snap *Snapshot, key []byte, eventManager * return errors.Wrapf(err, "DBDeleteWithTxn: Problem preparing ancestral record") } // Now delete the past record from the cache. - snap.DatabaseCache.Delete(keyString) + snap.DatabaseCache.Remove(keyString) // We have to remove the previous value from the state checksum. // Because checksum is commutative, we can safely remove the past value here. if !snap.disableChecksum { @@ -2810,7 +2810,7 @@ func DBGetAccessGroupExistenceByAccessGroupIdWithTxn(txn *badger.Txn, snap *Snap // Lookup the snapshot cache and check if we've already stored a value there. if isState { - if _, exists := snap.DatabaseCache.Lookup(keyString); exists { + if exists := snap.DatabaseCache.Contains(keyString); exists { return true, nil } } diff --git a/lib/network_manager.go b/lib/network_manager.go index bbbaa7c19..ff7f8b8fa 100644 --- a/lib/network_manager.go +++ b/lib/network_manager.go @@ -11,11 +11,11 @@ import ( "github.com/btcsuite/btcd/addrmgr" "github.com/btcsuite/btcd/wire" - "github.com/decred/dcrd/lru" "github.com/deso-protocol/core/bls" "github.com/deso-protocol/core/collections" "github.com/deso-protocol/core/consensus" "github.com/golang/glog" + "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" ) @@ -69,7 +69,7 @@ type NetworkManager struct { NonValidatorInboundIndex *collections.ConcurrentMap[RemoteNodeId, *RemoteNode] // Cache of nonces used during handshake. - usedNonces lru.Cache + usedNonces *lru.Cache[uint64, struct{}] // The address manager keeps track of peer addresses we're aware of. When // we need to connect to a new outbound peer, it chooses one of the addresses @@ -121,7 +121,7 @@ func NewNetworkManager( minTxFeeRateNanosPerKB uint64, nodeServices ServiceFlag, ) *NetworkManager { - + usedNoncesCache, _ := lru.New[uint64, struct{}](1000) return &NetworkManager{ params: params, srv: srv, @@ -136,7 +136,7 @@ func NewNetworkManager( ValidatorOutboundIndex: collections.NewConcurrentMap[bls.SerializedPublicKey, *RemoteNode](), NonValidatorOutboundIndex: collections.NewConcurrentMap[RemoteNodeId, *RemoteNode](), NonValidatorInboundIndex: collections.NewConcurrentMap[RemoteNodeId, *RemoteNode](), - usedNonces: lru.NewCache(1000), + usedNonces: usedNoncesCache, connectIps: connectIps, persistentIpToRemoteNodeIdsMap: collections.NewConcurrentMap[string, RemoteNodeId](), activeValidatorsMap: collections.NewConcurrentMap[bls.SerializedPublicKey, consensus.Validator](), @@ -261,7 +261,7 @@ func (nm *NetworkManager) _handleVersionMessage(origin *Peer, desoMsg DeSoMessag // If we've seen this nonce before then return an error since this is a connection from ourselves. msgNonce := verMsg.Nonce if nm.usedNonces.Contains(msgNonce) { - nm.usedNonces.Delete(msgNonce) + nm.usedNonces.Remove(msgNonce) glog.Errorf("NetworkManager.handleVersionMessage: Disconnecting RemoteNode with id: (%v) "+ "nonce collision, nonce (%v)", origin.ID, msgNonce) nm.Disconnect(rn, "nonce collision") @@ -277,7 +277,7 @@ func (nm *NetworkManager) _handleVersionMessage(origin *Peer, desoMsg DeSoMessag return } - nm.usedNonces.Add(responseNonce) + nm.usedNonces.Add(responseNonce, struct{}{}) } // _handleVerackMessage is called when a new verack message is received. @@ -1248,7 +1248,7 @@ func (nm *NetworkManager) InitiateHandshake(rn *RemoteNode) { glog.Errorf("NetworkManager.InitiateHandshake: Error initiating handshake: %v", err) nm.Disconnect(rn, fmt.Sprintf("error initiating handshake: %v", err)) } - nm.usedNonces.Add(nonce) + nm.usedNonces.Add(nonce, struct{}{}) } // handleHandshakeComplete is called on a completed handshake with a RemoteNodes. diff --git a/lib/peer.go b/lib/peer.go index aece3eeff..73c4c4912 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "time" - "github.com/decred/dcrd/lru" + "github.com/hashicorp/golang-lru/v2" "github.com/btcsuite/btcd/wire" "github.com/deso-protocol/go-deadlock" @@ -111,7 +111,7 @@ type Peer struct { // Inventory stuff. // The inventory that we know the peer already has. - knownInventory lru.Cache + knownInventory *lru.Cache[InvVect, struct{}] // Whether the peer is ready to receive INV messages. For a peer that // still needs a mempool download, this is false. @@ -292,7 +292,7 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { for _, invVect := range msg.InvList { // No matter what, add the inv to the peer's known inventory. - pp.knownInventory.Add(*invVect) + pp.knownInventory.Add(*invVect, struct{}{}) // If this is a hash we are currently processing, no need to do anything. // This check serves to fill the gap between the time when we've decided @@ -344,7 +344,7 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { // If we made it here, it means the inventory was added to one of the // lists so mark it as processed on the Server. - pp.srv.inventoryBeingProcessed.Add(*invVect) + pp.srv.inventoryBeingProcessed.Add(*invVect, struct{}{}) } // If there were any transactions we don't yet have, request them using @@ -644,7 +644,7 @@ func NewPeer(_id uint64, _conn net.Conn, _isOutbound bool, _netAddr *wire.NetAdd _cmgr *ConnectionManager, _srv *Server, _syncType NodeSyncType, peerDisconnectedChan chan *Peer) *Peer { - + knownInventory, _ := lru.New[InvVect, struct{}](maxKnownInventory) pp := Peer{ ID: _id, cmgr: _cmgr, @@ -657,7 +657,7 @@ func NewPeer(_id uint64, _conn net.Conn, _isOutbound bool, _netAddr *wire.NetAdd outputQueueChan: make(chan DeSoMessage), peerDisconnectedChan: peerDisconnectedChan, quit: make(chan interface{}), - knownInventory: lru.NewCache(maxKnownInventory), + knownInventory: knownInventory, blocksToSend: make(map[BlockHash]bool), stallTimeoutSeconds: _stallTimeoutSeconds, minTxFeeRateNanosPerKB: _minFeeRateNanosPerKB, @@ -983,7 +983,7 @@ out: // Add the new inventory to the peer's knownInventory. for _, invVect := range invMsg.InvList { - pp.knownInventory.Add(*invVect) + pp.knownInventory.Add(*invVect, struct{}{}) } } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 2b0383f95..ff793e940 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1937,16 +1937,9 @@ func (bc *Blockchain) GetUncommittedTipView() (*UtxoView, error) { return blockViewAndUtxoOps.UtxoView, nil } -func (bc *Blockchain) getCachedBlockViewAndUtxoOps(blockHash BlockHash) (*BlockViewAndUtxoOps, error, bool) { - if viewAndUtxoOpsAtHashItem, exists := bc.blockViewCache.Lookup(blockHash); exists { - viewAndUtxoOpsAtHash, ok := viewAndUtxoOpsAtHashItem.(*BlockViewAndUtxoOps) - if !ok { - glog.Errorf("getCachedBlockViewAndUtxoOps: Problem casting to BlockViewAndUtxoOps") - return nil, fmt.Errorf("getCachedBlockViewAndUtxoOps: Problem casting to BlockViewAndUtxoOps"), false - } - return viewAndUtxoOpsAtHash, nil, true - } - return nil, nil, false +func (bc *Blockchain) getCachedBlockViewAndUtxoOps(blockHash BlockHash) (*BlockViewAndUtxoOps, bool) { + viewAndUtxoOpsAtHash, exists := bc.blockViewCache.Get(blockHash) + return viewAndUtxoOpsAtHash, exists } // getUtxoViewAndUtxoOpsAtBlockHash builds a UtxoView to the block provided and returns a BlockViewAndUtxoOps @@ -1993,10 +1986,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc "getUtxoViewAndUtxoOpsAtBlockHash: extends from a committed block that isn't the committed tip") } } - viewAndUtxoOpsAtHash, err, exists := bc.getCachedBlockViewAndUtxoOps(blockHash) - if err != nil { - return nil, errors.Wrapf(err, "getUtxoViewAndUtxoOpsAtBlockHash: Problem getting cached BlockViewAndUtxoOps") - } + viewAndUtxoOpsAtHash, exists := bc.getCachedBlockViewAndUtxoOps(blockHash) if exists { viewAndUtxoOpsCopy := viewAndUtxoOpsAtHash.Copy() return viewAndUtxoOpsCopy, nil @@ -2011,6 +2001,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc var fullBlock *MsgDeSoBlock for ii := len(uncommittedAncestors) - 1; ii >= 0; ii-- { glog.Infof("Connecting block %v", uncommittedAncestors[ii]) + var err error // We need to get these blocks from badger fullBlock, err = GetBlock(uncommittedAncestors[ii].Hash, bc.db, bc.snapshot) if err != nil { diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index ae8a2c94a..73c162b27 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "time" - "github.com/decred/dcrd/lru" + "github.com/hashicorp/golang-lru/v2" "github.com/dgraph-io/badger/v3" "github.com/golang/glog" @@ -183,11 +183,11 @@ type PosMempool struct { // recentBlockTxnCache is an LRU KV cache used to track the transaction that have been included in blocks. // This cache is used to power logic that waits for a transaction to either be validated in the mempool // or be included in a block. - recentBlockTxnCache lru.KVCache + recentBlockTxnCache *lru.Cache[BlockHash, struct{}] // recentRejectedTxnCache is a cache to store the txns that were recently rejected so that we can return better // errors for them. - recentRejectedTxnCache lru.KVCache + recentRejectedTxnCache *lru.Cache[BlockHash, error] } func NewPosMempool() *PosMempool { @@ -234,8 +234,8 @@ func (mp *PosMempool) Init( mp.mempoolBackupIntervalMillis = mempoolBackupIntervalMillis mp.maxValidationViewConnects = maxValidationViewConnects mp.transactionValidationRefreshIntervalMillis = transactionValidationRefreshIntervalMillis - mp.recentBlockTxnCache = lru.NewKVCache(100000) // cache 100K latest txns from blocks. - mp.recentRejectedTxnCache = lru.NewKVCache(100000) // cache 100K rejected txns. + mp.recentBlockTxnCache, _ = lru.New[BlockHash, struct{}](100000) // cache 100K latest txns from blocks. + mp.recentRejectedTxnCache, _ = lru.New[BlockHash, error](100000) // cache 100K rejected txns. // Recreate and initialize the transaction register and the nonce tracker. mp.txnRegister = NewTransactionRegister() @@ -486,11 +486,11 @@ func (mp *PosMempool) AddTransaction(txn *MsgDeSoTxn, txnTimestamp time.Time) er } func (mp *PosMempool) addTxnHashToRecentBlockCache(txnHash BlockHash) { - mp.recentBlockTxnCache.Add(txnHash, nil) + mp.recentBlockTxnCache.Add(txnHash, struct{}{}) } func (mp *PosMempool) deleteTxnHashFromRecentBlockCache(txnHash BlockHash) { - mp.recentBlockTxnCache.Delete(txnHash) + mp.recentBlockTxnCache.Remove(txnHash) } func (mp *PosMempool) isTxnHashInRecentBlockCache(txnHash BlockHash) bool { return mp.recentBlockTxnCache.Contains(txnHash) @@ -1023,7 +1023,7 @@ func (mp *PosMempool) WaitForTxnValidation(txHash *BlockHash) error { checkIntervalMillis = 1 } for { - rejectionErr, wasRejected := mp.recentRejectedTxnCache.Lookup(*txHash) + rejectionErr, wasRejected := mp.recentRejectedTxnCache.Get(*txHash) if wasRejected { return rejectionErr.(error) } diff --git a/lib/postgres.go b/lib/postgres.go index 08d078006..e919c8e78 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - lru2 "github.com/hashicorp/golang-lru/v2" + "github.com/hashicorp/golang-lru/v2" "net/url" "regexp" "strings" @@ -1305,14 +1305,14 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { } // GetBlockIndex gets all the PGBlocks and creates a map of BlockHash to BlockNode as needed by blockchain.go -func (postgres *Postgres) GetBlockIndex() (*lru2.Cache[BlockHash, *BlockNode], error) { +func (postgres *Postgres) GetBlockIndex() (*lru.Cache[BlockHash, *BlockNode], error) { var blocks []PGBlock err := postgres.db.Model(&blocks).Select() if err != nil { return nil, err } - blockMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + blockMap, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, block := range blocks { blockMap.Add(*block.Hash, &BlockNode{ Hash: block.Hash, diff --git a/lib/server.go b/lib/server.go index 28b225a48..8a9b17a21 100644 --- a/lib/server.go +++ b/lib/server.go @@ -12,20 +12,17 @@ import ( "sync/atomic" "time" - "github.com/btcsuite/btcd/wire" - "github.com/deso-protocol/core/collections" - "github.com/deso-protocol/core/consensus" - - "github.com/decred/dcrd/lru" - "github.com/DataDog/datadog-go/v5/statsd" - "github.com/btcsuite/btcd/addrmgr" chainlib "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/wire" "github.com/davecgh/go-spew/spew" + "github.com/deso-protocol/core/collections" + "github.com/deso-protocol/core/consensus" "github.com/deso-protocol/go-deadlock" "github.com/dgraph-io/badger/v3" "github.com/golang/glog" + "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" ) @@ -91,7 +88,7 @@ type Server struct { // adding it to this map and checking this map before replying will make it // so that we only send a reply to the first peer that sent us the inv, which // is more efficient. - inventoryBeingProcessed lru.Cache + inventoryBeingProcessed *lru.Cache[InvVect, struct{}] // hasRequestedSync indicates whether we've bootstrapped our mempool // by requesting all mempool transactions from a // peer. It's initially false @@ -229,7 +226,7 @@ func (srv *Server) _removeRequest(hash *BlockHash) { Type: InvTypeTx, Hash: *hash, } - srv.inventoryBeingProcessed.Delete(*invVect) + srv.inventoryBeingProcessed.Remove(*invVect) } // dataLock must be acquired for writing before calling this function. @@ -704,7 +701,7 @@ func NewServer( srv.blockProducer = _blockProducer srv.incomingMessages = _incomingMessages // Make this hold a multiple of what we hold for individual peers. - srv.inventoryBeingProcessed = lru.NewCache(maxKnownInventory) + srv.inventoryBeingProcessed, _ = lru.New[InvVect, struct{}](maxKnownInventory) srv.requestTimeoutSeconds = 10 srv.statsdClient = statsd @@ -1781,7 +1778,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { } // We also reset the in-memory snapshot cache, because it is populated with stale records after // we've initialized the chain with seed transactions. - srv.snapshot.DatabaseCache = lru.NewKVCache(DatabaseCacheSize) + srv.snapshot.DatabaseCache, _ = lru.New[string, []byte](int(DatabaseCacheSize)) // If we got here then we finished the snapshot sync so set appropriate flags. srv.blockchain.syncingState = false @@ -2067,8 +2064,8 @@ func (srv *Server) _relayTransactions() { // Add the transaction to the peer's known inventory. We do // it here when we enqueue the message to the peers outgoing - // message queue so that we don't have remember to do it later. - pp.knownInventory.Add(*invVect) + // message queue so that we don't have to remember to do it later. + pp.knownInventory.Add(*invVect, struct{}{}) invMsg.InvList = append(invMsg.InvList, invVect) } if len(invMsg.InvList) > 0 { diff --git a/lib/snapshot.go b/lib/snapshot.go index 70cb41c1c..7472e00c7 100644 --- a/lib/snapshot.go +++ b/lib/snapshot.go @@ -13,12 +13,12 @@ import ( "time" "github.com/cloudflare/circl/group" - "github.com/decred/dcrd/lru" "github.com/deso-protocol/go-deadlock" "github.com/dgraph-io/badger/v3" "github.com/fatih/color" "github.com/golang/glog" "github.com/google/uuid" + "github.com/hashicorp/golang-lru/v2" "github.com/oleiade/lane" "github.com/pkg/errors" "golang.org/x/sync/semaphore" @@ -313,7 +313,7 @@ type Snapshot struct { // DatabaseCache is used to store most recent DB records that we've read/written. // This is a low-level optimization for ancestral records that // saves us read time when we're writing to the DB during UtxoView flush. - DatabaseCache lru.KVCache + DatabaseCache *lru.Cache[string, []byte] // AncestralFlushCounter is used to offset ancestral records flush to occur only after x blocks. AncestralFlushCounter uint64 @@ -483,11 +483,13 @@ func NewSnapshot( "This may lead to unexpected behavior.") } + databaseCache, _ := lru.New[string, []byte](int(DatabaseCacheSize)) + // Set the snapshot. snap := &Snapshot{ mainDb: mainDb, SnapshotDbMutex: &snapshotDbMutex, - DatabaseCache: lru.NewKVCache(DatabaseCacheSize), + DatabaseCache: databaseCache, AncestralFlushCounter: uint64(0), snapshotBlockHeightPeriod: snapshotBlockHeightPeriod, OperationChannel: operationChannel, @@ -1394,7 +1396,7 @@ type StateChecksum struct { ctx context.Context // hashToCurveCache is a cache of computed hashToCurve mappings - hashToCurveCache lru.KVCache + hashToCurveCache *lru.Cache[string, group.Element] // When we want to add a database record to the state checksum, we will first have to // map the record to the Ristretto255 curve using the hash_to_curve. We will then add the @@ -1422,7 +1424,7 @@ func (sc *StateChecksum) Initialize(mainDb *badger.DB, snapshotDbMutex *sync.Mut sc.maxWorkers = int64(runtime.GOMAXPROCS(0)) // Set the hashToCurveCache - sc.hashToCurveCache = lru.NewKVCache(HashToCurveCache) + sc.hashToCurveCache, _ = lru.New[string, group.Element](int(HashToCurveCache)) // Set the worker pool semaphore and context. sc.semaphore = semaphore.NewWeighted(sc.maxWorkers) @@ -1487,7 +1489,7 @@ func (sc *StateChecksum) HashToCurve(bytes []byte) group.Element { // Check if we've already mapped this element, if so we will save some computation this way. bytesStr := hex.EncodeToString(bytes) - if elem, exists := sc.hashToCurveCache.Lookup(bytesStr); exists { + if elem, exists := sc.hashToCurveCache.Get(bytesStr); exists { hashElement = elem.(group.Element) } else { // Compute the hash_to_curve primitive, mapping the bytes to an elliptic curve point. From dbad6493af488515c0740bbedf77e3f00db5cb4b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 23:07:55 -0400 Subject: [PATCH 30/62] go mod tidy --- go.sum | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/go.sum b/go.sum index 65e5bdded..d0944531d 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,7 @@ github.com/btcsuite/snappy-go v1.0.0 h1:ZxaA6lo2EpxGddsA8JwWOcxlzRybb444sgmeJQMJ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bwesterb/go-ristretto v1.2.0 h1:xxWOVbN5m8NNKiSDZXE1jtZvZnC6JSJ9cYFADiZcWtw= -github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= @@ -102,8 +101,6 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= -github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.4.0 h1:BV7h5MgrktNzytKmWjpOtdYrf0lkkbF8YMlBGPhJQrY= github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= @@ -586,8 +583,6 @@ golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -707,7 +702,6 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -716,8 +710,6 @@ golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -731,8 +723,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From ebb117593b226ca99313378d92ba4caaea251c98 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 23:17:41 -0400 Subject: [PATCH 31/62] expose SetBestChain --- lib/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 06ef50632..5dd5da6e5 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1951,7 +1951,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChai } // TODO: need to figure out how to handle this for exchange api tests. -func (bc *Blockchain) setBestChain(bestChain []*BlockNode) { +func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { for _, blockNode := range bestChain { bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) if bc.blockIndex.GetTip() == nil { From ae34cff3f76f1c4a2e90784503e9c7dd8384ce3f Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 11:49:48 -0400 Subject: [PATCH 32/62] fix exit condition for GetBlocksToStore --- lib/server.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/server.go b/lib/server.go index 8a9b17a21..bad4143b3 100644 --- a/lib/server.go +++ b/lib/server.go @@ -897,16 +897,18 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } srv.blockchain.lowestBlockNotStored = uint64(blockNode.Height) numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) - currentHeight := int(blockNode.Height) + currentHeight := uint64(blockNode.Height) blockNodesToFetch := []*BlockNode{} // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. // We filter those out in the loop below by checking IsFullyProcessed. // Find the blocks that we should download. for len(blockNodesToFetch) < numBlocksToFetch { - + if currentHeight > uint64(srv.blockchain.blockTip().Height) { + break + } // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to // the array index. - currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(currentHeight), false) + currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(currentHeight, false) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) return From b48c52d9574d7a39e6ee848d8935909f8a8849e9 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 17:59:17 -0400 Subject: [PATCH 33/62] use bytes.Equal instead of reflect.DeepEqual for comparing block hashes --- lib/types.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/types.go b/lib/types.go index 5395d360b..36382ecbd 100644 --- a/lib/types.go +++ b/lib/types.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - "reflect" "sort" "github.com/holiman/uint256" @@ -238,7 +237,7 @@ func (bh *BlockHash) IsEqual(target *BlockHash) bool { return false } - return reflect.DeepEqual(bh[:], target[:]) + return bytes.Equal(bh[:], target[:]) } func (bh *BlockHash) NewBlockHash() *BlockHash { From 925e44e34e914be38a651f95817f55cad529f20d Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 18:24:53 -0400 Subject: [PATCH 34/62] use get block from best chain by height to speed up syncing --- lib/blockchain.go | 8 ++++---- lib/pos_blockchain.go | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 5dd5da6e5..df1c1b7a4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1901,12 +1901,12 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade } currNode := &BlockNode{} *currNode = *blockTip - for currNode != nil && !currNode.Hash.IsEqual(blockHash) && currNode.Height > committedTip.Height { + for currNode != nil && currNode.Height > committedTip.Height { + if !currNode.Hash.IsEqual(blockHash) { + return currNode, true, nil + } currNode = currNode.GetParent(bc.blockIndex) } - if currNode != nil && currNode.Hash.IsEqual(blockHash) { - return currNode, true, nil - } return nil, false, nil } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index ff793e940..426a97bac 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -54,11 +54,12 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. // TODO: make sure this is ok or do we need to explicitly check the block index's cache? - _, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) + //_, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) + blockNode, isHeightInBestHeaderChain, err := bc.GetBlockFromBestChainByHeight(header.Height, true) if err != nil { return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") } - if isInBestHeaderChain { + if isHeightInBestHeaderChain && blockNode.Hash.IsEqual(headerHash) { return true, false, nil } From dddd64cd99f7e0db5b0337680932e84bd5f44a56 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 18:32:30 -0400 Subject: [PATCH 35/62] fix condition for returning from get block by hash --- lib/blockchain.go | 2 +- lib/pos_blockchain_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index df1c1b7a4..dd2419669 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1902,7 +1902,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade currNode := &BlockNode{} *currNode = *blockTip for currNode != nil && currNode.Height > committedTip.Height { - if !currNode.Hash.IsEqual(blockHash) { + if currNode.Hash.IsEqual(blockHash) { return currNode, true, nil } currNode = currNode.GetParent(bc.blockIndex) diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 053d33bac..cf2e33f04 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -1276,7 +1276,7 @@ func TestShouldReorg(t *testing.T) { // 1. Simple reorg. Just replacing the uncommitted tip. // 2. Create a longer chain and reorg to it. // 3. Make sure no reorg when current view is greater than block's view -// 4. Super happy path of simply extending current uncommitted tip. +// 4. Super happy path of simply extet anding current uncommitted tip. func TestTryApplyNewTip(t *testing.T) { setBalanceModelBlockHeights(t) bc, _, _ := NewTestBlockchain(t) From 62503c12f2c63012485aee13f1678b80aa9a8a5d Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 18:47:04 -0400 Subject: [PATCH 36/62] revert using get block from best chain by height --- lib/blockchain.go | 2 +- lib/pos_blockchain.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index dd2419669..00a366dfa 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1896,7 +1896,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade if !exists { return nil, false, errors.New("GetBlockFromBestChainByHash: Committed tip not found") } - if uint64(bn.Height) > uint64(blockTip.Height) { + if uint64(bn.Height) > uint64(blockTip.Height) || uint64(bn.Height) < uint64(committedTip.Height) { return nil, false, nil } currNode := &BlockNode{} diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 426a97bac..ff793e940 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -54,12 +54,11 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. // TODO: make sure this is ok or do we need to explicitly check the block index's cache? - //_, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) - blockNode, isHeightInBestHeaderChain, err := bc.GetBlockFromBestChainByHeight(header.Height, true) + _, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) if err != nil { return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") } - if isHeightInBestHeaderChain && blockNode.Hash.IsEqual(headerHash) { + if isInBestHeaderChain { return true, false, nil } From 131f38910eb043bcfc6ee1c491e445a1e974f1f6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 21:16:05 -0400 Subject: [PATCH 37/62] add height check to reduce calls to compare hashes --- lib/blockchain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 00a366dfa..a0a9e954f 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1901,8 +1901,8 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade } currNode := &BlockNode{} *currNode = *blockTip - for currNode != nil && currNode.Height > committedTip.Height { - if currNode.Hash.IsEqual(blockHash) { + for currNode != nil && currNode.Height > bn.Height { + if currNode.Height == bn.Height && currNode.Hash.IsEqual(blockHash) { return currNode, true, nil } currNode = currNode.GetParent(bc.blockIndex) From f57a6993bef8fe9336a05b7c1a8492585e48b7ba Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 21:24:27 -0400 Subject: [PATCH 38/62] fix exit condition for height in get block from best chain by hash --- lib/blockchain.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index a0a9e954f..3aff181b6 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1901,9 +1901,12 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade } currNode := &BlockNode{} *currNode = *blockTip - for currNode != nil && currNode.Height > bn.Height { - if currNode.Height == bn.Height && currNode.Hash.IsEqual(blockHash) { - return currNode, true, nil + for currNode != nil && currNode.Height >= bn.Height { + if currNode.Height == bn.Height { + if currNode.Hash.IsEqual(blockHash) { + return currNode, true, nil + } + return nil, false, nil } currNode = currNode.GetParent(bc.blockIndex) } From 2bae1a2132bb1922cb9dc087ebe094289e94f7f7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 21:25:59 -0400 Subject: [PATCH 39/62] allow early return from GetBlockFromBestChainByHash if we're using header chain and the header is validated if we're syncing --- lib/blockchain.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/blockchain.go b/lib/blockchain.go index 3aff181b6..ed5356688 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1885,6 +1885,10 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade // have to get a bunch of parents in order to be sure it is part of the best header chain. I guess we could // have a map, but kinda defeats the purpose of this refactor. } + // TODO: is this legit? It seems like it's fair game... + if bc.isSyncing() && useHeaderChain && bn.IsHeaderValidated() { + return bn, true, nil + } blockTip := bc.BlockTip() if useHeaderChain { blockTip = bc.HeaderTip() From ea243f37009fc3ab59153792132ff69b6d9309e8 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 21:50:36 -0400 Subject: [PATCH 40/62] minor enhancement to isTipCurrent --- lib/blockchain.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index ed5356688..b8e20dde0 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1617,14 +1617,16 @@ func (bc *Blockchain) isTipCurrent(tip *BlockNode) bool { return tip.Height >= bc.MaxSyncBlockHeight } - minChainWorkBytes, _ := hex.DecodeString(bc.params.MinChainWorkHex) - // Not current if the cumulative work is below the threshold. - if bc.params.IsPoWBlockHeight(uint64(tip.Height)) && tip.CumWork.Cmp(BytesToBigint(minChainWorkBytes)) < 0 { - //glog.V(2).Infof("Blockchain.isTipCurrent: Tip not current because "+ - //"CumWork (%v) is less than minChainWorkBytes (%v)", - //tip.CumWork, BytesToBigint(minChainWorkBytes)) - return false + if bc.params.IsPoWBlockHeight(uint64(tip.Height)) { + minChainWorkBytes, _ := hex.DecodeString(bc.params.MinChainWorkHex) + + if tip.CumWork.Cmp(BytesToBigint(minChainWorkBytes)) < 0 { + //glog.V(2).Infof("Blockchain.isTipCurrent: Tip not current because "+ + //"CumWork (%v) is less than minChainWorkBytes (%v)", + //tip.CumWork, BytesToBigint(minChainWorkBytes)) + return false + } } // Not current if the tip has a timestamp older than the maximum From 3f73bb6349fbde366a625b4abb14f0be8e2d1bc5 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 11:21:41 -0400 Subject: [PATCH 41/62] fix txindex issue w/ genesis block --- lib/txindex.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/txindex.go b/lib/txindex.go index d3794d935..fa173e955 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -394,7 +394,9 @@ func (txi *TXIndex) Update() error { // // Only set a BitcoinManager if we have one. This makes some tests pass. utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) - if blockToAttach.Header.PrevBlockHash != nil && !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { + if blockToAttach.Height > 0 && + blockToAttach.Header.PrevBlockHash != nil && + !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { var utxoViewAndUtxoOps *BlockViewAndUtxoOps utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) if err != nil { From e20040765db199d1667afec195dcd6254003c21b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 11:34:47 -0400 Subject: [PATCH 42/62] fix process block pow logic for genesis block --- lib/blockchain.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index b8e20dde0..35735e4b5 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2622,7 +2622,6 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.Start("Blockchain.ProcessBlock: BlockNode") // See if a node for the block exists in our node index. - // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set @@ -2664,10 +2663,13 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // In this case go ahead and return early. If its parents are truly legitimate then we // should re-request it and its parents from a node and reprocess it // once it is no longer an orphan. - // TODO: validate that current height - 1 > 0 - parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) - if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { - return false, true, nil + var parentNode *BlockNode + if blockHeader.Height > 0 { + var parentNodeExists bool + parentNode, parentNodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) + if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { + return false, true, nil + } } if nodeToValidate.Status.IsFullyProcessed() { @@ -2697,11 +2699,14 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Reject the block if any of the following apply to the parent: // - Its header is nil. // - Its header or its block validation failed. - if parentNode.Header == nil || - (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { + if blockHeight > 0 { + if parentNode == nil || + parentNode.Header == nil || + (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { - bc.MarkBlockInvalid(nodeToValidate, RuleErrorPreviousBlockInvalid) - return false, false, RuleErrorPreviousBlockInvalid + bc.MarkBlockInvalid(nodeToValidate, RuleErrorPreviousBlockInvalid) + return false, false, RuleErrorPreviousBlockInvalid + } } // At this point, we know that we are processing a block we haven't seen From 3ad68be4113b5a050806cc9b919676e53e8b8637 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 11:41:56 -0400 Subject: [PATCH 43/62] fix - get next block after tip hash in update process of txindex --- lib/blockchain.go | 23 +++++++++-------------- lib/txindex.go | 11 ++++++----- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 35735e4b5..b8e20dde0 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2622,6 +2622,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.Start("Blockchain.ProcessBlock: BlockNode") // See if a node for the block exists in our node index. + // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set @@ -2663,13 +2664,10 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // In this case go ahead and return early. If its parents are truly legitimate then we // should re-request it and its parents from a node and reprocess it // once it is no longer an orphan. - var parentNode *BlockNode - if blockHeader.Height > 0 { - var parentNodeExists bool - parentNode, parentNodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) - if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { - return false, true, nil - } + // TODO: validate that current height - 1 > 0 + parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) + if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { + return false, true, nil } if nodeToValidate.Status.IsFullyProcessed() { @@ -2699,14 +2697,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Reject the block if any of the following apply to the parent: // - Its header is nil. // - Its header or its block validation failed. - if blockHeight > 0 { - if parentNode == nil || - parentNode.Header == nil || - (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { + if parentNode.Header == nil || + (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { - bc.MarkBlockInvalid(nodeToValidate, RuleErrorPreviousBlockInvalid) - return false, false, RuleErrorPreviousBlockInvalid - } + bc.MarkBlockInvalid(nodeToValidate, RuleErrorPreviousBlockInvalid) + return false, false, RuleErrorPreviousBlockInvalid } // At this point, we know that we are processing a block we haven't seen diff --git a/lib/txindex.go b/lib/txindex.go index fa173e955..d43e5f6a2 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -369,8 +369,11 @@ func (txi *TXIndex) Update() error { // For each of the blocks we're adding, process them on our txindex chain // and add their mappings to our txn index. Compute any metadata that might // be useful. - blockToAttach := &BlockNode{} - *blockToAttach = *txindexTipNode + // Get the next block after the current txindex tip hash. we know we've already processed the txindex tip hash. + blockToAttach, exists, err := txi.CoreChain.GetBlockFromBestChainByHeight(uint64(txindexTipNode.Height+1), false) + if !exists || err != nil { + return fmt.Errorf("Update: Problem getting block at height %d: %v", txindexTipNode.Height+1, err) + } for !blockToAttach.Hash.IsEqual(blockTipNode.Hash) { if txi.killed { glog.Infof(CLog(Yellow, "TxIndex: Update: Killed while attaching blocks")) @@ -394,9 +397,7 @@ func (txi *TXIndex) Update() error { // // Only set a BitcoinManager if we have one. This makes some tests pass. utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) - if blockToAttach.Height > 0 && - blockToAttach.Header.PrevBlockHash != nil && - !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { + if blockToAttach.Header.PrevBlockHash != nil && !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { var utxoViewAndUtxoOps *BlockViewAndUtxoOps utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) if err != nil { From 78034d40d3571adb751c750daa0b6ec97e432d3e Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 12:03:18 -0400 Subject: [PATCH 44/62] reduce direct access to BlockNode.Parent and get rid of heal orphan pointers since we don't need it --- lib/block_view.go | 1 + lib/block_view_lockups_test.go | 1 - lib/blockchain.go | 21 ++++++++---- lib/pos_blockchain.go | 63 +++++++++++++++++----------------- lib/postgres.go | 2 ++ 5 files changed, 49 insertions(+), 39 deletions(-) diff --git a/lib/block_view.go b/lib/block_view.go index 2cd959419..ab0e87285 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -5112,6 +5112,7 @@ func (bav *UtxoView) GetSpendableDeSoBalanceNanosForPublicKey(pkBytes []byte, // but we do have the header. As a result, this condition always evaluates to false and thus // we only process the block reward for the previous block instead of all immature block rewards // as defined by the params. + // NOTE: we are not using .GetParent here as it changes the meaning of this code. if blockNode.Parent != nil { nextBlockHash = blockNode.Parent.Hash } else { diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index 41e53e74e..a43beecf4 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2461,7 +2461,6 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, err) require.NoError(t, utxoView.FlushToDb(blk2.Header.Height)) - // TODO: make sure this works? // Update the tip testMeta.chain.blockIndex.tip = testMeta.chain.blockIndex.tip.Parent diff --git a/lib/blockchain.go b/lib/blockchain.go index b8e20dde0..dbcb96dc4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -390,6 +390,8 @@ func (nn *BlockNode) String() string { var parentHash *BlockHash if nn.Parent != nil { parentHash = nn.Parent.Hash + } else { + parentHash = nn.Header.PrevBlockHash } tstamp := uint32(0) if nn.Header != nil { @@ -429,6 +431,9 @@ func (nn *BlockNode) Ancestor(height uint32, blockIndex *BlockIndex) *BlockNode } node := nn + // NOTE: using .Parent here is okay b/c it explicitly set it + // if we don't already have it when we fetch the parent from + // the block index. for ; node != nil && node.Height != height; node = node.Parent { // Keep iterating node until the condition no longer holds. if node.Parent == nil { @@ -1478,7 +1483,7 @@ func (bc *Blockchain) GetBlockNodesToFetch( // StatusBlockProcessed so this loop is guaranteed to terminate successfully. headerNodeStart = bc.headerTip() for headerNodeStart != nil && (headerNodeStart.Status&StatusBlockProcessed) == 0 { - headerNodeStart = headerNodeStart.Parent + headerNodeStart = headerNodeStart.GetParent(bc.blockIndex) } if headerNodeStart == nil { @@ -3006,11 +3011,13 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If we're syncing there's no risk of concurrency issues. Otherwise, we // need to make a copy in order to be save. - if bc.isSyncing() { - bc.blockIndex.setTip(nodeToValidate) - } else { - bc.blockIndex.setTip(nodeToValidate) - } + // We no longer need to worry about whether we're syncing or not. Just + // set the tip. + //if bc.isSyncing() { + // bc.blockIndex.setTip(nodeToValidate) + //} else { + bc.blockIndex.setTip(nodeToValidate) + //} // This node is on the main chain so set this variable. isMainChain = true @@ -3165,7 +3172,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If the parent node has been marked as invalid then mark this node as // invalid as well. - if (attachNode.Parent.Status & StatusBlockValidateFailed) != 0 { + if (attachNode.GetParent(bc.blockIndex).Status & StatusBlockValidateFailed) != 0 { bc.MarkBlockInvalid(attachNode, RuleErrorPreviousBlockInvalid) continue } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index ff793e940..4b610f017 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -76,11 +76,11 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH } // Don't worry about healing orphan children when we're syncing. - if !bc.isSyncing() { - // Now that we know we have a valid header, we check the block index for it any orphan children for it - // and heal the parent pointers for all of them. - bc.healPointersForOrphanChildren(blockNode) - } + //if !bc.isSyncing() { + // // Now that we know we have a valid header, we check the block index for it any orphan children for it + // // and heal the parent pointers for all of them. + // bc.healPointersForOrphanChildren(blockNode) + //} // Exit early if the header is an orphan. if isOrphan { @@ -116,31 +116,31 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // later on, we not only need to store the parent in the block index but also need to update the // pointer from the orphan block's BlockNode to the parent. We do that dynamically here as we // process headers. -func (bc *Blockchain) healPointersForOrphanChildren(blockNode *BlockNode) { - // Fetch all potential children of this blockNode from the block index. - blockNodesAtNextHeight := bc.blockIndex.GetBlockNodesByHeight(blockNode.Header.Height + 1) - exists := len(blockNodesAtNextHeight) > 0 - if !exists { - // No children of this blockNode exist in the block index. Exit early. - return - } - - // Iterate through all block nodes at the next block height and update their parent pointers. - for _, blockNodeAtNextHeight := range blockNodesAtNextHeight { - // Check if it's a child of the parent block node. - if !blockNodeAtNextHeight.Header.PrevBlockHash.IsEqual(blockNode.Hash) { - continue - } - - // Check if it has its parent pointer set. If it does, then we exit early. - if blockNodeAtNextHeight.Parent != nil { - continue - } - - // If the parent block node is not set, then we set it to the parent block node. - blockNodeAtNextHeight.Parent = blockNode - } -} +//func (bc *Blockchain) healPointersForOrphanChildren(blockNode *BlockNode) { +// // Fetch all potential children of this blockNode from the block index. +// blockNodesAtNextHeight := bc.blockIndex.GetBlockNodesByHeight(blockNode.Header.Height + 1) +// exists := len(blockNodesAtNextHeight) > 0 +// if !exists { +// // No children of this blockNode exist in the block index. Exit early. +// return +// } +// +// // Iterate through all block nodes at the next block height and update their parent pointers. +// for _, blockNodeAtNextHeight := range blockNodesAtNextHeight { +// // Check if it's a child of the parent block node. +// if !blockNodeAtNextHeight.Header.PrevBlockHash.IsEqual(blockNode.Hash) { +// continue +// } +// +// // Check if it has its parent pointer set. If it does, then we exit early. +// if blockNodeAtNextHeight.Parent != nil { +// continue +// } +// +// // If the parent block node is not set, then we set it to the parent block node. +// blockNodeAtNextHeight.Parent = blockNode +// } +//} func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHash *BlockHash, verifySignatures bool) ( _headerBlockNode *BlockNode, _isOrphan bool, _err error, @@ -1391,14 +1391,15 @@ func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock, ha } } blockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(hash, block.Header.Height) - prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) if blockNode != nil { // If the block node already exists, we should set its parent if it doesn't have one already. if blockNode.Parent == nil { + prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) blockNode.Parent = prevBlockNode } return blockNode, nil } + prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) newBlockNode := NewBlockNode(prevBlockNode, hash, uint32(block.Header.Height), nil, nil, block.Header, StatusNone) bc.addNewBlockNodeToBlockIndex(newBlockNode) return newBlockNode, nil diff --git a/lib/postgres.go b/lib/postgres.go index e919c8e78..c4eef6bb0 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -1298,6 +1298,8 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { // The genesis block has a nil parent if blockNode.Parent != nil { block.ParentHash = blockNode.Parent.Hash + } else if !blockNode.Header.PrevBlockHash.IsEqual(GenesisBlockHash) { + block.ParentHash = blockNode.Header.PrevBlockHash } _, err := tx.Model(block).WherePK().OnConflict("(hash) DO UPDATE").Insert() From 02e9e15decd5d3abf8f61d6a30f3c4d8f27b8f3c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 16:45:49 -0400 Subject: [PATCH 45/62] fix postgres get best chain logic --- lib/postgres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/postgres.go b/lib/postgres.go index c4eef6bb0..1a2136c0f 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -1341,7 +1341,7 @@ func (postgres *Postgres) GetBlockIndex() (*lru.Cache[BlockHash, *BlockNode], er parentHash := blockNode.Header.PrevBlockHash if parentHash != nil { parent, exists := blockMap.Get(*parentHash) - if !exists { + if !exists && blockNode.Height > 0 { glog.Fatal("Parent block not found in block map") } blockNode.Parent = parent From 2a82deaa2a57040e1f6b46b41f0a784a0d8d68fb Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Tue, 5 Nov 2024 13:17:57 -0500 Subject: [PATCH 46/62] Ln/downgrade badger (#1430) * downgrade badger, ristretto, and klauspost/compress dependencies * go mod tidy --- cmd/node.go | 2 +- go.mod | 10 ++-- go.sum | 61 +++++++++++++++++---- integration_testing/tools.go | 2 +- lib/block_view.go | 2 +- lib/block_view_atomic_txns_test.go | 2 +- lib/block_view_bitcoin_test.go | 2 +- lib/block_view_creator_coin_test.go | 2 +- lib/block_view_dao_coin_limit_order_test.go | 2 +- lib/block_view_dao_coin_test.go | 2 +- lib/block_view_derived_key_test.go | 2 +- lib/block_view_flush.go | 2 +- lib/block_view_follow_test.go | 2 +- lib/block_view_like_test.go | 2 +- lib/block_view_lockups.go | 2 +- lib/block_view_lockups_test.go | 2 +- lib/block_view_message_test.go | 2 +- lib/block_view_nft_test.go | 2 +- lib/block_view_post.go | 2 +- lib/block_view_post_test.go | 2 +- lib/block_view_profile_test.go | 2 +- lib/block_view_stake.go | 2 +- lib/block_view_test.go | 2 +- lib/block_view_validator.go | 2 +- lib/blockchain.go | 2 +- lib/blockchain_test.go | 2 +- lib/db_adapter.go | 2 +- lib/db_utils.go | 2 +- lib/db_utils_test.go | 2 +- lib/legacy_mempool.go | 2 +- lib/notifier.go | 2 +- lib/pos_blockchain.go | 2 +- lib/pos_epoch.go | 2 +- lib/pos_mempool.go | 2 +- lib/pos_mempool_persister.go | 2 +- lib/pos_mempool_persister_test.go | 2 +- lib/pos_mempool_test.go | 2 +- lib/pos_random_seed.go | 2 +- lib/pos_snapshot_cache.go | 2 +- lib/pos_snapshot_entries.go | 2 +- lib/postgres.go | 2 +- lib/server.go | 3 +- lib/snapshot.go | 2 +- lib/txindex.go | 2 +- 44 files changed, 100 insertions(+), 56 deletions(-) diff --git a/cmd/node.go b/cmd/node.go index 52bf98b70..b1501978b 100644 --- a/cmd/node.go +++ b/cmd/node.go @@ -18,7 +18,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/deso-protocol/core/lib" "github.com/deso-protocol/core/migrate" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/go-pg/pg/v10" "github.com/golang/glog" migrations "github.com/robinjoseph08/go-pg-migrations/v3" diff --git a/go.mod b/go.mod index f4f3cb798..8ff891d0f 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/deso-protocol/go-deadlock v1.0.1 github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/deso-protocol/uint256 v1.3.2 - github.com/dgraph-io/badger/v4 v4.3.1 + github.com/dgraph-io/badger/v3 v3.2103.5 github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.14.11 github.com/fatih/color v1.17.0 @@ -36,7 +36,7 @@ require ( github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.19.0 + github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.1.0 github.com/unrolled/secure v1.16.0 @@ -69,11 +69,12 @@ require ( github.com/andygrunwald/go-jira v1.16.0 // indirect github.com/btcsuite/btclog v0.0.0-20241017175713-3428138b75c7 // indirect github.com/bwesterb/go-ristretto v1.2.3 // indirect + github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect - github.com/dgraph-io/ristretto v1.0.0 // indirect + github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect github.com/ebitengine/purego v0.8.0 // indirect @@ -85,6 +86,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20241009165004-a3522334989c // indirect @@ -98,7 +100,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.17.1 // indirect github.com/kyokomi/emoji/v2 v2.2.13 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index 29ce83362..c0af42e8b 100644 --- a/go.sum +++ b/go.sum @@ -33,9 +33,12 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8= github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -72,6 +75,8 @@ github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7N github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -79,8 +84,12 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -104,10 +113,11 @@ github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+Ls github.com/deso-protocol/go-merkle-tree v1.0.0/go.mod h1:V/vbg/maaNv6G7zf9VVs645nLFx/jsO2L/awFB/S/ZU= github.com/deso-protocol/uint256 v1.3.2 h1:nHwqfdCKgWimWLJbiN/9DV95qDJ5lZcf8n5cAHbdG6o= github.com/deso-protocol/uint256 v1.3.2/go.mod h1:Wq2bibbApz3TsiL+VPUnzr+UkhG4eBeQ0DpbQcjQYcA= -github.com/dgraph-io/badger/v4 v4.3.1 h1:7r5wKqmoRpGgSxqa0S/nGdpOpvvzuREGPLSua73C8tw= -github.com/dgraph-io/badger/v4 v4.3.1/go.mod h1:oObz97DImXpd6O/Dt8BqdKLLTDmEmarAimo72VV5whQ= -github.com/dgraph-io/ristretto v1.0.0 h1:SYG07bONKMlFDUYu5pEu3DGAh8c2OFNzKm6G9J4Si84= -github.com/dgraph-io/ristretto v1.0.0/go.mod h1:jTi2FiYEhQ1NsMmA7DeBykizjOuY88NhKBkepyu1jPc= +github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= +github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= +github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -154,12 +164,14 @@ github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -172,9 +184,11 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -184,6 +198,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -207,6 +222,7 @@ github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9 github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= github.com/hashicorp/golang-lru/v2 v2.0.3/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= @@ -218,6 +234,7 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -230,8 +247,9 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -243,6 +261,7 @@ github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -261,6 +280,7 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -287,6 +307,7 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= @@ -307,6 +328,7 @@ github.com/robinjoseph08/go-pg-migrations/v3 v3.1.0 h1:EjexnDlSIZoK/gMfQmKIqB7tY github.com/robinjoseph08/go-pg-migrations/v3 v3.1.0/go.mod h1:9yEG60N97UVFGD/UKQUXoGVZh/t8KXx3JxEpxhKFlKY= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -326,18 +348,25 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -345,6 +374,7 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -369,6 +399,7 @@ github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df h1:Y2l28Jr3 github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df/go.mod h1:pnyouUty/nBr/zm3GYwTIt+qFTLWbdjeLjZmJdzJOu8= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/unrolled/secure v1.16.0 h1:XgdAsS/Zl50ZfZPRJK6WpicFttfrsFYFd0+ONDBJubU= github.com/unrolled/secure v1.16.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= @@ -383,12 +414,14 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -400,6 +433,7 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -445,6 +479,7 @@ golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -455,9 +490,11 @@ golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -477,6 +514,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -527,15 +565,17 @@ gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -553,6 +593,7 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.69.0 h1:zSY6DDsFRMQDNQYKWCv/AEwJXoPpDf1FfMyw7 gopkg.in/DataDog/dd-trace-go.v1 v1.69.0/go.mod h1:U9AOeBHNAL95JXcd/SPf4a7O5GNeF/yD13sJtli/yaU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/integration_testing/tools.go b/integration_testing/tools.go index 26e322335..94617bd18 100644 --- a/integration_testing/tools.go +++ b/integration_testing/tools.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" "github.com/stretchr/testify/require" diff --git a/lib/block_view.go b/lib/block_view.go index c9d3f590a..3a8eacb12 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -18,7 +18,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_atomic_txns_test.go b/lib/block_view_atomic_txns_test.go index ead9a7049..53d71c7e1 100644 --- a/lib/block_view_atomic_txns_test.go +++ b/lib/block_view_atomic_txns_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_bitcoin_test.go b/lib/block_view_bitcoin_test.go index 99352cf84..c12b8af0a 100644 --- a/lib/block_view_bitcoin_test.go +++ b/lib/block_view_bitcoin_test.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/btcsuite/btcd/wire" merkletree "github.com/deso-protocol/go-merkle-tree" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_creator_coin_test.go b/lib/block_view_creator_coin_test.go index 7e4a3a05b..e25af8ac9 100644 --- a/lib/block_view_creator_coin_test.go +++ b/lib/block_view_creator_coin_test.go @@ -5,7 +5,7 @@ import ( "strconv" "testing" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_dao_coin_limit_order_test.go b/lib/block_view_dao_coin_limit_order_test.go index 98034b5e0..07d214725 100644 --- a/lib/block_view_dao_coin_limit_order_test.go +++ b/lib/block_view_dao_coin_limit_order_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_dao_coin_test.go b/lib/block_view_dao_coin_test.go index aa64707bd..b006c3d7d 100644 --- a/lib/block_view_dao_coin_test.go +++ b/lib/block_view_dao_coin_test.go @@ -6,7 +6,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_derived_key_test.go b/lib/block_view_derived_key_test.go index bc59dd3f9..e9d079786 100644 --- a/lib/block_view_derived_key_test.go +++ b/lib/block_view_derived_key_test.go @@ -13,7 +13,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_flush.go b/lib/block_view_flush.go index 52a210636..e65341b4f 100644 --- a/lib/block_view_flush.go +++ b/lib/block_view_flush.go @@ -5,7 +5,7 @@ import ( "reflect" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_follow_test.go b/lib/block_view_follow_test.go index ef3823f63..fedb91a03 100644 --- a/lib/block_view_follow_test.go +++ b/lib/block_view_follow_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_like_test.go b/lib/block_view_like_test.go index a757c7f70..01b715475 100644 --- a/lib/block_view_like_test.go +++ b/lib/block_view_like_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_lockups.go b/lib/block_view_lockups.go index c2c3e34ea..3a4a8d532 100644 --- a/lib/block_view_lockups.go +++ b/lib/block_view_lockups.go @@ -8,7 +8,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index af461cd5f..48f435363 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -7,7 +7,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_message_test.go b/lib/block_view_message_test.go index a47c3d85c..dbe204f99 100644 --- a/lib/block_view_message_test.go +++ b/lib/block_view_message_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_nft_test.go b/lib/block_view_nft_test.go index 2f4cfd64c..ba00d8ece 100644 --- a/lib/block_view_nft_test.go +++ b/lib/block_view_nft_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_post.go b/lib/block_view_post.go index e80d20daf..a6b9e6d30 100644 --- a/lib/block_view_post.go +++ b/lib/block_view_post.go @@ -13,7 +13,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/davecgh/go-spew/spew" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/gernest/mention" "github.com/golang/glog" "github.com/pkg/errors" diff --git a/lib/block_view_post_test.go b/lib/block_view_post_test.go index 264df7d69..e84eaa1c9 100644 --- a/lib/block_view_post_test.go +++ b/lib/block_view_post_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_profile_test.go b/lib/block_view_profile_test.go index c59e37cd5..01101d005 100644 --- a/lib/block_view_profile_test.go +++ b/lib/block_view_profile_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_stake.go b/lib/block_view_stake.go index 10ad6ab7f..b83d5d7ac 100644 --- a/lib/block_view_stake.go +++ b/lib/block_view_stake.go @@ -7,7 +7,7 @@ import ( "sort" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_test.go b/lib/block_view_test.go index cb7a30fb8..8aecfb24f 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -13,7 +13,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" embeddedpostgres "github.com/fergusstrange/embedded-postgres" "github.com/golang/glog" "github.com/hashicorp/golang-lru/v2" diff --git a/lib/block_view_validator.go b/lib/block_view_validator.go index 8cc00a14b..3b2560654 100644 --- a/lib/block_view_validator.go +++ b/lib/block_view_validator.go @@ -15,7 +15,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/blockchain.go b/lib/blockchain.go index 640b1cd71..90b20c2f2 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -30,7 +30,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/davecgh/go-spew/spew" merkletree "github.com/deso-protocol/go-merkle-tree" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 37930a11c..6ce522f7c 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -19,7 +19,7 @@ import ( chainlib "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/db_adapter.go b/lib/db_adapter.go index 8cff5d757..4eea4d0da 100644 --- a/lib/db_adapter.go +++ b/lib/db_adapter.go @@ -4,7 +4,7 @@ import ( "bytes" "sort" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/db_utils.go b/lib/db_utils.go index 126cc703b..63f78fcbf 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -24,7 +24,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/davecgh/go-spew/spew" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/db_utils_test.go b/lib/db_utils_test.go index 82a452df6..6a6a0f208 100644 --- a/lib/db_utils_test.go +++ b/lib/db_utils_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/legacy_mempool.go b/lib/legacy_mempool.go index 247092d1e..7e9dceddf 100644 --- a/lib/legacy_mempool.go +++ b/lib/legacy_mempool.go @@ -18,7 +18,7 @@ import ( "time" "github.com/btcsuite/btcd/btcutil" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/gernest/mention" "github.com/btcsuite/btcd/btcec/v2" diff --git a/lib/notifier.go b/lib/notifier.go index b6c88c37d..a42ffb3ab 100644 --- a/lib/notifier.go +++ b/lib/notifier.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/gernest/mention" "github.com/go-pg/pg/v10" diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 588df734a..4b610f017 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -9,7 +9,7 @@ import ( "github.com/deso-protocol/core/collections" "github.com/deso-protocol/core/consensus" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_epoch.go b/lib/pos_epoch.go index ef4e3c78d..fed62d90e 100644 --- a/lib/pos_epoch.go +++ b/lib/pos_epoch.go @@ -3,7 +3,7 @@ package lib import ( "bytes" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index 6cfd630d2..5592553f0 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" diff --git a/lib/pos_mempool_persister.go b/lib/pos_mempool_persister.go index ceeface31..05a1ef617 100644 --- a/lib/pos_mempool_persister.go +++ b/lib/pos_mempool_persister.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool_persister_test.go b/lib/pos_mempool_persister_test.go index 73887a01f..6cd1ef25d 100644 --- a/lib/pos_mempool_persister_test.go +++ b/lib/pos_mempool_persister_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/require" ) diff --git a/lib/pos_mempool_test.go b/lib/pos_mempool_test.go index 8dced4464..d04883919 100644 --- a/lib/pos_mempool_test.go +++ b/lib/pos_mempool_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/pos_random_seed.go b/lib/pos_random_seed.go index 9c739c41d..6777de6e6 100644 --- a/lib/pos_random_seed.go +++ b/lib/pos_random_seed.go @@ -7,7 +7,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_snapshot_cache.go b/lib/pos_snapshot_cache.go index f320e99b0..ec495633c 100644 --- a/lib/pos_snapshot_cache.go +++ b/lib/pos_snapshot_cache.go @@ -3,7 +3,7 @@ package lib import ( "sync" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" ) diff --git a/lib/pos_snapshot_entries.go b/lib/pos_snapshot_entries.go index 7788d170c..b727c384a 100644 --- a/lib/pos_snapshot_entries.go +++ b/lib/pos_snapshot_entries.go @@ -9,7 +9,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/core/collections" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/postgres.go b/lib/postgres.go index 6518d7a9b..9c9ba432f 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -12,7 +12,7 @@ import ( "strings" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/go-pg/pg/v10" "github.com/go-pg/pg/v10/orm" "github.com/golang/glog" diff --git a/lib/server.go b/lib/server.go index 6165592f7..28f1947ff 100644 --- a/lib/server.go +++ b/lib/server.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "github.com/dgraph-io/badger/v4" "net" "path/filepath" "reflect" @@ -20,7 +21,7 @@ import ( "github.com/deso-protocol/core/collections" "github.com/deso-protocol/core/consensus" "github.com/deso-protocol/go-deadlock" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" diff --git a/lib/snapshot.go b/lib/snapshot.go index d10adc0f4..2d8ffb895 100644 --- a/lib/snapshot.go +++ b/lib/snapshot.go @@ -15,7 +15,7 @@ import ( "github.com/cloudflare/circl/group" "github.com/deso-protocol/go-deadlock" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/fatih/color" "github.com/golang/glog" "github.com/google/uuid" diff --git a/lib/txindex.go b/lib/txindex.go index 2868245b6..29bc0336c 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" chainlib "github.com/btcsuite/btcd/blockchain" "github.com/golang/glog" From 35f7a6af30190c63659dc29dec94910b2d2a9dd6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 4 Dec 2024 22:51:19 -0500 Subject: [PATCH 47/62] remove LocateBestBlockChainHeaders and replace with DH suggestion. flush headers in a batch at the end of each bundle, merge in latest main --- lib/blockchain.go | 72 ++++++++++++++++++--------------- lib/blockchain_test.go | 10 ++--- lib/pos_blockchain.go | 24 +++++------ lib/server.go | 91 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 143 insertions(+), 54 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 0937f7a61..653764400 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -624,6 +624,8 @@ func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { blocksAtHeight = []*BlockNode{} } else { // Make sure we don't add the same block node twice. + // TODO: we *could* make this more efficient by using a map, + // but generally we won't have many blocks at the same height. for ii, blockAtHeight := range blocksAtHeight { if blockAtHeight.Hash.IsEqual(blockNode.Hash) { blocksAtHeight[ii] = blockNode @@ -1191,6 +1193,7 @@ func fastLog2Floor(n uint32) uint8 { return rv } +// TODO: deprecate // locateInventory returns the node of the block after the first known block in // the locator along with the number of subsequent nodes needed to either reach // the provided stop hash or the provided max number of entries. @@ -1279,6 +1282,7 @@ func (bc *Blockchain) locateInventory(locator []*BlockHash, stopHash *BlockHash, return startNode, total } +// TODO: deprecate // locateHeaders returns the headers of the blocks after the first known block // in the locator until the provided stop hash is reached, or up to the provided // max number of block headers. @@ -1319,6 +1323,7 @@ func (bc *Blockchain) locateHeaders(locator []*BlockHash, stopHash *BlockHash, m return headers } +// TODO: deprecate // LocateBestBlockChainHeaders returns the headers of the blocks after the first known block // in the locator until the provided stop hash is reached, or up to a max of // wire.MaxBlockHeadersPerMsg headers. Note that it returns the best headers @@ -1589,16 +1594,6 @@ func (bc *Blockchain) GetBlock(blockHash *BlockHash) *MsgDeSoBlock { return blk } -func (bc *Blockchain) GetBlockAtHeight(height uint32, isHeaderChain bool) (*MsgDeSoBlock, error) { - bn, bnExists, err := bc.GetBlockFromBestChainByHeight(uint64(height), isHeaderChain) - if !bnExists || err != nil { - glog.Errorf("Blockchain.GetBlockAtHeight: Problem getting block by height: %v", err) - return nil, err - } - - return bc.GetBlock(bn.Hash), nil -} - // GetBlockNodeWithHash looks for a block node in the bestChain list that matches the hash. func (bc *Blockchain) GetBlockNodeWithHash(hash *BlockHash) *BlockNode { if hash == nil { @@ -1936,6 +1931,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade } func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChain bool) (*BlockNode, bool, error) { + // TODO: figure out an optimization for header chain handling uncommitted state. if !useHeaderChain { committedTip, exists := bc.GetCommittedTip() if !exists { @@ -1944,9 +1940,6 @@ func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChai if height >= uint64(committedTip.Height) { // For this, we can just loop back from the tip block. currentNode := bc.blockIndex.GetTip() - if useHeaderChain { - currentNode = bc.blockIndex.GetHeaderTip() - } for currentNode != nil { if uint64(currentNode.Height) == height { return currentNode, true, nil @@ -1964,10 +1957,15 @@ func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChai return nil, false, nil } for _, blockNode := range blockNodes { - if !useHeaderChain && blockNode.IsCommitted() { + // If block node is committed, then we know it is + // in the best chain, whether we're looking at the header chain or not. + if blockNode.IsCommitted() { return blockNode, true, nil } - // TODO: this is crude and incorrect. + // TODO: this is crude and incorrect. We can have multiple headers + // at a specific height. It's possible that none of the blocks at + // this height are committed yet, but one of them is in the best chain. + // How can we figure it out? if useHeaderChain && blockNode.IsHeaderValidated() { return blockNode, true, nil } @@ -2340,10 +2338,13 @@ func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru.Cache } // Caller must acquire the ChainLock for writing prior to calling this. -func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *BlockHash) (_isMainChain bool, _isOrphan bool, _err error) { +func (bc *Blockchain) processHeaderPoW( + blockHeader *MsgDeSoHeader, + headerHash *BlockHash, +) (_blockNode *BlockNode, _isMainChain bool, _isOrphan bool, _err error) { // Only accept the header if its height is below the PoS cutover height. if !bc.params.IsPoWBlockHeight(blockHeader.Height) { - return false, false, HeaderErrorBlockHeightAfterProofOfStakeCutover + return nil, false, false, HeaderErrorBlockHeightAfterProofOfStakeCutover } // Only accept headers if the best chain is still in PoW. Once the best chain reaches the final @@ -2351,16 +2352,16 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // headers past this point because they will un-commit blocks that are already committed to the PoS // chain. if bc.BlockTip().Header.Height >= bc.params.GetFinalPoWBlockHeight() { - return false, false, HeaderErrorBestChainIsAtProofOfStakeCutover + return nil, false, false, HeaderErrorBestChainIsAtProofOfStakeCutover } // Start by checking if the header already exists in our node // index. If it does, then return an error. We should generally // expect that processHeaderPoW will only be called on headers we // haven't seen before. - _, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, blockHeader.Height) + blockNode, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, blockHeader.Height) if nodeExists { - return false, false, HeaderErrorDuplicateHeader + return blockNode, false, false, HeaderErrorDuplicateHeader } // If we're here then it means we're processing a header we haven't @@ -2373,7 +2374,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B "MaxTstampOffsetSeconds %d. blockHeader.TstampSecs=%d; adjustedTime=%d", tstampDiff, bc.params.MaxTstampOffsetSeconds, blockHeader.GetTstampSecs(), bc.timeSource.AdjustedTime().Unix()) - return false, false, HeaderErrorBlockTooFarInTheFuture + return nil, false, false, HeaderErrorBlockTooFarInTheFuture } // Try to find this header's parent in our block index. @@ -2381,13 +2382,13 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // can return early because we don't process unconnectedTxns. // TODO: Should we just return an error if the header is an orphan? if blockHeader.PrevBlockHash == nil { - return false, false, HeaderErrorNilPrevHash + return nil, false, false, HeaderErrorNilPrevHash } parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) if !parentNodeExists { // This block is an orphan if its parent doesn't exist and we don't // process unconnectedTxns. - return false, true, nil + return nil, false, true, nil } // If the parent node is invalid then this header is invalid as well. Note that @@ -2395,7 +2396,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // ValidateFailed. parentHeader := parentNode.Header if parentHeader == nil || (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { - return false, false, errors.Wrapf( + return nil, false, false, errors.Wrapf( HeaderErrorInvalidParent, "Parent header: %v, Status check: %v, Parent node status: %v, Parent node header: %v", parentHeader, (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0, parentNode.Status, @@ -2407,7 +2408,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if blockHeader.Height != prevHeight+1 { glog.Errorf("processHeaderPoW: Height of block (=%d) is not equal to one greater "+ "than the parent height (=%d)", blockHeader.Height, prevHeight) - return false, false, HeaderErrorHeightInvalid + return nil, false, false, HeaderErrorHeightInvalid } // Make sure the block timestamp is greater than the previous block's timestamp. @@ -2438,7 +2439,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B "before timestamp of previous block %v", time.Unix(int64(blockHeader.GetTstampSecs()), 0), time.Unix(int64(parentHeader.GetTstampSecs()), 0)) - return false, false, HeaderErrorTimestampTooEarly + return nil, false, false, HeaderErrorTimestampTooEarly } // Check that the proof of work beats the difficulty as calculated from @@ -2448,14 +2449,14 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B diffTarget, err := bc.CalcNextDifficultyTarget( parentNode, blockHeader.Version) if err != nil { - return false, false, errors.Wrapf(err, + return nil, false, false, errors.Wrapf(err, "ProcessBlock: Problem computing difficulty "+ "target from parent block %s", hex.EncodeToString(parentNode.Hash[:])) } diffTargetBigint := HashToBigint(diffTarget) blockHashBigint := HashToBigint(headerHash) if diffTargetBigint.Cmp(blockHashBigint) < 0 { - return false, false, + return nil, false, false, errors.Wrapf(HeaderErrorBlockDifficultyAboveTarget, "Target: %v, Actual: %v", diffTarget, headerHash) } @@ -2515,17 +2516,21 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B bc.blockIndex.setHeaderTip(newNode) } - return isMainChain, false, nil + return blockNode, isMainChain, false, nil } // ProcessHeader is a wrapper around processHeaderPoW and processHeaderPoS, which do the leg-work. -func (bc *Blockchain) ProcessHeader(blockHeader *MsgDeSoHeader, headerHash *BlockHash, verifySignatures bool) (_isMainChain bool, _isOrphan bool, _err error) { +func (bc *Blockchain) ProcessHeader( + blockHeader *MsgDeSoHeader, + headerHash *BlockHash, + verifySignatures bool, +) (_blockNode *BlockNode, _isMainChain bool, _isOrphan bool, _err error) { bc.ChainLock.Lock() defer bc.ChainLock.Unlock() if blockHeader == nil { // If the header is nil then we return an error. Nothing we can do here. - return false, false, fmt.Errorf("ProcessHeader: Header is nil") + return nil, false, false, fmt.Errorf("ProcessHeader: Header is nil") } // If the header's height is after the PoS cut-over fork height, then we use the PoS header processing logic. @@ -2644,7 +2649,10 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // first before we do anything. This should create a node and set // the header validation status for it. if !nodeExists { - _, isOrphan, err := bc.processHeaderPoW(blockHeader, blockHash) + // Note: it's okay that we don't write the block node for the header + // to the db here as it happens below when we call + // PutHeightHashToNodeInfo + _, _, isOrphan, err := bc.processHeaderPoW(blockHeader, blockHash) if err != nil { // If an error occurred processing the header, then the header // should be marked as invalid, which should be sufficient. diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 6ce522f7c..be4ffed0e 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -707,7 +707,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockA1.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockA1.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockA1.Header, headerHash, false) require.NoError(err) require.True(isMainChain) require.False(isOrphan) @@ -723,7 +723,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockA2.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockA2.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockA2.Header, headerHash, false) require.NoError(err) require.True(isMainChain) require.False(isOrphan) @@ -739,7 +739,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockB1.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockB1.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockB1.Header, headerHash, false) require.NoError(err) // Should not be main chain yet require.False(isMainChain) @@ -756,7 +756,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockB2.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockB2.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockB2.Header, headerHash, false) require.NoError(err) // Should not be main chain yet require.False(isMainChain) @@ -773,7 +773,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockB3.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockB3.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockB3.Header, headerHash, false) require.NoError(err) // Should not be main chain yet require.True(isMainChain) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index dcfc6e3bf..8e412e50b 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -39,10 +39,10 @@ import ( // 5. Exit early if the's view is less than the current header chain's tip. // 6. Reorg the best header chain if the header's view is higher than the current tip. func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockHash, verifySignatures bool) ( - _isMainChain bool, _isOrphan bool, _err error, + _blockNode *BlockNode, _isMainChain bool, _isOrphan bool, _err error, ) { if !bc.params.IsPoSBlockHeight(header.Height) { - return false, false, errors.Errorf( + return nil, false, false, errors.Errorf( "processHeaderPoS: Header height %d is less than the ProofOfStake2ConsensusCutoverBlockHeight %d", header.Height, bc.params.GetFirstPoSBlockHeight(), ) @@ -53,26 +53,26 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. - // TODO: make sure this is ok or do we need to explicitly check the block index's cache? - _, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) + blockNode, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) if err != nil { - return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") + return nil, false, false, + errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") } if isInBestHeaderChain { - return true, false, nil + return blockNode, true, false, nil } // If the incoming header is part of a reorg that uncommits the committed tip from the best chain, // then we exit early. Such headers are invalid and should not be synced. committedBlockchainTip, _ := bc.GetCommittedTip() if committedBlockchainTip != nil && committedBlockchainTip.Header.Height >= header.Height { - return false, false, errors.New("processHeaderPoS: Header conflicts with committed tip") + return nil, false, false, errors.New("processHeaderPoS: Header conflicts with committed tip") } // Validate the header and index it in the block index. blockNode, isOrphan, err := bc.validateAndIndexHeaderPoS(header, headerHash, verifySignatures) if err != nil { - return false, false, errors.Wrapf(err, "processHeaderPoS: Problem validating and indexing header: ") + return blockNode, false, false, errors.Wrapf(err, "processHeaderPoS: Problem validating and indexing header: ") } // Don't worry about healing orphan children when we're syncing. @@ -84,14 +84,14 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // Exit early if the header is an orphan. if isOrphan { - return false, true, nil + return blockNode, false, true, nil } // Exit early if the header's view is less than the current header chain's tip. The header is not // the new tip for the best header chain. currentTip := bc.headerTip() if header.ProposedInView <= currentTip.Header.ProposedInView { - return false, false, nil + return blockNode, false, false, nil } bc.blockIndex.setHeaderTip(blockNode) @@ -107,7 +107,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH //) // Success. The header is at the tip of the best header chain. - return true, false, nil + return blockNode, true, false, nil } // healPointersForOrphanChildren fixes an inconsistency in the block index that may have @@ -364,7 +364,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // header and applying it to the header chain will result in the two chains being out of // sync. The header chain is less critical and mutations to it are reversible. So we attempt // to mutate it first before attempting to mutate the block chain. - if _, _, err = bc.processHeaderPoS(block.Header, blockHash, verifySignatures); err != nil { + if _, _, _, err = bc.processHeaderPoS(block.Header, blockHash, verifySignatures); err != nil { return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem processing header") } diff --git a/lib/server.go b/lib/server.go index bcf9fa521..c5f25c75c 100644 --- a/lib/server.go +++ b/lib/server.go @@ -364,19 +364,20 @@ func ValidateHyperSyncFlags(isHypersync bool, syncType NodeSyncType) { func RunBlockIndexMigrationOnce(db *badger.DB, params *DeSoParams) error { blockIndexMigrationFileName := filepath.Join(db.Opts().Dir, BlockIndexMigrationFileName) - glog.V(0).Info("FileName: ", blockIndexMigrationFileName) + glog.V(2).Info("FileName: ", blockIndexMigrationFileName) hasRunMigration, err := ReadBoolFromFile(blockIndexMigrationFileName) if err == nil && hasRunMigration { - glog.V(0).Info("Block index migration has already been run") + glog.V(2).Info("Block index migration has already been run") return nil } - glog.V(0).Info("Running block index migration") + glog.V(2).Info("Running block index migration") if err = RunBlockIndexMigration(db, nil, nil, params); err != nil { return errors.Wrapf(err, "Problem running block index migration") } if err = SaveBoolToFile(blockIndexMigrationFileName, true); err != nil { return errors.Wrapf(err, "Problem saving block index migration file") } + glog.V(2).Info("Block index migration complete") return nil } @@ -771,7 +772,21 @@ func (srv *Server) _handleGetHeaders(pp *Peer, msg *MsgDeSoGetHeaders) { if pp.NegotiatedProtocolVersion >= ProtocolVersion2 { maxHeadersPerMsg = MaxHeadersPerMsgPos } - headers := srv.blockchain.LocateBestBlockChainHeaders(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) + // FIXME: We can eliminate the call to LocateBestBlockChainHeaders and do a much + // simpler "shortcut" version that doesn't require complicated tree-traversal bs. + // The shortcut would be to just return all headers starting from msg.BlockLocator[0] + // up to msg.StopHash or maxHeadersPerMsg, whichever comes first. This would allow + // other nodes to sync from us and *keep* in sync with us, while allowing us to delete + // ALL of the complicated logic around locators and the best header chain. This all works + // because msg.BlockLocator[0] is the requesting-node's tip hash. The rest of the + // hashes, and all of the locator bs, are only needed to resolve forks, which can't + // happen with PoS anymore. + //headers := srv.blockchain.LocateBestBlockChainHeaders(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) + + headers, err := srv.GetHeadersForLocatorAndStopHash(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) + if err != nil { + glog.Errorf("Server._handleGetHeadersMessage: Error getting headers: %v", err) + } // Send found headers to the requesting peer. blockTip := srv.blockchain.blockTip() @@ -785,6 +800,51 @@ func (srv *Server) _handleGetHeaders(pp *Peer, msg *MsgDeSoGetHeaders) { headers, blockTip.Hash, blockTip.Height, pp) } +func (srv *Server) GetHeadersForLocatorAndStopHash( + locator []*BlockHash, + stopHash *BlockHash, + maxHeadersPerMsg uint32, +) ([]*MsgDeSoHeader, error) { + var headers []*MsgDeSoHeader + + stopNode, stopNodeExists, stopNodeError := srv.blockchain.GetBlockFromBestChainByHash(stopHash, true) + // Special case when there is no block locator provided but only a stop hash. + if len(locator) == 0 { + if stopNodeError != nil || !stopNodeExists || stopNode == nil { + return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Stop hash provided but no stop node found") + } + return []*MsgDeSoHeader{stopNode.Header}, nil + } + startNode, startNodeExists, startNodeError := srv.blockchain.GetBlockFromBestChainByHash(locator[0], true) + if startNodeError != nil || !startNodeExists || startNode == nil { + return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Start hash provided but no start node found") + } + nextNodeHeight := startNode.Header.Height + 1 + nextNode, nextNodeExists, nextNodeError := srv.blockchain.GetBlockFromBestChainByHeight(nextNodeHeight, true) + if nextNodeError != nil { + return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Error getting start node by height: %v", startNodeError) + } + if !nextNodeExists || nextNode == nil { + return nil, nil + } + for ii := uint32(0); ii < maxHeadersPerMsg; ii++ { + headers = append(headers, nextNode.Header) + if stopNode != nil && nextNode.Hash.IsEqual(stopNode.Hash) { + break + } + nextNode, nextNodeExists, nextNodeError = srv.blockchain.GetBlockFromBestChainByHeight( + nextNode.Header.Height+1, true) + if nextNodeError != nil { + glog.Errorf("Server._handleGetHeadersMessage: Error getting next node by height: %v", nextNodeError) + break + } + if !nextNodeExists || nextNode == nil { + break + } + } + return headers, nil +} + // GetSnapshot is used for sending MsgDeSoGetSnapshot messages to peers. We will // check if the passed peer has been assigned to an in-progress prefix and if so, // we will request a snapshot data chunk from them. Otherwise, we will assign a @@ -1118,6 +1178,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // Start by processing all the headers given to us. They should start // right after the tip of our header chain ideally. While going through them // tally up the number that we actually process. + var blockNodeBatch []*BlockNode for ii, headerReceived := range msg.Headers { // If we've set a maximum height for node sync and we've reached it, // then we will not process any more headers. @@ -1169,7 +1230,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // Process the header, as we haven't seen it before, set verifySignatures to false // if we're in the process of syncing. - _, isOrphan, err := srv.blockchain.ProcessHeader(headerReceived, headerHash, verifySignatures) + blockNode, _, isOrphan, err := srv.blockchain.ProcessHeader(headerReceived, headerHash, verifySignatures) numLogHeaders := 2000 if ii%numLogHeaders == 0 { @@ -1189,9 +1250,29 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { pp, srv.blockchain.chainState(), err, isOrphan) pp.Disconnect("Error processing header") + // Just to be safe, we flush all the headers we just got even tho we have a header. + currTime := time.Now() + if err := PutHeightHashToNodeInfoBatch( + srv.blockchain.db, srv.snapshot, blockNodeBatch, false /*bitcoinNodes*/, srv.eventManager); err != nil { + glog.Errorf("Server._handleHeaderBundle: Problem writing block nodes to db, error: (%v)", err) + return + } + glog.V(0).Info("Server._handleHeaderBundle: PutHeightHashToNodeInfoBatch took: ", time.Since(currTime)) return } + + // Append the block node to the block node batch. + if blockNode != nil { + blockNodeBatch = append(blockNodeBatch, blockNode) + } + } + currTime := time.Now() + if err := PutHeightHashToNodeInfoBatch( + srv.blockchain.db, srv.snapshot, blockNodeBatch, false /*bitcoinNodes*/, srv.eventManager); err != nil { + glog.Errorf("Server._handleHeaderBundle: Problem writing block nodes to db, error: (%v)", err) + return } + glog.V(0).Info("Server._handleHeaderBundle: PutHeightHashToNodeInfoBatch took: ", time.Since(currTime)) // After processing all the headers this will check to see if we are fully current // and send a request to our Peer to start a Mempool sync if so. From 69e18251aa255e1cf95a5e7c996fe2c470f7de60 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 4 Dec 2024 22:55:24 -0500 Subject: [PATCH 48/62] make GetUtxoViewAndUtxoOpsAtBlockHash public --- lib/pos_blockchain.go | 12 ++++++------ lib/pos_blockchain_test.go | 2 +- lib/pos_consensus.go | 4 ++-- lib/state_change_syncer.go | 2 +- lib/txindex.go | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 8e412e50b..7c8e79f4f 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -320,7 +320,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // We expect the utxoView for the parent block to be valid because we check that all ancestor blocks have // been validated. - parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) + parentUtxoViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) if err != nil { // This should never happen. If the parent is validated and extends from the tip, then we should // be able to build a UtxoView for it. This failure can only happen due to transient or badger issues. @@ -824,7 +824,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( return nil, errors.Wrapf(err, "validatePreviouslyIndexedBlockPoS: Problem fetching block from DB") } // Build utxoView for the block's parent. - parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) + parentUtxoViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) if err != nil { // This should never happen. If the parent is validated and extends from the tip, then we should // be able to build a UtxoView for it. This failure can only happen due to transient or badger issues. @@ -1755,7 +1755,7 @@ func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, blockHeight uint64, v return errors.Errorf("commitBlockPoS: Block %v is already committed", blockHash.String()) } // Connect a view up to block we are committing. - utxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockHash, uint64(blockNode.Height)) + utxoViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*blockHash, uint64(blockNode.Height)) if err != nil { return errors.Wrapf(err, "commitBlockPoS: Problem initializing UtxoView: ") } @@ -1931,7 +1931,7 @@ func (viewAndUtxoOps *BlockViewAndUtxoOps) Copy() *BlockViewAndUtxoOps { func (bc *Blockchain) GetUncommittedTipView() (*UtxoView, error) { // Connect the uncommitted blocks to the tip so that we can validate subsequent blocks blockTip := bc.BlockTip() - blockViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockTip.Hash, uint64(blockTip.Height)) + blockViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*blockTip.Hash, uint64(blockTip.Height)) if err != nil { return nil, errors.Wrapf(err, "GetUncommittedTipView: Problem getting UtxoView at block hash") } @@ -1949,7 +1949,7 @@ func (bc *Blockchain) getCachedBlockViewAndUtxoOps(blockHash BlockHash) (*BlockV // all uncommitted ancestors of this block. Then it checks the block view cache to see if we have already // computed this view. If not, connecting the uncommitted ancestor blocks and saving to the cache. The // returned UtxoOps and FullBlock should NOT be modified. -func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, blockHeight uint64) ( +func (bc *Blockchain) GetUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, blockHeight uint64) ( *BlockViewAndUtxoOps, error) { // Always fetch the lineage from the committed tip to the block provided first to // ensure that a valid UtxoView is returned. @@ -1962,7 +1962,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc highestCommittedBlock, exists := bc.GetCommittedTip() glog.Infof("Highest committed block: %v", highestCommittedBlock) if !exists || highestCommittedBlock == nil { - return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: No committed blocks found") + return nil, errors.Errorf("GetUtxoViewAndUtxoOpsAtBlockHash: No committed blocks found") } // If the provided block is committed, we need to make sure it's the committed tip. // Otherwise, we return an error. diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 5c32df53a..75ef9beea 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -2486,7 +2486,7 @@ func _generateRealBlockWithFailingTxn(testMeta *TestMeta, blockHeight uint64, vi prevBlock, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(prevBlockHash, blockHeight-1) require.True(testMeta.t, exists) // Always update the testMeta latestBlockView - latestBlockViewAndUtxoOps, err := testMeta.chain.getUtxoViewAndUtxoOpsAtBlockHash(*prevBlockHash, blockHeight-1) + latestBlockViewAndUtxoOps, err := testMeta.chain.GetUtxoViewAndUtxoOpsAtBlockHash(*prevBlockHash, blockHeight-1) require.NoError(testMeta.t, err) latestBlockView := latestBlockViewAndUtxoOps.UtxoView latestBlockNode, latestBlockNodeExists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(prevBlockHash, blockHeight-1) diff --git a/lib/pos_consensus.go b/lib/pos_consensus.go index a33983240..ab7a73190 100644 --- a/lib/pos_consensus.go +++ b/lib/pos_consensus.go @@ -701,7 +701,7 @@ func (fc *FastHotStuffConsensus) tryProcessBlockAsNewTip(block *MsgDeSoBlock) ([ return nil, errors.Errorf("Error hashing tip block: %v", err) } - utxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*tipBlockHash, tipBlock.Height) + utxoViewAndUtxoOps, err := fc.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*tipBlockHash, tipBlock.Height) if err != nil { return nil, errors.Errorf("Error fetching UtxoView for tip block: %v", err) } @@ -750,7 +750,7 @@ func (fc *FastHotStuffConsensus) produceUnsignedBlockForBlockProposalEvent( } // Build a UtxoView at the parent block - parentUtxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*parentBlockHash, uint64(parentBlock.Height)) + parentUtxoViewAndUtxoOps, err := fc.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*parentBlockHash, uint64(parentBlock.Height)) if err != nil { // This should never happen as long as the parent block is a descendant of the committed tip. return nil, errors.Errorf("Error fetching UtxoView for parent block: %v", parentBlockHash) diff --git a/lib/state_change_syncer.go b/lib/state_change_syncer.go index 358d86294..f81bcf421 100644 --- a/lib/state_change_syncer.go +++ b/lib/state_change_syncer.go @@ -847,7 +847,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser // TODO: Have Z look at if we need to do some caching in the uncommitted blocks logic. // First connect the uncommitted blocks to the mempool view. for _, uncommittedBlock := range uncommittedBlocks { - utxoViewAndOpsAtBlockHash, err := server.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*uncommittedBlock.Hash, uint64(uncommittedBlock.Height)) + utxoViewAndOpsAtBlockHash, err := server.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*uncommittedBlock.Hash, uint64(uncommittedBlock.Height)) if err != nil { mempoolUtxoView.EventManager.stateSyncerFlushed(&StateSyncerFlushedEvent{ FlushId: originalCommittedFlushId, diff --git a/lib/txindex.go b/lib/txindex.go index 29bc0336c..1f5158c1b 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -399,7 +399,7 @@ func (txi *TXIndex) Update() error { utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) if blockToAttach.Header.PrevBlockHash != nil && !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { var utxoViewAndUtxoOps *BlockViewAndUtxoOps - utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) + utxoViewAndUtxoOps, err = txi.TXIndexChain.GetUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) if err != nil { return fmt.Errorf("Update: Problem getting UtxoView at block hash %v: %v", blockToAttach.Header.PrevBlockHash, err) From a859c72e50639e87291bdefce91ab7471cc7734a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 4 Dec 2024 22:59:18 -0500 Subject: [PATCH 49/62] cleanup --- lib/blockchain.go | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 653764400..fde5a0940 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1770,7 +1770,6 @@ func (bc *Blockchain) checkArchivalMode() bool { } firstSnapshotHeight := bc.snapshot.CurrentEpochSnapshotMetadata.FirstSnapshotBlockHeight - _ = firstSnapshotHeight // @diamondhands - can we spot check just a few blocks such as firstSnapshotHeight - 1, // firstSnapshotHeight / 2 - 1, and firstSnapshotHeight / 4 - 1 to see if they are stored? // We take a sampling of blocks to determine if we've downloaded all the blocks up to the first snapshot height. From 01cd500ab7eb573136c2c7e9af2ece2d396acbce Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 4 Dec 2024 23:52:52 -0500 Subject: [PATCH 50/62] change GetBlockFromBestChainByHash to GetBlockFromBestChainByHashAndOptionalHeight to improve performance when height is available --- lib/blockchain.go | 27 +++++++++++++++++++-------- lib/pos_blockchain.go | 2 +- lib/pos_blockchain_test.go | 22 +++++++++++----------- lib/server.go | 20 ++++++-------------- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index fde5a0940..1f2a30ea8 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -454,9 +454,9 @@ func (nn *BlockNode) Ancestor(height uint32, blockIndex *BlockIndex) *BlockNode // height minus provided distance. // // This function is safe for concurrent access. -func (nn *BlockNode) RelativeAncestor(distance uint32, blockIndex *BlockIndex) *BlockNode { - return nn.Ancestor(nn.Height-distance, blockIndex) -} +//func (nn *BlockNode) RelativeAncestor(distance uint32, blockIndex *BlockIndex) *BlockNode { +// return nn.Ancestor(nn.Height-distance, blockIndex) +//} // CalcNextDifficultyTarget computes the difficulty target expected of the // next block. @@ -1215,7 +1215,7 @@ func (bc *Blockchain) locateInventory(locator []*BlockHash, stopHash *BlockHash, // There are no block locators so a specific block is being requested // as identified by the stop hash. - stopNode, stopNodeExists, stopNodeError := bc.GetBlockFromBestChainByHash(stopHash, true) + stopNode, stopNodeExists, stopNodeError := bc.GetBlockFromBestChainByHashAndOptionalHeight(stopHash, nil, true) if len(locator) == 0 { if stopNodeError != nil || !stopNodeExists || stopNode == nil { // TODO: what should we really do here? @@ -1884,8 +1884,19 @@ func (bc *Blockchain) BestChain() []*BlockNode { panic("BestChain not supported.") } -func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeaderChain bool) (*BlockNode, bool, error) { - bn, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) +func (bc *Blockchain) GetBlockFromBestChainByHashAndOptionalHeight( + blockHash *BlockHash, + optionalHeight *uint64, + useHeaderChain bool, +) (*BlockNode, bool, error) { + var bn *BlockNode + var exists bool + var err error + if optionalHeight != nil { + bn, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, *optionalHeight) + } else { + bn, exists, err = bc.blockIndex.GetBlockNodeByHashOnly(blockHash) + } if err != nil { return nil, false, err } @@ -1906,11 +1917,11 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade blockTip = bc.HeaderTip() } if blockTip == nil { - return nil, false, fmt.Errorf("GetBlockFromBestChainByHash: Block tip not found: use header chain: %v", useHeaderChain) + return nil, false, fmt.Errorf("GetBlockFromBestChainByHashAndOptionalHeight: Block tip not found: use header chain: %v", useHeaderChain) } committedTip, exists := bc.GetCommittedTip() if !exists { - return nil, false, errors.New("GetBlockFromBestChainByHash: Committed tip not found") + return nil, false, errors.New("GetBlockFromBestChainByHashAndOptionalHeight: Committed tip not found") } if uint64(bn.Height) > uint64(blockTip.Height) || uint64(bn.Height) < uint64(committedTip.Height) { return nil, false, nil diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 7c8e79f4f..1c959dde5 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -53,7 +53,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. - blockNode, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) + blockNode, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(headerHash, &header.Height, true) if err != nil { return nil, false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 75ef9beea..d3ee5ac1c 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -1338,7 +1338,7 @@ func TestTryApplyNewTip(t *testing.T) { ancestors, _, err := bc.getStoredLineageFromCommittedTip(newBlock.Header) require.NoError(t, err) checkBestChainForHash := func(hash *BlockHash) bool { - _, exists, err := bc.GetBlockFromBestChainByHash(hash, false) + _, exists, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash, nil, false) require.NoError(t, err) return exists } @@ -1353,7 +1353,7 @@ func TestTryApplyNewTip(t *testing.T) { require.NoError(t, err) require.True(t, appliedNewTip) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash3, false) + _, hash3ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash3, nil, false) require.NoError(t, err) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) @@ -1361,14 +1361,14 @@ func TestTryApplyNewTip(t *testing.T) { require.Len(t, disconnectedBlockHashes, 1) // newBlock should be in the best chain and the best chain map and should be the tip. - _, newBlockExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(newBlockHash, false) + _, newBlockExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(newBlockHash, nil, false) require.NoError(t, err) require.True(t, newBlockExistsInBestChainMap) require.True(t, checkBestChainForHash(newBlockHash)) require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // Make sure block 2 and block 1 are still in the best chain. - _, hash2ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash2, false) + _, hash2ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash2, nil, false) require.NoError(t, err) require.True(t, hash2ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash2)) @@ -1428,22 +1428,22 @@ func TestTryApplyNewTip(t *testing.T) { // newBlockHash should be tip. require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHash(hash3, false) + _, hash3ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHashAndOptionalHeight(hash3, nil, false) require.NoError(t, err) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) // hash 2 should no longer be in the best chain or best chain map - _, hash2ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHash(hash2, false) + _, hash2ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHashAndOptionalHeight(hash2, nil, false) require.NoError(t, err) require.False(t, hash2ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash2)) // hash 4 should be in the best chain and the best chain map - _, hash4ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash4, false) + _, hash4ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash4, nil, false) require.NoError(t, err) require.True(t, hash4ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash4)) // hash 5 should be in the best chain and the best chain map - _, hash5ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash5, false) + _, hash5ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash5, nil, false) require.NoError(t, err) require.True(t, hash5ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash5)) @@ -1670,7 +1670,7 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, committedHash := range committedBlocks { // Okay so let's make sure the block is committed. - blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHash(committedHash, false) + blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHashAndOptionalHeight(committedHash, nil, false) require.NoError(testMeta.t, err) require.True(testMeta.t, exists) require.True(testMeta.t, blockNode.IsCommitted()) @@ -1696,7 +1696,7 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, uncommittedBlockHash := range uncommittedBlocks { // Okay so let's make sure the block is uncommitted. - blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHash(uncommittedBlockHash, false) + blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHashAndOptionalHeight(uncommittedBlockHash, nil, false) require.NoError(testMeta.t, err) require.True(testMeta.t, exists) require.False(testMeta.t, blockNode.IsCommitted()) @@ -1889,7 +1889,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { // Timeout block will no longer be in best chain, and will still be in an uncommitted state in the block index _verifyCommitRuleHelper(testMeta, []*BlockHash{blockHash1, blockHash2}, []*BlockHash{blockHash3, reorgBlockHash}, blockHash2) _verifyRandomSeedHashHelper(testMeta, reorgBlock) - _, exists, err := testMeta.chain.GetBlockFromBestChainByHash(timeoutBlockHash, false) + _, exists, err := testMeta.chain.GetBlockFromBestChainByHashAndOptionalHeight(timeoutBlockHash, nil, false) require.NoError(t, err) require.False(t, exists) diff --git a/lib/server.go b/lib/server.go index c5f25c75c..179660029 100644 --- a/lib/server.go +++ b/lib/server.go @@ -807,7 +807,7 @@ func (srv *Server) GetHeadersForLocatorAndStopHash( ) ([]*MsgDeSoHeader, error) { var headers []*MsgDeSoHeader - stopNode, stopNodeExists, stopNodeError := srv.blockchain.GetBlockFromBestChainByHash(stopHash, true) + stopNode, stopNodeExists, stopNodeError := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight(stopHash, nil, true) // Special case when there is no block locator provided but only a stop hash. if len(locator) == 0 { if stopNodeError != nil || !stopNodeExists || stopNode == nil { @@ -815,7 +815,7 @@ func (srv *Server) GetHeadersForLocatorAndStopHash( } return []*MsgDeSoHeader{stopNode.Header}, nil } - startNode, startNodeExists, startNodeError := srv.blockchain.GetBlockFromBestChainByHash(locator[0], true) + startNode, startNodeExists, startNodeError := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight(locator[0], nil, true) if startNodeError != nil || !startNodeExists || startNode == nil { return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Start hash provided but no start node found") } @@ -1108,18 +1108,10 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b if checkpointBlockInfo == nil { return true, false } - var hasSeenCheckpointBlockHash bool - var checkpointBlockNode *BlockNode - var err error srv.blockchain.ChainLock.RLock() defer srv.blockchain.ChainLock.RUnlock() - if isHeaderChain { - checkpointBlockNode, hasSeenCheckpointBlockHash, err = srv.blockchain.GetBlockFromBestChainByHash( - checkpointBlockInfo.Hash, true) - } else { - checkpointBlockNode, hasSeenCheckpointBlockHash, err = srv.blockchain.GetBlockFromBestChainByHash( - checkpointBlockInfo.Hash, false) - } + checkpointBlockNode, hasSeenCheckpointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight( + checkpointBlockInfo.Hash, &checkpointBlockInfo.Height, isHeaderChain) if err != nil { glog.Fatalf("shouldVerifySignatures: Problem getting checkpoint block node from best chain: %v", err) } @@ -1149,8 +1141,8 @@ func (srv *Server) getCheckpointSyncingStatus(isHeaders bool) string { if checkpointBlockInfo == nil { return "" } - _, hasSeenCheckPointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHash( - checkpointBlockInfo.Hash, isHeaders) + _, hasSeenCheckPointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight( + checkpointBlockInfo.Hash, &checkpointBlockInfo.Height, isHeaders) if err != nil { glog.Fatalf("getCheckpointSyncingStatus: Problem getting checkpoint block node from best chain: %v", err) From 09091df5e447224ad222f8703fce30b6157f17ae Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 00:01:35 -0500 Subject: [PATCH 51/62] use GetBlockFromBestChainByHashAndOptionalHeight in GetBlockNodesToFetch --- lib/blockchain.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 1f2a30ea8..40d958929 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1482,7 +1482,9 @@ func (bc *Blockchain) GetBlockNodesToFetch( // If the tip of the best block chain is in the main header chain, make that // the start point for our fetch. - headerNodeStart, blockTipExistsInBestHeaderChain, err := bc.GetBlockFromBestChainByHeight(uint64(bestBlockTip.Height), true) + bestBlockTipHeight := uint64(bestBlockTip.Height) + headerNodeStart, blockTipExistsInBestHeaderChain, err := bc.GetBlockFromBestChainByHashAndOptionalHeight( + bestBlockTip.Hash, &bestBlockTipHeight, true) if err != nil { glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) return nil From 77a1f57e8fe20c656ca547c2327ba53251106b47 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 00:32:47 -0500 Subject: [PATCH 52/62] nuke locator and inventory logic, just return a slice of the tip hash --- lib/blockchain.go | 287 +--------------------------------------------- lib/server.go | 15 +-- 2 files changed, 8 insertions(+), 294 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 40d958929..6cc75255b 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1174,289 +1174,8 @@ func NewBlockchain( return bc, nil } -// log2FloorMasks defines the masks to use when quickly calculating -// floor(log2(x)) in a constant log2(32) = 5 steps, where x is a uint32, using -// shifts. They are derived from (2^(2^x) - 1) * (2^(2^x)), for x in 4..0. -var log2FloorMasks = []uint32{0xffff0000, 0xff00, 0xf0, 0xc, 0x2} - -// fastLog2Floor calculates and returns floor(log2(x)) in a constant 5 steps. -func fastLog2Floor(n uint32) uint8 { - rv := uint8(0) - exponent := uint8(16) - for i := 0; i < 5; i++ { - if n&log2FloorMasks[i] != 0 { - rv += exponent - n >>= exponent - } - exponent >>= 1 - } - return rv -} - -// TODO: deprecate -// locateInventory returns the node of the block after the first known block in -// the locator along with the number of subsequent nodes needed to either reach -// the provided stop hash or the provided max number of entries. -// -// In addition, there are two special cases: -// -// - When no locators are provided, the stop hash is treated as a request for -// that block, so it will either return the node associated with the stop hash -// if it is known, or nil if it is unknown -// - When locators are provided, but none of them are known, nodes starting -// after the genesis block will be returned -// -// This is primarily a helper function for the locateBlocks and locateHeaders -// functions. -// -// This function MUST be called with the chain state lock held (for reads). -// TODO: this function needs a whole bunch of work. -func (bc *Blockchain) locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32) (*BlockNode, uint32) { - - // There are no block locators so a specific block is being requested - // as identified by the stop hash. - stopNode, stopNodeExists, stopNodeError := bc.GetBlockFromBestChainByHashAndOptionalHeight(stopHash, nil, true) - if len(locator) == 0 { - if stopNodeError != nil || !stopNodeExists || stopNode == nil { - // TODO: what should we really do here? - glog.Errorf("locateInventory: Block %v is not known", stopHash) - // No blocks with the stop hash were found so there is - // nothing to do. - return nil, 0 - } - return stopNode, 1 - } - - // Find the most recent locator block hash in the main chain. In the - // case none of the hashes in the locator are in the main chain, fall - // back to the genesis block. - startNode, startNodeExists, err := bc.GetBlockFromBestChainByHeight(0, true) - if err != nil { - glog.Errorf("locateInventory: Problem getting block by height: %v", err) - return nil, 0 - } - if !startNodeExists { - glog.Errorf("locateInventory: Genesis block not found") - return nil, 0 - } - for _, hash := range locator { - // TODO: replace w/ read-through cache call. - node := bc.GetBlockNodeWithHash(hash) - if node != nil { - startNode = node - break - } - } - - // Start at the block after the most recently known block. When there - // is no next block it means the most recently known block is the tip of - // the best chain, so there is nothing more to do. - nextNodeHeight := uint32(startNode.Header.Height) + 1 - startNode, startNodeExists, err = bc.GetBlockFromBestChainByHeight(uint64(nextNodeHeight), true) - if err != nil { - glog.Errorf("locateInventory: Problem getting block by height: %v", err) - return nil, 0 - } - if !startNodeExists { - return nil, 0 - } - - // Calculate how many entries are needed. - total := (bc.blockIndex.GetTip().Height - startNode.Height) + 1 - if stopNodeError != nil && stopNodeExists && stopNode != nil && - stopNode.Header.Height >= startNode.Header.Height { - - _, bestChainContainsStopNode, err := bc.blockIndex.GetBlockNodeByHashOnly(stopNode.Hash) - if err != nil { - glog.Errorf("locateInventory: Problem getting block by hash: %v", err) - return nil, 0 - } - if bestChainContainsStopNode { - total = uint32((stopNode.Header.Height - startNode.Header.Height) + 1) - } - } - if total > maxEntries { - total = maxEntries - } - - return startNode, total -} - -// TODO: deprecate -// locateHeaders returns the headers of the blocks after the first known block -// in the locator until the provided stop hash is reached, or up to the provided -// max number of block headers. -// -// See the comment on the exported function for more details on special cases. -// -// This function MUST be called with the ChainLock held (for reads). -func (bc *Blockchain) locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32) []*MsgDeSoHeader { - - // Find the node after the first known block in the locator and the - // total number of nodes after it needed while respecting the stop hash - // and max entries. - node, total := bc.locateInventory(locator, stopHash, maxHeaders) - if total == 0 { - return nil - } - - // Populate and return the found headers. - headers, err := SafeMakeSliceWithLengthAndCapacity[*MsgDeSoHeader](0, uint64(total)) - if err != nil { - // TODO: do we really want to introduce an error here? - } - for ii := uint32(0); ii < total; ii++ { - headers = append(headers, node.Header) - if uint32(len(headers)) == total { - break - } - var nodeExists bool - node, nodeExists, err = bc.GetBlockFromBestChainByHeight(node.Header.Height+1, true) - if err != nil { - glog.Errorf("locateHeaders: Problem getting block by height: %v", err) - break - } - if !nodeExists { - break - } - } - return headers -} - -// TODO: deprecate -// LocateBestBlockChainHeaders returns the headers of the blocks after the first known block -// in the locator until the provided stop hash is reached, or up to a max of -// wire.MaxBlockHeadersPerMsg headers. Note that it returns the best headers -// considering only headers for which we have blocks (that is, it considers the -// best *block* chain we have rather than the best *header* chain). This is -// the correct thing to do because in general this function is called in order -// to serve a response to a peer's GetHeaders request. -// -// In addition, there are two special cases: -// -// - When no locators are provided, the stop hash is treated as a request for -// that header, so it will either return the header for the stop hash itself -// if it is known, or nil if it is unknown -// - When locators are provided, but none of them are known, headers starting -// after the genesis block will be returned -// -// This function is safe for concurrent access. -func (bc *Blockchain) LocateBestBlockChainHeaders( - locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32) []*MsgDeSoHeader { - - bc.ChainLock.RLock() - defer bc.ChainLock.RUnlock() - headers := bc.locateHeaders(locator, stopHash, maxHeaders) - - return headers -} - -// LatestLocator returns a block locator for the passed block node. The passed -// node can be nil in which case the block locator for the current tip -// associated with the view will be returned. -// -// BlockLocator is used to help locate a specific block. The algorithm for -// building the block locator is to add the hashes in reverse order until -// the genesis block is reached. In order to keep the list of locator hashes -// to a reasonable number of entries, first the most recent previous 12 block -// hashes are added, then the step is doubled each loop iteration to -// exponentially decrease the number of hashes as a function of the distance -// from the block being located. -// -// For example, assume a block chain with a side chain as depicted below: -// -// genesis -> 1 -> 2 -> ... -> 15 -> 16 -> 17 -> 18 -// \-> 16a -> 17a -// -// The block locator for block 17a would be the hashes of blocks: -// [17a 16a 15 14 13 12 11 10 9 8 7 6 4 genesis] -// -// Caller is responsible for acquiring the ChainLock before calling this function. -func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { - - // Calculate the max number of entries that will ultimately be in the - // block locator. See the description of the algorithm for how these - // numbers are derived. - var maxEntries uint8 - if tip.Header.Height <= 12 { - maxEntries = uint8(tip.Header.Height) + 1 - } else { - // Requested hash itself + previous 10 entries + genesis block. - // Then floor(log2(height-10)) entries for the skip portion. - adjustedHeight := uint32(tip.Header.Height) - 10 - maxEntries = 12 + fastLog2Floor(adjustedHeight) - } - locator := make([]*BlockHash, 0, maxEntries) - - step := int32(1) - for tip != nil { - locator = append(locator, tip.Hash) - - // Nothing more to add once the genesis block has been added. - if tip.Header.Height == 0 { - break - } - - // Calculate height of previous node to include ensuring the - // final node is the genesis block. - height := int32(tip.Header.Height) - step - if height < 0 { - height = 0 - } - - // When the node is in the current chain view, all of its - // ancestors must be too, so use a much faster O(1) lookup in - // that case. Otherwise, fall back to walking backwards through - // the nodes of the other chain to the correct ancestor. - _, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(tip.Hash) - if err != nil { - glog.Errorf("LatestLocator: Problem getting block by hash: %v", err) - exists = false - } - if exists { - var innerExists bool - tip, innerExists, err = bc.GetBlockFromBestChainByHeight(uint64(height), true) - if err != nil { - glog.Errorf("LatestLocator: Problem getting block by height: %v", err) - break - } - if !innerExists { - glog.Errorf("LatestLocator: Block %v not found in best header chain", height) - break - } - } else { - tip = tip.Ancestor(uint32(height), bc.blockIndex) - } - - // Once 11 entries have been included, start doubling the - // distance between included hashes. - if len(locator) > 10 { - step *= 2 - } - } - - return locator -} - -func (bc *Blockchain) HeaderLocatorWithNodeHash(blockHash *BlockHash) ([]*BlockHash, error) { - // We can acquire the ChainLock because the only place this is called currently is from - // _handleHeaderBundle, which doesn't have the lock. - // If we do not acquire the lock, we may hit a concurrent map read write error which causes panic. - bc.ChainLock.RLock() - defer bc.ChainLock.RUnlock() - node, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) - if err != nil { - return nil, fmt.Errorf("Blockchain.HeaderLocatorWithNodeHash: Problem getting node for hash %v: %v", blockHash, err) - } - if !exists || node == nil { - return nil, fmt.Errorf("Blockchain.HeaderLocatorWithNodeHash: Node for hash %v is not in our blockIndexByHash", blockHash) - } - - return bc.LatestLocator(node), nil -} - -// LatestHeaderLocator calls LatestLocator in order to fetch a locator -// for the best header chain. +// LatestHeaderLocator calls returns a block locator for the current tip of the +// header chain. func (bc *Blockchain) LatestHeaderLocator() []*BlockHash { // We can acquire the ChainLock here because all calls to this function happen in peer.go // and server.go, which don't hold the lock. @@ -1465,7 +1184,7 @@ func (bc *Blockchain) LatestHeaderLocator() []*BlockHash { defer bc.ChainLock.RUnlock() headerTip := bc.headerTip() - return bc.LatestLocator(headerTip) + return []*BlockHash{headerTip.Hash} } func (bc *Blockchain) GetBlockNodesToFetch( diff --git a/lib/server.go b/lib/server.go index 179660029..11154b265 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1124,6 +1124,9 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b } return false, false } + // TODO: @diamondhands - why can't we move this up in this function? It seems like we can avoid + // checking if we have the checkpoint block node if the header we're processing is below the height. + // This will save us 17-18% of the time it takes to process headers. // If the current header has a height below the checkpoint block height, we should skip signature verification // even if we've seen the checkpoint block hash. if header.Height < checkpointBlockInfo.Height { @@ -1210,6 +1213,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // If we get here then we have a header we haven't seen before. // check if we need to verify signatures + // TODO: we can add some logic into should verify signatures to avoid trying to get the checkpoint block node. verifySignatures, shouldDisconnect := srv.shouldVerifySignatures(headerReceived, true) if shouldDisconnect { glog.Errorf("Server._handleHeaderBundle: Disconnecting peer %v in state %s because a mismatch was "+ @@ -1476,18 +1480,9 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // even if the peer has a long fork with more work than our current header // chain. lastHash, _ := msg.Headers[len(msg.Headers)-1].Hash() - locator, err := srv.blockchain.HeaderLocatorWithNodeHash(lastHash) - if err != nil { - glog.Warningf("Server._handleHeaderBundle: Disconnecting peer %v because "+ - "she indicated that she has more headers but the last hash %v in "+ - "the header bundle does not correspond to a block in our index.", - pp, lastHash) - pp.Disconnect("Last hash in header bundle not in our index") - return - } pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, - BlockLocator: locator, + BlockLocator: []*BlockHash{lastHash}, }, false) headerTip := srv.blockchain.headerTip() glog.V(1).Infof("Server._handleHeaderBundle: *Syncing* headers for blocks starting at "+ From 0585ebc1a7464c87181093953a18d5e4d8ef747e Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 00:35:11 -0500 Subject: [PATCH 53/62] move check in should verify signatures up in function to reduce amount of times we need to fetch a block node we clearly know isn't there --- lib/server.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/server.go b/lib/server.go index 11154b265..69d376591 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1108,6 +1108,14 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b if checkpointBlockInfo == nil { return true, false } + // TODO: @diamondhands - why can't we move this up in this function? It seems like we can avoid + // checking if we have the checkpoint block node if the header we're processing is below the height. + // This will save us 17-18% of the time it takes to process headers. + // If the current header has a height below the checkpoint block height, we should skip signature verification + // even if we've seen the checkpoint block hash. + if header.Height < checkpointBlockInfo.Height { + return false, false + } srv.blockchain.ChainLock.RLock() defer srv.blockchain.ChainLock.RUnlock() checkpointBlockNode, hasSeenCheckpointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight( @@ -1124,14 +1132,6 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b } return false, false } - // TODO: @diamondhands - why can't we move this up in this function? It seems like we can avoid - // checking if we have the checkpoint block node if the header we're processing is below the height. - // This will save us 17-18% of the time it takes to process headers. - // If the current header has a height below the checkpoint block height, we should skip signature verification - // even if we've seen the checkpoint block hash. - if header.Height < checkpointBlockInfo.Height { - return false, false - } // Make sure that the header in the best chain map has the correct height, otherwise we need to disconnect this peer. if uint64(checkpointBlockNode.Height) != checkpointBlockInfo.Height { return true, true From 139fb2ca0da00c0daf7f0e0e1bfe2f9596404617 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 00:41:02 -0500 Subject: [PATCH 54/62] reduce size of block index cache --- lib/blockchain.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 6cc75255b..6e9457ae9 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -59,7 +59,9 @@ const ( // have room for multiple forks each an entire history's length with this value). If // each node takes up 100 bytes of space this amounts to around 500MB, which also seems // like a reasonable size. - MaxBlockIndexNodes = 50000000 // TODO: trim this down somehow... + // UPDATE: now that we don't keep everything in memory, we reduced this value from + // 50000000 to 1000000 + MaxBlockIndexNodes = 1000000 // TODO: trim this down somehow... ) type BlockStatus uint32 From 4259c393b406419bc7be0cdef2f5a5a60e96b70c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 11:57:49 -0500 Subject: [PATCH 55/62] use header chain for get blocks to store --- lib/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/server.go b/lib/server.go index 69d376591..9b2a4b592 100644 --- a/lib/server.go +++ b/lib/server.go @@ -941,7 +941,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { // Go through the block nodes in the blockchain and download the blocks if they're not stored. for ii := uint32(srv.blockchain.lowestBlockNotStored); ii <= srv.blockchain.blockTip().Height; ii++ { - blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(ii), false) + blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(ii), true) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) return @@ -1248,7 +1248,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { pp.Disconnect("Error processing header") // Just to be safe, we flush all the headers we just got even tho we have a header. currTime := time.Now() - if err := PutHeightHashToNodeInfoBatch( + if err = PutHeightHashToNodeInfoBatch( srv.blockchain.db, srv.snapshot, blockNodeBatch, false /*bitcoinNodes*/, srv.eventManager); err != nil { glog.Errorf("Server._handleHeaderBundle: Problem writing block nodes to db, error: (%v)", err) return From aea51a8c129b60640beb3a14e3aee65a723ecb3a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 14:08:23 -0500 Subject: [PATCH 56/62] fix return value from processHeaderPoW --- lib/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 6e9457ae9..231580b33 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2249,7 +2249,7 @@ func (bc *Blockchain) processHeaderPoW( bc.blockIndex.setHeaderTip(newNode) } - return blockNode, isMainChain, false, nil + return newNode, isMainChain, false, nil } // ProcessHeader is a wrapper around processHeaderPoW and processHeaderPoS, which do the leg-work. From fc63a94af39875b92f98149f758d94ce741be5a7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 15:04:28 -0500 Subject: [PATCH 57/62] disable deadlock detection around putting block nodes into db after handling snapshot --- lib/server.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/server.go b/lib/server.go index 9b2a4b592..6099dc5b7 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1826,6 +1826,13 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { // being too large and possibly causing an error in badger. glog.V(0).Infof("Server._handleSnapshot: Updating snapshot block nodes in the database") var blockNodeBatch []*BlockNode + flushBlockNodeStartTime := time.Now() + // Disable deadlock detection, as the process of flushing entries to file can take a long time and + // if it takes longer than the deadlock detection timeout interval, it will cause an error to be thrown. + deadlock.Opts.Disable = true + defer func() { + deadlock.Opts.Disable = false + }() // acquire the chain lock while we update the best chain and best chain map. srv.blockchain.ChainLock.Lock() for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { @@ -1861,6 +1868,8 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { glog.Errorf("Server._handleSnapshot: Problem updating snapshot block nodes, error: (%v)", err) } } + glog.V(0).Infof("Time to store %v block nodes in the database: %v", + srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight, time.Since(flushBlockNodeStartTime)) err = PutBestHash(srv.blockchain.db, srv.snapshot, msg.SnapshotMetadata.CurrentEpochBlockHash, ChainTypeDeSoBlock, srv.eventManager) if err != nil { From 90b5372b68f1d0ae81d0c6b432f2bd8543f65ba6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 15:17:39 -0500 Subject: [PATCH 58/62] add more logging as we're processing headers after finishing downloading snapshot --- lib/server.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/server.go b/lib/server.go index 6099dc5b7..35449a593 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1852,6 +1852,14 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) srv.blockchain.blockIndex.setTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) + if currentNode.Height%100000 == 0 { + glog.V(0).Infof("Time to process %v of %v block nodes in %v", + currentNode.Height, + srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight, + time.Since(flushBlockNodeStartTime), + ) + } + // TODO: should we adjust this value for batch sizes? if len(blockNodeBatch) < 10000 { continue } From f9a4f3e3b5ae34008e70cede9e59b955546dc39a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 16:20:01 -0500 Subject: [PATCH 59/62] iterate backwards to update status of block nodes after finishing hypersync --- lib/server.go | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/lib/server.go b/lib/server.go index 35449a593..399ef4d23 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1835,16 +1835,19 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { }() // acquire the chain lock while we update the best chain and best chain map. srv.blockchain.ChainLock.Lock() - for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { - currentNode, currentNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(ii, true) - if err != nil { - glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) - break - } - if !currentNodeExists { - glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") - break - } + // TODO: we should iterate in reverse so we can use GetBlockFromBestChainByHashAndOptionalHeight + // by doing currentNode.Height - 1 and currentNode.Header.PrevBlockHash. + currentNode, currentNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight, true) + if err != nil { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) + // TODO: should we return here? + } + if !currentNodeExists { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") + // TODO: should we return here? + } + for currentNode.Height > 0 { + //for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { // Do not set the StatusBlockStored flag, because we still need to download the past blocks. currentNode.Status |= StatusBlockProcessed currentNode.Status |= StatusBlockValidated @@ -1852,13 +1855,24 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) srv.blockchain.blockIndex.setTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) - if currentNode.Height%100000 == 0 { + if (srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight-uint64(currentNode.Height))%100000 == 0 { glog.V(0).Infof("Time to process %v of %v block nodes in %v", - currentNode.Height, + srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight-uint64(currentNode.Height), srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight, time.Since(flushBlockNodeStartTime), ) } + + prevNodeHeight := uint64(currentNode.Height) - 1 + currentNode, currentNodeExists, err = srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight(currentNode.Header.PrevBlockHash, &prevNodeHeight, true) + if err != nil { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) + break + } + if !currentNodeExists { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") + break + } // TODO: should we adjust this value for batch sizes? if len(blockNodeBatch) < 10000 { continue From f425317c89e173057c9c4d70ccb4403290b46614 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 6 Dec 2024 15:03:46 -0500 Subject: [PATCH 60/62] fix set tip call since we reverse backwards through the block nodes at the end of hypersync --- lib/server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/server.go b/lib/server.go index 399ef4d23..0c913fd95 100644 --- a/lib/server.go +++ b/lib/server.go @@ -972,7 +972,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to // the array index. - currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(currentHeight, false) + currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(currentHeight, true) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) return @@ -1846,6 +1846,8 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") // TODO: should we return here? } + // Set the block tip to the snapshot height block node. + srv.blockchain.blockIndex.setTip(currentNode) for currentNode.Height > 0 { //for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { // Do not set the StatusBlockStored flag, because we still need to download the past blocks. @@ -1853,7 +1855,6 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { currentNode.Status |= StatusBlockValidated currentNode.Status |= StatusBlockCommitted srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) - srv.blockchain.blockIndex.setTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) if (srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight-uint64(currentNode.Height))%100000 == 0 { glog.V(0).Infof("Time to process %v of %v block nodes in %v", From 231a6dc398e8bf47bb7c8de7800bdb2387c6a581 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 9 Dec 2024 12:09:17 -0500 Subject: [PATCH 61/62] small clean up, add some todos --- cmd/config.go | 1 - lib/server.go | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/config.go b/cmd/config.go index fe04cbac7..7a5278900 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -266,7 +266,6 @@ func (config *Config) Print() { glog.Infof("MaxSyncBlockHeight: %v", config.MaxSyncBlockHeight) } - glog.Infof("Connect IPs: %s", config.ConnectIPs) if len(config.ConnectIPs) > 0 { glog.Infof("Connect IPs: %s", config.ConnectIPs) } diff --git a/lib/server.go b/lib/server.go index 0c913fd95..2332d0f89 100644 --- a/lib/server.go +++ b/lib/server.go @@ -941,6 +941,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { // Go through the block nodes in the blockchain and download the blocks if they're not stored. for ii := uint32(srv.blockchain.lowestBlockNotStored); ii <= srv.blockchain.blockTip().Height; ii++ { + // TODO: this may be really slow. blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(ii), true) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) @@ -972,6 +973,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to // the array index. + // TODO: this may be really slow. currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(currentHeight, true) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) From 8eb3e941c1235bf0b2f7a9847e5330ca38763d45 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 9 Dec 2024 13:51:09 -0500 Subject: [PATCH 62/62] call hash less often to speed up block processing --- lib/block_view_lockups_test.go | 4 +-- lib/blockchain.go | 36 ++++++++++++++++++++------- lib/blockchain_test.go | 16 ++++++------ lib/db_utils.go | 45 +++++++++++++++++++++++----------- lib/load_test.go | 4 +-- lib/miner.go | 2 +- lib/pos_blockchain.go | 42 ++++++++++++++++++++----------- lib/pos_blockchain_test.go | 20 +++++++-------- lib/pos_consensus.go | 16 +++++++++--- lib/server.go | 6 ++--- lib/txindex.go | 2 +- 11 files changed, 125 insertions(+), 68 deletions(-) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index 48f435363..d542f3eef 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2353,7 +2353,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { // Process the first block err = testMeta.miner.BlockProducer.SignBlock(blk1) require.NoError(t, err) - _, _, _, err = testMeta.chain.ProcessBlock(blk1, false) + _, _, _, err = testMeta.chain.ProcessBlock(blk1, nil, false) require.NoError(t, err) // Validate state update @@ -2417,7 +2417,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { // Process the second block err = testMeta.miner.BlockProducer.SignBlock(blk2) require.NoError(t, err) - _, _, _, err = testMeta.chain.ProcessBlock(blk2, false) + _, _, _, err = testMeta.chain.ProcessBlock(blk2, nil, false) require.NoError(t, err) // Validate state update diff --git a/lib/blockchain.go b/lib/blockchain.go index 231580b33..fe9fd6eda 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2275,7 +2275,11 @@ func (bc *Blockchain) ProcessHeader( return bc.processHeaderPoW(blockHeader, headerHash) } -func (bc *Blockchain) ProcessBlock(desoBlock *MsgDeSoBlock, verifySignatures bool) (_isMainChain bool, _isOrphan bool, _missingBlockHashes []*BlockHash, _err error) { +func (bc *Blockchain) ProcessBlock( + desoBlock *MsgDeSoBlock, + hash *BlockHash, // hash is not required and will be computed if not provided, but speeds things up if provided. + verifySignatures bool, +) (_isMainChain bool, _isOrphan bool, _missingBlockHashes []*BlockHash, _err error) { bc.ChainLock.Lock() defer bc.ChainLock.Unlock() @@ -2284,17 +2288,29 @@ func (bc *Blockchain) ProcessBlock(desoBlock *MsgDeSoBlock, verifySignatures boo return false, false, nil, fmt.Errorf("ProcessBlock: Block is nil") } + if hash == nil { + var err error + hash, err = desoBlock.Hash() + if err != nil { + return false, false, nil, errors.Wrapf(err, "ProcessBlock: Problem computing block hash") + } + } + // If the block's height is after the PoS cut-over fork height, then we use the PoS block processing logic. // Otherwise, fall back to the PoW logic. if bc.params.IsPoSBlockHeight(desoBlock.Header.Height) { - return bc.processBlockPoS(desoBlock, 1, verifySignatures) + return bc.processBlockPoS(desoBlock, hash, 1, verifySignatures) } - isMainChain, isOrphan, err := bc.processBlockPoW(desoBlock, verifySignatures) + isMainChain, isOrphan, err := bc.processBlockPoW(desoBlock, hash, verifySignatures) return isMainChain, isOrphan, nil, err } -func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bool) (_isMainChain bool, _isOrphan bool, err error) { +func (bc *Blockchain) processBlockPoW( + desoBlock *MsgDeSoBlock, + blockHash *BlockHash, // hash is not required and will be computed if not provided, but speeds things up if provided. + verifySignatures bool, +) (_isMainChain bool, _isOrphan bool, err error) { // Only accept the block if its height is below the PoS cutover height. if !bc.params.IsPoWBlockHeight(desoBlock.Header.Height) { return false, false, RuleErrorBlockHeightAfterProofOfStakeCutover @@ -2308,9 +2324,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures if blockHeader == nil { return false, false, fmt.Errorf("ProcessBlock: Block header was nil") } - blockHash, err := blockHeader.Hash() - if err != nil { - return false, false, errors.Wrapf(err, "ProcessBlock: Problem computing block hash") + if blockHash == nil { + blockHash, err = blockHeader.Hash() + if err != nil { + return false, false, errors.Wrapf(err, "ProcessBlock: Problem computing block hash") + } } // If a trusted block producer public key is set, then we only accept blocks // if they have been signed by one of these public keys. @@ -2545,7 +2563,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // This is needed for disconnects, otherwise GetBlock() will fail (e.g. when we reorg). if err == nil { err = bc.db.Update(func(txn *badger.Txn) error { - if err := PutBlockWithTxn(txn, nil, desoBlock, bc.eventManager); err != nil { + if err := PutBlockWithTxn(txn, nil, desoBlock, blockHash, bc.eventManager); err != nil { return errors.Wrapf(err, "ProcessBlock: Problem putting block with txns") } return nil @@ -2564,7 +2582,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // set in PutBlockWithTxn. Block rewards are part of the state, and they should be identical to the ones // we've fetched during Hypersync. Is there an edge-case where for some reason they're not identical? Or // somehow ancestral records get corrupted? - if innerErr := PutBlockWithTxn(txn, bc.snapshot, desoBlock, bc.eventManager); innerErr != nil { + if innerErr := PutBlockWithTxn(txn, bc.snapshot, desoBlock, blockHash, bc.eventManager); innerErr != nil { return errors.Wrapf(err, "ProcessBlock: Problem calling PutBlock") } diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index be4ffed0e..fb0bed9ca 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -618,7 +618,7 @@ func TestBasicTransferReorg(t *testing.T) { // Process all of the fork blocks on the original chain to make it // experience a reorg. for _, forkBlock := range forkBlocks { - _, _, _, err := chain1.ProcessBlock(forkBlock, true /*verifySignatures*/) + _, _, _, err := chain1.ProcessBlock(forkBlock, nil, true /*verifySignatures*/) require.NoError(err) } @@ -661,7 +661,7 @@ func _shouldConnectBlock(blk *MsgDeSoBlock, t *testing.T, chain *Blockchain) { blockHash, _ := blk.Hash() verifySignatures := true - isMainChain, isOrphan, _, err := chain.ProcessBlock(blk, verifySignatures) + isMainChain, isOrphan, _, err := chain.ProcessBlock(blk, nil, verifySignatures) require.NoError(err) require.Falsef(isOrphan, "Block %v should not be an orphan", blockHash) require.Truef(isMainChain, "Block %v should be on the main chain", blockHash) @@ -826,7 +826,7 @@ func TestProcessBlockReorgBlocks(t *testing.T) { // Block b1 fmt.Println("Connecting block b1") require.Equal(uint64(3), GetUtxoNumEntries(db, chain.snapshot)) - isMainChain, isOrphan, _, err := chain.ProcessBlock(blockB1, verifySignatures) + isMainChain, isOrphan, _, err := chain.ProcessBlock(blockB1, nil, verifySignatures) require.NoError(err) require.Falsef(isOrphan, "Block b1 should not be an orphan") require.Falsef(isMainChain, "Block b1 should not be on the main chain") @@ -842,7 +842,7 @@ func TestProcessBlockReorgBlocks(t *testing.T) { // Block b2 fmt.Println("Connecting block b2") require.Equal(uint64(3), GetUtxoNumEntries(db, chain.snapshot)) - isMainChain, isOrphan, _, err := chain.ProcessBlock(blockB2, verifySignatures) + isMainChain, isOrphan, _, err := chain.ProcessBlock(blockB2, nil, verifySignatures) require.NoError(err) require.Falsef(isOrphan, "Block b2 should not be an orphan") require.Falsef(isMainChain, "Block b2 should not be on the main chain") @@ -1695,7 +1695,7 @@ func TestBadBlockSignature(t *testing.T) { // A bad signature with the right public key should fail. finalBlock1.BlockProducerInfo.PublicKey = senderPkBytes - _, _, _, err = chain.ProcessBlock(finalBlock1, true) + _, _, _, err = chain.ProcessBlock(finalBlock1, nil, true) require.Error(err) require.Contains(err.Error(), RuleErrorInvalidBlockProducerSIgnature) @@ -1704,20 +1704,20 @@ func TestBadBlockSignature(t *testing.T) { require.NoError(err) finalBlock1.BlockProducerInfo.PublicKey = blockSignerPkBytes finalBlock1.BlockProducerInfo.Signature = nil - _, _, _, err = chain.ProcessBlock(finalBlock1, true) + _, _, _, err = chain.ProcessBlock(finalBlock1, nil, true) require.Error(err) require.Contains(err.Error(), RuleErrorMissingBlockProducerSignature) // If all the BlockProducerInfo is missing, things should fail finalBlock1.BlockProducerInfo = nil - _, _, _, err = chain.ProcessBlock(finalBlock1, true) + _, _, _, err = chain.ProcessBlock(finalBlock1, nil, true) require.Error(err) require.Contains(err.Error(), RuleErrorMissingBlockProducerSignature) // Now let's add blockSignerPK to the map of trusted keys and confirm that the block processes. chain.trustedBlockProducerPublicKeys[MakePkMapKey(blockSignerPkBytes)] = true finalBlock1.BlockProducerInfo = blockProducerInfoCopy - _, _, _, err = chain.ProcessBlock(finalBlock1, true) + _, _, _, err = chain.ProcessBlock(finalBlock1, nil, true) require.NoError(err) _, _ = finalBlock1, db diff --git a/lib/db_utils.go b/lib/db_utils.go index 63f78fcbf..013048655 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5048,13 +5048,22 @@ func GetBlock(blockHash *BlockHash, handle *badger.DB, snap *Snapshot) (*MsgDeSo return blockRet, nil } -func PutBlockHashToBlockWithTxn(txn *badger.Txn, snap *Snapshot, block *MsgDeSoBlock, eventManager *EventManager) error { +func PutBlockHashToBlockWithTxn( + txn *badger.Txn, + snap *Snapshot, + blockHash *BlockHash, + block *MsgDeSoBlock, + eventManager *EventManager, +) error { if block.Header == nil { return fmt.Errorf("PutBlockHashToBlockWithTxn: Header was nil in block %v", block) } - blockHash, err := block.Header.Hash() - if err != nil { - return errors.Wrap(err, "PutBlockHashToBlockWithTxn: Problem hashing header: ") + if blockHash == nil { + var err error + blockHash, err = block.Header.Hash() + if err != nil { + return errors.Wrap(err, "PutBlockHashToBlockWithTxn: Problem hashing header: ") + } } blockKey := BlockHashToBlockKey(blockHash) data, err := block.ToBytes(false) @@ -5074,12 +5083,14 @@ func PutBlockHashToBlockWithTxn(txn *badger.Txn, snap *Snapshot, block *MsgDeSoB return nil } -func PutBlockWithTxn(txn *badger.Txn, snap *Snapshot, desoBlock *MsgDeSoBlock, eventManager *EventManager) error { - blockHash, err := desoBlock.Header.Hash() - if err != nil { - return errors.Wrapf(err, "PutBlockWithTxn: Problem hashing header: ") - } - if err = PutBlockHashToBlockWithTxn(txn, snap, desoBlock, eventManager); err != nil { +func PutBlockWithTxn( + txn *badger.Txn, + snap *Snapshot, + desoBlock *MsgDeSoBlock, + blockHash *BlockHash, + eventManager *EventManager, +) error { + if err := PutBlockHashToBlockWithTxn(txn, snap, blockHash, desoBlock, eventManager); err != nil { return errors.Wrap(err, "PutBlockWithTxn: Problem putting block hash to block") } blockRewardTxn := desoBlock.Txns[0] @@ -5100,7 +5111,7 @@ func PutBlockWithTxn(txn *badger.Txn, snap *Snapshot, desoBlock *MsgDeSoBlock, e pkMapKey := pkMapKeyIter blockRewardKey := PublicKeyBlockHashToBlockRewardKey(pkMapKey[:], blockHash) - if err = DBSetWithTxn(txn, snap, blockRewardKey, EncodeUint64(blockReward), eventManager); err != nil { + if err := DBSetWithTxn(txn, snap, blockRewardKey, EncodeUint64(blockReward), eventManager); err != nil { return err } } @@ -5108,9 +5119,15 @@ func PutBlockWithTxn(txn *badger.Txn, snap *Snapshot, desoBlock *MsgDeSoBlock, e return nil } -func PutBlock(handle *badger.DB, snap *Snapshot, desoBlock *MsgDeSoBlock, eventManager *EventManager) error { +func PutBlock( + handle *badger.DB, + snap *Snapshot, + desoBlock *MsgDeSoBlock, + blockHash *BlockHash, + eventManager *EventManager, +) error { err := handle.Update(func(txn *badger.Txn) error { - return PutBlockWithTxn(txn, snap, desoBlock, eventManager) + return PutBlockWithTxn(txn, snap, desoBlock, blockHash, eventManager) }) return err @@ -5376,7 +5393,7 @@ func InitDbWithDeSoGenesisBlock(params *DeSoParams, handle *badger.DB, return errors.Wrapf(err, "InitDbWithGenesisBlock: Problem putting genesis block hash into db for block chain") } // Add the genesis block to the (hash -> block) index. - if err := PutBlockWithTxn(txn, snap, genesisBlock, eventManager); err != nil { + if err := PutBlockWithTxn(txn, snap, genesisBlock, blockHash, eventManager); err != nil { return errors.Wrapf(err, "InitDbWithGenesisBlock: Problem putting genesis block into db") } // Add the genesis block to the (height, hash -> node info) index in the db. diff --git a/lib/load_test.go b/lib/load_test.go index 2092ad91c..78a3b056e 100644 --- a/lib/load_test.go +++ b/lib/load_test.go @@ -189,7 +189,7 @@ func TestComputeMaxTPS(t *testing.T) { _, _ = newParams, newDB timeStart := time.Now() for _, blockToConnect := range blocksMined { - _, _, _, err := newChain.ProcessBlock(blockToConnect, true /*verifySignatures*/) + _, _, _, err := newChain.ProcessBlock(blockToConnect, nil, true /*verifySignatures*/) require.NoError(err) } elapsedSecs := (time.Since(timeStart)).Seconds() @@ -236,7 +236,7 @@ func TestConnectBlocksLoadTest(t *testing.T) { pprof.StartCPUProfile(ff) timeStart := time.Now() for _, blockToConnect := range blocksMined { - _, _, _, err := newChain.ProcessBlock(blockToConnect, false /*verifySignatures*/) + _, _, _, err := newChain.ProcessBlock(blockToConnect, nil, false /*verifySignatures*/) require.NoError(err) } elapsedSecs := (time.Since(timeStart)).Seconds() diff --git a/lib/miner.go b/lib/miner.go index d912f5858..6703cd554 100644 --- a/lib/miner.go +++ b/lib/miner.go @@ -249,7 +249,7 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo verifySignatures := true // TODO(miner): Replace with a call to SubmitBlock. isMainChain, isOrphan, _, err := desoMiner.BlockProducer.chain.ProcessBlock( - blockToMine, verifySignatures) + blockToMine, nil, verifySignatures) glog.V(2).Infof("Called ProcessBlock: isMainChain=(%v), isOrphan=(%v), err=(%v)", isMainChain, isOrphan, err) if err != nil { diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 1c959dde5..ec7a00897 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -205,7 +205,7 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas } // Verify that the header is properly formed. - if err := bc.isValidBlockHeaderPoS(header); err != nil { + if err = bc.isValidBlockHeaderPoS(header); err != nil { return nil, false, bc.storeValidateFailedHeaderInBlockIndexWithWrapperError( header, headerHash, errors.New("validateAndIndexHeaderPoS: Header failed validations"), ) @@ -249,8 +249,11 @@ func (bc *Blockchain) ProcessBlockPoS(block *MsgDeSoBlock, currentView uint64, v if block == nil { return false, false, nil, fmt.Errorf("ProcessBlockPoS: Block is nil") } - - return bc.processBlockPoS(block, currentView, verifySignatures) + blockHash, err := block.Hash() + if err != nil { + return false, false, nil, errors.Wrap(err, "ProcessBlockPoS: Problem hashing block") + } + return bc.processBlockPoS(block, blockHash, currentView, verifySignatures) } // processBlockPoS runs the Fast-HotStuff block connect and commit rule as follows: @@ -262,7 +265,12 @@ func (bc *Blockchain) ProcessBlockPoS(block *MsgDeSoBlock, currentView uint64, v // 5. Try to apply the incoming block as the tip (performing reorgs as necessary). If it can't be applied, exit here. // 6. Run the commit rule - If applicable, flushes the incoming block's grandparent to the DB // 7. Notify listeners via the EventManager of which blocks have been removed and added. -func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, verifySignatures bool) ( +func (bc *Blockchain) processBlockPoS( + block *MsgDeSoBlock, + blockHash *BlockHash, + currentView uint64, + verifySignatures bool, +) ( _success bool, _isOrphan bool, _missingBlockHashes []*BlockHash, @@ -277,9 +285,12 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v } // If we can't hash the block, we can never store in the block index and we should throw it out immediately. - blockHash, err := block.Hash() - if err != nil { - return false, false, nil, errors.Wrapf(err, "processBlockPoS: Problem hashing block") + if blockHash == nil { + var err error + blockHash, err = block.Hash() + if err != nil { + return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem hashing block") + } } // In hypersync archival mode, we may receive blocks that have already been processed and committed during state @@ -310,7 +321,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // on our best chain. Try to process the orphan by running basic validations. // If it passes basic integrity checks, we'll store it with the hope that we // will eventually get a parent that connects to our best chain. - return false, true, missingBlockHashes, bc.processOrphanBlockPoS(block) + return false, true, missingBlockHashes, bc.processOrphanBlockPoS(block, blockHash) } if err != nil { @@ -425,7 +436,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v } var appliedNewTipOrphan bool if appliedNewTipOrphan, _, _, err = bc.processBlockPoS( - orphanBlock, currentView, verifySignatures); err != nil { + orphanBlock, blockNodeAtNextHeight.Hash, currentView, verifySignatures); err != nil { glog.Errorf("processBlockPoS: Problem validating orphan block %v", blockNodeAtNextHeight.Hash) continue } @@ -444,7 +455,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // As a spam-prevention measure, we will not store a block if it fails the QC or leader check // and simply throw it away. If it fails the other integrity checks, we'll store it // as validate failed. -func (bc *Blockchain) processOrphanBlockPoS(block *MsgDeSoBlock) error { +func (bc *Blockchain) processOrphanBlockPoS(block *MsgDeSoBlock, blockHash *BlockHash) error { // Construct a UtxoView, so we can perform the QC and leader checks. utxoView := bc.GetCommittedTipView() @@ -555,9 +566,11 @@ func (bc *Blockchain) processOrphanBlockPoS(block *MsgDeSoBlock) error { return nil } - blockHash, err := block.Header.Hash() - if err != nil { - return errors.Wrap(err, "processOrphanBlockPoS: Problem hashing block") + if blockHash == nil { + blockHash, err = block.Header.Hash() + if err != nil { + return errors.Wrap(err, "processOrphanBlockPoS: Problem hashing block") + } } // All blocks should pass the basic integrity validations, which ensure the block @@ -1542,7 +1555,8 @@ func (bc *Blockchain) upsertBlockAndBlockNodeToDB(block *MsgDeSoBlock, blockNode // Store the block in badger err := bc.db.Update(func(txn *badger.Txn) error { if storeFullBlock { - if innerErr := PutBlockHashToBlockWithTxn(txn, bc.snapshot, block, bc.eventManager); innerErr != nil { + if innerErr := PutBlockHashToBlockWithTxn( + txn, bc.snapshot, blockNode.Hash, block, bc.eventManager); innerErr != nil { return errors.Wrapf(innerErr, "upsertBlockAndBlockNodeToDB: Problem calling PutBlockHashToBlockWithTxn") } } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index d3ee5ac1c..7e3ee7430 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -2084,7 +2084,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Give the block a random parent, so it is truly an orphan. realBlock.Header.PrevBlockHash = NewBlockHash(RandomBytes(32)) updateProposerVotePartialSignatureForBlock(testMeta, realBlock) - err := testMeta.chain.processOrphanBlockPoS(realBlock) + err := testMeta.chain.processOrphanBlockPoS(realBlock, nil) require.NoError(t, err) // Get the block node from the block index. blockHash, err := realBlock.Hash() @@ -2105,7 +2105,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { realBlock.Header.Version = 1 updateProposerVotePartialSignatureForBlock(testMeta, realBlock) // There should be no error, but the block should be marked as ValidateFailed. - err := testMeta.chain.processOrphanBlockPoS(realBlock) + err := testMeta.chain.processOrphanBlockPoS(realBlock, nil) require.NoError(t, err) // Get the block node from the block index. blockHash, err := realBlock.Hash() @@ -2132,7 +2132,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { realBlock.Header.ProposerVotingPublicKey = _generateRandomBLSPrivateKey(t).PublicKey() updateProposerVotePartialSignatureForBlock(testMeta, realBlock) // There should be no error, but the block should be marked as ValidateFailed. - err = testMeta.chain.processOrphanBlockPoS(realBlock) + err = testMeta.chain.processOrphanBlockPoS(realBlock, nil) require.NoError(t, err) // Get the block node from the block index. blockHash, err := realBlock.Hash() @@ -2184,7 +2184,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { } updateProposerVotePartialSignatureForBlock(testMeta, realBlock) // There should be no error, but the block should be marked as ValidateFailed. - err = testMeta.chain.processOrphanBlockPoS(realBlock) + err = testMeta.chain.processOrphanBlockPoS(realBlock, nil) require.NoError(t, err) // Get the block node from the block index. blockHash, err := realBlock.Hash() @@ -2202,7 +2202,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Give the block a random parent, so it is truly an orphan. nextEpochBlock.Header.PrevBlockHash = NewBlockHash(RandomBytes(32)) updateProposerVotePartialSignatureForBlock(testMeta, nextEpochBlock) - err = testMeta.chain.processOrphanBlockPoS(nextEpochBlock) + err = testMeta.chain.processOrphanBlockPoS(nextEpochBlock, nil) require.NoError(t, err) // Get the block node from the block index. blockHash, err := nextEpochBlock.Hash() @@ -2227,7 +2227,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { nextEpochBlock.Header.ProposerVotingPublicKey = _generateRandomBLSPrivateKey(t).PublicKey() updateProposerVotePartialSignatureForBlock(testMeta, nextEpochBlock) // There should be no error, but the block should be marked as ValidateFailed. - err = testMeta.chain.processOrphanBlockPoS(nextEpochBlock) + err = testMeta.chain.processOrphanBlockPoS(nextEpochBlock, nil) require.NoError(t, err) // Get the block node from the block index. blockHash, err := nextEpochBlock.Hash() @@ -2246,7 +2246,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { nextEpochBlock.Header.PrevBlockHash = NewBlockHash(RandomBytes(32)) updateProposerVotePartialSignatureForBlock(testMeta, nextEpochBlock) // Update the QC to not have a supermajority. - err = testMeta.chain.processOrphanBlockPoS(nextEpochBlock) + err = testMeta.chain.processOrphanBlockPoS(nextEpochBlock, nil) require.NoError(t, err) // Update the QC to not have a supermajority. // Get all the bls keys for the validators that aren't the leader. @@ -2278,7 +2278,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { Signature: aggregatedSignature, } updateProposerVotePartialSignatureForBlock(testMeta, nextEpochBlock) - err = testMeta.chain.processOrphanBlockPoS(nextEpochBlock) + err = testMeta.chain.processOrphanBlockPoS(nextEpochBlock, nil) require.NoError(t, err) // Get the block node from the block index. blockHash, err := nextEpochBlock.Hash() @@ -2307,7 +2307,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { twoEpochsInFutureBlock.Header.PrevBlockHash = NewBlockHash(RandomBytes(32)) updateProposerVotePartialSignatureForBlock(testMeta, twoEpochsInFutureBlock) // We should get an error that this block is too far in the future. - err = testMeta.chain.processOrphanBlockPoS(twoEpochsInFutureBlock) + err = testMeta.chain.processOrphanBlockPoS(twoEpochsInFutureBlock, nil) require.Error(t, err) // The block shouldn't be in the block index. blockHash, err := twoEpochsInFutureBlock.Hash() @@ -2324,7 +2324,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { require.NoError(t, err) var prevEpochBlock *MsgDeSoBlock prevEpochBlock = _generateRealBlock(testMeta, prevEpochEntry.FinalBlockHeight, prevEpochEntry.FinalBlockHeight, 17283, testMeta.chain.BlockTip().Hash, false) - err = testMeta.chain.processOrphanBlockPoS(prevEpochBlock) + err = testMeta.chain.processOrphanBlockPoS(prevEpochBlock, nil) require.NoError(t, err) // The block should be in the block index. blockHash, err := prevEpochBlock.Hash() diff --git a/lib/pos_consensus.go b/lib/pos_consensus.go index ab7a73190..ceb3923e8 100644 --- a/lib/pos_consensus.go +++ b/lib/pos_consensus.go @@ -317,7 +317,7 @@ func (fc *FastHotStuffConsensus) handleBlockProposalEvent( } // Process the block locally - missingBlockHashes, err := fc.tryProcessBlockAsNewTip(blockProposal) + missingBlockHashes, err := fc.tryProcessBlockAsNewTip(blockProposal, blockHash) if err != nil { return errors.Errorf("Error processing block locally: %v", err) } @@ -583,7 +583,11 @@ func (fc *FastHotStuffConsensus) HandleValidatorTimeout(pp *Peer, msg *MsgDeSoVa return nil, nil } -func (fc *FastHotStuffConsensus) HandleBlock(pp *Peer, msg *MsgDeSoBlock) (missingBlockHashes []*BlockHash, _err error) { +func (fc *FastHotStuffConsensus) HandleBlock( + pp *Peer, + msg *MsgDeSoBlock, + blockHash *BlockHash, +) (missingBlockHashes []*BlockHash, _err error) { glog.V(2).Infof("FastHotStuffConsensus.HandleBlock: Received block: \n%s", msg.String()) glog.V(2).Infof("FastHotStuffConsensus.HandleBlock: %s", fc.fastHotStuffEventLoop.ToString()) @@ -605,7 +609,7 @@ func (fc *FastHotStuffConsensus) HandleBlock(pp *Peer, msg *MsgDeSoBlock) (missi // Try to apply the block as the new tip of the blockchain. If the block is an orphan, then // we will get back a list of missing ancestor block hashes. We can fetch the missing blocks // from the network and retry. - missingBlockHashes, err := fc.tryProcessBlockAsNewTip(msg) + missingBlockHashes, err := fc.tryProcessBlockAsNewTip(msg, blockHash) if err != nil { // If we get an error here, it means something went wrong with the block processing algorithm. // Nothing we can do to recover here. @@ -635,10 +639,14 @@ func (fc *FastHotStuffConsensus) HandleBlock(pp *Peer, msg *MsgDeSoBlock) (missi // // Reference Implementation: // https://github.com/deso-protocol/hotstuff_pseudocode/blob/6409b51c3a9a953b383e90619076887e9cebf38d/fast_hotstuff_bls.go#L573 -func (fc *FastHotStuffConsensus) tryProcessBlockAsNewTip(block *MsgDeSoBlock) ([]*BlockHash, error) { +func (fc *FastHotStuffConsensus) tryProcessBlockAsNewTip( + block *MsgDeSoBlock, + blockHash *BlockHash, +) ([]*BlockHash, error) { // Try to apply the block locally as the new tip of the blockchain successfullyAppliedNewTip, _, missingBlockHashes, err := fc.blockchain.processBlockPoS( block, // Pass in the block itself + blockHash, fc.fastHotStuffEventLoop.GetCurrentView(), // Pass in the current view to ensure we don't process a stale block true, // Make sure we verify signatures in the block ) diff --git a/lib/server.go b/lib/server.go index 2332d0f89..47638cf2e 100644 --- a/lib/server.go +++ b/lib/server.go @@ -2510,14 +2510,14 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { "Server._handleBlock: Processing block %v with FastHotStuffConsensus with SyncState=%v for peer %v", blk, srv.blockchain.chainState(), pp, ))) - blockHashesToRequest, err = srv.fastHotStuffConsensus.HandleBlock(pp, blk) + blockHashesToRequest, err = srv.fastHotStuffConsensus.HandleBlock(pp, blk, blockHash) isOrphan = len(blockHashesToRequest) > 0 } else if !verifySignatures { glog.V(0).Infof(CLog(Cyan, fmt.Sprintf( "Server._handleBlock: Processing block %v WITHOUT signature checking because SyncState=%v for peer %v", blk, srv.blockchain.chainState(), pp, ))) - _, isOrphan, blockHashesToRequest, err = srv.blockchain.ProcessBlock(blk, false) + _, isOrphan, blockHashesToRequest, err = srv.blockchain.ProcessBlock(blk, blockHash, false) } else { // TODO: Signature checking slows things down because it acquires the ChainLock. // The optimal solution is to check signatures in a way that doesn't acquire the @@ -2526,7 +2526,7 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { "Server._handleBlock: Processing block %v WITH signature checking because SyncState=%v for peer %v", blk, srv.blockchain.chainState(), pp, ))) - _, isOrphan, blockHashesToRequest, err = srv.blockchain.ProcessBlock(blk, true) + _, isOrphan, blockHashesToRequest, err = srv.blockchain.ProcessBlock(blk, blockHash, true) } // If we hit an error then abort mission entirely. We should generally never diff --git a/lib/txindex.go b/lib/txindex.go index 1f5158c1b..55dd06b99 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -439,7 +439,7 @@ func (txi *TXIndex) Update() error { // Now that we have added all the txns to our TxIndex db, attach the block // to update our chain. - _, _, _, err = txi.TXIndexChain.ProcessBlock(blockMsg, false /*verifySignatures*/) + _, _, _, err = txi.TXIndexChain.ProcessBlock(blockMsg, blockToAttach.Hash, false /*verifySignatures*/) if err != nil { return fmt.Errorf("Update: Problem attaching block %v: %v", blockToAttach, err)