From 8687e54a36cb957ae59a4b5e88f2846fd8184057 Mon Sep 17 00:00:00 2001 From: Jack Coble Date: Sun, 28 Apr 2019 16:22:20 +0100 Subject: [PATCH 1/9] Start of implementation for peer caching --- cmd/phantom/main.go | 20 ++++++++++++++++ pkg/storage/storage.go | 54 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 pkg/storage/storage.go diff --git a/cmd/phantom/main.go b/cmd/phantom/main.go index 86f5d81..768f6ae 100644 --- a/cmd/phantom/main.go +++ b/cmd/phantom/main.go @@ -34,6 +34,7 @@ import ( "flag" "fmt" "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/jackkdev/phantom/pkg/storage" "log" "github.com/breakcrypto/phantom/pkg/socket/wire" "github.com/breakcrypto/phantom/pkg/phantom" @@ -55,6 +56,7 @@ var sentinelVersion uint32 var daemonVersion uint32 var masternodeConf string var coinCon phantom.CoinConf +var dbPath string var userAgent string const VERSION = "0.0.4" @@ -90,6 +92,7 @@ func main() { flag.StringVar(&userAgent, "user_agent", "@_breakcrypto phantom", "The user agent string to connect to remote peers with.") + flag.StringVar(&dbPath, "db_path", "./peers.db", "The destination for database storage.") flag.Parse() @@ -130,9 +133,19 @@ func main() { if userAgent == "@_breakcrypto phantom" && coinInfo.UserAgent != "" { userAgent = coinInfo.UserAgent } + if dbPath == "" { + dbPath = "peers.db" + } } } + db, err := storage.InitialiseDB("peers.db") + if err != nil { + log.Fatal("An error occurred initialising the database") + } else { + log.Println("Database was initialised:", db) + } + magicMsgNewLine = true magicBytes64, _ := strconv.ParseUint(magicHex, 16, 32) @@ -313,6 +326,13 @@ func getNextPeer(connectionSet map[string]*phantom.PingerConnection, peerSet map log.Println("Found new peer: ", peer) + // Cache peers + db, err := storage.InitialiseDB("peers.dat") + if err != nil { + log.Fatal(err) + } + err = storage.CachePeerToDB(db, peer) + return returnValue, nil } } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go new file mode 100644 index 0000000..d958de5 --- /dev/null +++ b/pkg/storage/storage.go @@ -0,0 +1,54 @@ +package storage + +import( + "encoding/json" + "fmt" + "github.com/boltdb/bolt" + "log" +) + +func InitialiseDB(path string) (*bolt.DB, error) { + db, err := bolt.Open(path, 0600, nil) + if err != nil { + return nil, err + } + + err = db.Update(func(tx *bolt.Tx) error { + peerBucket, err := tx.CreateBucketIfNotExists([]byte("peers")) + if err != nil { + return fmt.Errorf("Could not create peers bucket: %v", err) + } else { + log.Println("Bucket created:", peerBucket) + } + return nil + }) + + if err != nil { + return nil, err + } + + log.Println("Database successfully initialised...") + + return db, nil +} + +func CachePeerToDB(db *bolt.DB, peer string) error { + entry := peer + entryBytes, err := json.Marshal(entry) + + if err != nil { + return err + } + + err = db.Update(func(tx *bolt.Tx) error { + err := tx.Bucket([]byte("peers")).Put([]byte("peer"), (entryBytes)) + if err != nil { + return err + } + + return nil + }) + + log.Println("Peer added to cache", entry) + return err +} \ No newline at end of file From aa3824c2222cd44c88657db4f8c761e049f0afa5 Mon Sep 17 00:00:00 2001 From: Jack Coble Date: Sun, 28 Apr 2019 17:07:41 +0100 Subject: [PATCH 2/9] Function to load cached peers --- pkg/storage/storage.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index d958de5..fd066dc 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -51,4 +51,18 @@ func CachePeerToDB(db *bolt.DB, peer string) error { log.Println("Peer added to cache", entry) return err +} + +func LoadPeersFromDB(db *bolt.DB) error { + err := db.View(func(tx *bolt.Tx) error { + peers := tx.Bucket([]byte("peers")).Get([]byte("peer")) + log.Println("Peers: %s\n", peers) + return nil + }) + + if err != nil { + return err + } + + return nil } \ No newline at end of file From 74d775fa43a667a8f36d74c7e5d5c76a840ee204 Mon Sep 17 00:00:00 2001 From: Jack Coble Date: Sun, 28 Apr 2019 17:36:12 +0100 Subject: [PATCH 3/9] Cache peers and if any, showcase them on daemon startup --- cmd/phantom/main.go | 16 ++++++++++------ pkg/storage/storage.go | 6 +++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cmd/phantom/main.go b/cmd/phantom/main.go index 768f6ae..dc9fba6 100644 --- a/cmd/phantom/main.go +++ b/cmd/phantom/main.go @@ -59,6 +59,8 @@ var coinCon phantom.CoinConf var dbPath string var userAgent string +var cachedPeers error + const VERSION = "0.0.4" func main() { @@ -146,6 +148,8 @@ func main() { log.Println("Database was initialised:", db) } + cachedPeers = storage.LoadPeersFromDB(db) + magicMsgNewLine = true magicBytes64, _ := strconv.ParseUint(magicHex, 16, 32) @@ -321,18 +325,18 @@ func getNextPeer(connectionSet map[string]*phantom.PingerConnection, peerSet map //we have a peer that isn't in the conncetion list return it returnValue = peerSet[peer] - //remove the peer from the connection list - delete(peerSet, peer) - - log.Println("Found new peer: ", peer) - // Cache peers - db, err := storage.InitialiseDB("peers.dat") + db, err := storage.InitialiseDB("peers.db") if err != nil { log.Fatal(err) } err = storage.CachePeerToDB(db, peer) + //remove the peer from the connection list + delete(peerSet, peer) + + log.Println("Found new peer: ", peer) + return returnValue, nil } } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index fd066dc..5e91448 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -50,13 +50,15 @@ func CachePeerToDB(db *bolt.DB, peer string) error { }) log.Println("Peer added to cache", entry) + defer db.Close() + return err } func LoadPeersFromDB(db *bolt.DB) error { err := db.View(func(tx *bolt.Tx) error { peers := tx.Bucket([]byte("peers")).Get([]byte("peer")) - log.Println("Peers: %s\n", peers) + log.Println("Peers:\n", string(peers)) return nil }) @@ -64,5 +66,7 @@ func LoadPeersFromDB(db *bolt.DB) error { return err } + defer db.Close() + return nil } \ No newline at end of file From e77545a9bc8a54e6392c69c171f66356dc640b98 Mon Sep 17 00:00:00 2001 From: Jack Coble Date: Sun, 28 Apr 2019 17:37:33 +0100 Subject: [PATCH 4/9] Reflect to use breakcrypto's future package --- cmd/phantom/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/phantom/main.go b/cmd/phantom/main.go index dc9fba6..9b7b38c 100644 --- a/cmd/phantom/main.go +++ b/cmd/phantom/main.go @@ -34,7 +34,7 @@ import ( "flag" "fmt" "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/jackkdev/phantom/pkg/storage" + "github.com/breakcrypto/phantom/pkg/storage" "log" "github.com/breakcrypto/phantom/pkg/socket/wire" "github.com/breakcrypto/phantom/pkg/phantom" From f066d01860c864c97385afefe76ff3b114ebaca3 Mon Sep 17 00:00:00 2001 From: Jack Coble Date: Sun, 28 Apr 2019 17:46:22 +0100 Subject: [PATCH 5/9] Update README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index bfe9bbf..6796c1c 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,8 @@ There is a coinconf generator included that can auto-generate settings for most The string to use for the sentinel version number (i.e. 1.20.0) (default "0.0.0") -user_agent The user agent string to connect to remote peers with. + -db_path + The destination for database storage (default path is ./peers.db) ``` ## Building (using Docker) From 2c9c4c589aa0cd2ea9a6ac312de6f8b2f28da03f Mon Sep 17 00:00:00 2001 From: Jack Coble Date: Mon, 29 Apr 2019 10:17:04 +0100 Subject: [PATCH 6/9] Improve logging output --- pkg/storage/storage.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 5e91448..ea41870 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,6 +1,6 @@ package storage -import( +import ( "encoding/json" "fmt" "github.com/boltdb/bolt" @@ -14,11 +14,11 @@ func InitialiseDB(path string) (*bolt.DB, error) { } err = db.Update(func(tx *bolt.Tx) error { - peerBucket, err := tx.CreateBucketIfNotExists([]byte("peers")) + _, err := tx.CreateBucketIfNotExists([]byte("peers")) if err != nil { - return fmt.Errorf("Could not create peers bucket: %v", err) + return fmt.Errorf("DB: Could not create peers bucket: %v", err) } else { - log.Println("Bucket created:", peerBucket) + log.Println("DB: Bucket created was created") } return nil }) @@ -27,7 +27,7 @@ func InitialiseDB(path string) (*bolt.DB, error) { return nil, err } - log.Println("Database successfully initialised...") + log.Println("DB: Successfully initialised...") return db, nil } From 4207282895d35d8a3099658adddff0f5675a6418 Mon Sep 17 00:00:00 2001 From: Jack Coble Date: Mon, 29 Apr 2019 10:17:36 +0100 Subject: [PATCH 7/9] Start to cache peers when new peers are being processed --- .gitignore | 2 ++ cmd/phantom/main.go | 36 +++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 5c51faa..5a26392 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .idea/ **/.DS_Store +peers.db +masternode.conf \ No newline at end of file diff --git a/cmd/phantom/main.go b/cmd/phantom/main.go index 9b7b38c..7346434 100644 --- a/cmd/phantom/main.go +++ b/cmd/phantom/main.go @@ -34,7 +34,7 @@ import ( "flag" "fmt" "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/breakcrypto/phantom/pkg/storage" + "github.com/jackkdev/phantom/pkg/storage" "log" "github.com/breakcrypto/phantom/pkg/socket/wire" "github.com/breakcrypto/phantom/pkg/phantom" @@ -141,15 +141,6 @@ func main() { } } - db, err := storage.InitialiseDB("peers.db") - if err != nil { - log.Fatal("An error occurred initialising the database") - } else { - log.Println("Database was initialised:", db) - } - - cachedPeers = storage.LoadPeersFromDB(db) - magicMsgNewLine = true magicBytes64, _ := strconv.ParseUint(magicHex, 16, 32) @@ -245,6 +236,15 @@ func main() { fmt.Println("Sentinel Version: ", daemonVersion) fmt.Println("\n\n") + db, err := storage.InitialiseDB("peers.db") + if err != nil { + log.Fatal("An error occurred initialising the database") + } else { + log.Println("Database was initialised:", db) + } + + cachedPeers = storage.LoadPeersFromDB(db) + for ip := range peerSet { //make the ping channel pingChannel := make(chan phantom.MasternodePing, 1500) @@ -308,6 +308,13 @@ func processNewHashes(hashChannel chan chainhash.Hash, queue *phantom.Queue) { } func processNewAddresses(addrChannel chan wire.NetAddress, peerSet map[string]wire.NetAddress) { + + // Cache peers + db, err := storage.InitialiseDB("peers.db") + if err != nil { + log.Fatal(err) + } + for { addr := <-addrChannel @@ -316,6 +323,8 @@ func processNewAddresses(addrChannel chan wire.NetAddress, peerSet map[string]wi } peerSet[addr.IP.String()] = addr + + err = storage.CachePeerToDB(db, addr.IP.String()) } } @@ -325,13 +334,6 @@ func getNextPeer(connectionSet map[string]*phantom.PingerConnection, peerSet map //we have a peer that isn't in the conncetion list return it returnValue = peerSet[peer] - // Cache peers - db, err := storage.InitialiseDB("peers.db") - if err != nil { - log.Fatal(err) - } - err = storage.CachePeerToDB(db, peer) - //remove the peer from the connection list delete(peerSet, peer) From 0bdf77e09729969ea01d8c2e6e15f190e88d9d31 Mon Sep 17 00:00:00 2001 From: Jack Coble Date: Thu, 2 May 2019 21:48:03 +0100 Subject: [PATCH 8/9] Rewritten the way peers are cached --- cmd/phantom/main.go | 42 +++++++++++------------ pkg/storage/storage.go | 76 ++++++++++++++++++++++++------------------ 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/cmd/phantom/main.go b/cmd/phantom/main.go index 7346434..e1dea05 100644 --- a/cmd/phantom/main.go +++ b/cmd/phantom/main.go @@ -56,11 +56,8 @@ var sentinelVersion uint32 var daemonVersion uint32 var masternodeConf string var coinCon phantom.CoinConf -var dbPath string var userAgent string -var cachedPeers error - const VERSION = "0.0.4" func main() { @@ -94,8 +91,6 @@ func main() { flag.StringVar(&userAgent, "user_agent", "@_breakcrypto phantom", "The user agent string to connect to remote peers with.") - flag.StringVar(&dbPath, "db_path", "./peers.db", "The destination for database storage.") - flag.Parse() if coinConfString != "" { @@ -135,9 +130,6 @@ func main() { if userAgent == "@_breakcrypto phantom" && coinInfo.UserAgent != "" { userAgent = coinInfo.UserAgent } - if dbPath == "" { - dbPath = "peers.db" - } } } @@ -236,15 +228,6 @@ func main() { fmt.Println("Sentinel Version: ", daemonVersion) fmt.Println("\n\n") - db, err := storage.InitialiseDB("peers.db") - if err != nil { - log.Fatal("An error occurred initialising the database") - } else { - log.Println("Database was initialised:", db) - } - - cachedPeers = storage.LoadPeersFromDB(db) - for ip := range peerSet { //make the ping channel pingChannel := make(chan phantom.MasternodePing, 1500) @@ -309,10 +292,11 @@ func processNewHashes(hashChannel chan chainhash.Hash, queue *phantom.Queue) { func processNewAddresses(addrChannel chan wire.NetAddress, peerSet map[string]wire.NetAddress) { - // Cache peers - db, err := storage.InitialiseDB("peers.db") + // init database + db, err := storage.SetupDB() if err != nil { - log.Fatal(err) + log.Fatal("an error occurred:", err) + return } for { @@ -324,11 +308,27 @@ func processNewAddresses(addrChannel chan wire.NetAddress, peerSet map[string]wi peerSet[addr.IP.String()] = addr - err = storage.CachePeerToDB(db, addr.IP.String()) + err = storage.AddPeer(db, addr.IP.String()) } + + // close db connection + defer db.Close() + } func getNextPeer(connectionSet map[string]*phantom.PingerConnection, peerSet map[string]wire.NetAddress) (returnValue wire.NetAddress, err error) { + + // load the database + db, err := storage.SetupDB() + if err != nil { + log.Fatal("an error occurred:", err) + return + } + + // fetch my peers + list, err := storage.FetchPeers(db) + fmt.Println(list) + for peer := range peerSet { if _, ok := connectionSet[peer]; !ok { //we have a peer that isn't in the conncetion list return it diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index ea41870..4b58bcc 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,72 +1,84 @@ package storage -import ( - "encoding/json" +import( "fmt" - "github.com/boltdb/bolt" "log" + "github.com/boltdb/bolt" + "github.com/satori/go.uuid" + //"github.com/breakcrypto/phantom/pkg/socket/wire" ) -func InitialiseDB(path string) (*bolt.DB, error) { - db, err := bolt.Open(path, 0600, nil) +// set up the database file +func SetupDB() (*bolt.DB, error) { + db, err := bolt.Open("nodes.db", 0600, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("could not open db, %v", err) } err = db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte("peers")) + root, err := tx.CreateBucketIfNotExists([]byte("peers")) + if err != nil { + return fmt.Errorf("could not create root bucket: %v", err) + } + + _, err = root.CreateBucketIfNotExists([]byte("nodes")) if err != nil { - return fmt.Errorf("DB: Could not create peers bucket: %v", err) - } else { - log.Println("DB: Bucket created was created") + return fmt.Errorf("could not create peers bucket: %v", err) } + return nil }) if err != nil { - return nil, err + return nil, fmt.Errorf("could not set up buckets, %v", err) } - log.Println("DB: Successfully initialised...") - + log.Println("Database setup complete...") return db, nil } -func CachePeerToDB(db *bolt.DB, peer string) error { - entry := peer - entryBytes, err := json.Marshal(entry) +// add a peer (in string format) to the database +func AddPeer(db *bolt.DB, peer string) error { - if err != nil { - return err - } + // generate a uuid for the key + id := uuid.Must(uuid.NewV4()).String() - err = db.Update(func(tx *bolt.Tx) error { - err := tx.Bucket([]byte("peers")).Put([]byte("peer"), (entryBytes)) + err := db.Update(func(tx *bolt.Tx) error { + err := tx.Bucket([]byte("peers")).Bucket([]byte("nodes")).Put([]byte(id), []byte(peer)) if err != nil { - return err + return fmt.Errorf("could not insert peer: %v", err) } return nil }) - log.Println("Peer added to cache", entry) - defer db.Close() - + fmt.Println("Added peer to database:", peer) return err } -func LoadPeersFromDB(db *bolt.DB) error { +// retrieve all stored peers +func FetchPeers(db *bolt.DB) ([]string, error) { + + var list []string + err := db.View(func(tx *bolt.Tx) error { - peers := tx.Bucket([]byte("peers")).Get([]byte("peer")) - log.Println("Peers:\n", string(peers)) + b := tx.Bucket([]byte("peers")).Bucket([]byte("nodes")) + b.ForEach(func(k, v []byte) error { + fmt.Println("Loaded from database:", string(k), string(v)) + + // load into array + list = append(list, string(v)) + + return nil + }) + fmt.Println(list) + return nil }) if err != nil { - return err + log.Fatal(err) } - defer db.Close() - - return nil + return list, nil } \ No newline at end of file From 5eda46b1e4297eb875c18f8403cbf7fdea8b2b28 Mon Sep 17 00:00:00 2001 From: Jack Coble Date: Thu, 2 May 2019 22:03:34 +0100 Subject: [PATCH 9/9] Load in peers and dump into map --- .gitignore | 2 +- cmd/phantom/main.go | 32 +++++++++++++++++++------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 5a26392..ed00c96 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ .idea/ **/.DS_Store -peers.db +nodes.db masternode.conf \ No newline at end of file diff --git a/cmd/phantom/main.go b/cmd/phantom/main.go index e1dea05..a6a1b32 100644 --- a/cmd/phantom/main.go +++ b/cmd/phantom/main.go @@ -316,7 +316,7 @@ func processNewAddresses(addrChannel chan wire.NetAddress, peerSet map[string]wi } -func getNextPeer(connectionSet map[string]*phantom.PingerConnection, peerSet map[string]wire.NetAddress) (returnValue wire.NetAddress, err error) { +func getNextPeer(peerSet map[string]wire.NetAddress) (returnValue wire.NetAddress, err error) { // load the database db, err := storage.SetupDB() @@ -329,19 +329,25 @@ func getNextPeer(connectionSet map[string]*phantom.PingerConnection, peerSet map list, err := storage.FetchPeers(db) fmt.Println(list) - for peer := range peerSet { - if _, ok := connectionSet[peer]; !ok { - //we have a peer that isn't in the conncetion list return it - returnValue = peerSet[peer] - - //remove the peer from the connection list - delete(peerSet, peer) + for i := 0; i <= len(list); i++ { + returnValue = peerSet[list[i]] + } - log.Println("Found new peer: ", peer) + // LEFT FOR NOW UNTIL FURTHER TESTING IS COMPLETE + //for peer := range peerSet { + // if _, ok := connectionSet[peer]; !ok { + // //we have a peer that isn't in the conncetion list return it + // returnValue = peerSet[peer] + // + // //remove the peer from the connection list + // delete(peerSet, peer) + // + // log.Println("Found new peer: ", peer) + // + // return returnValue, nil + // } + //} - return returnValue, nil - } - } return returnValue, errors.New("No peers found.") } @@ -402,7 +408,7 @@ func sendPings(connectionSet map[string]*phantom.PingerConnection, peerSet map[s for i := 0; i < int(maxConnections) - len(connectionSet); i++ { //spawn off a new connection - peer, err := getNextPeer(connectionSet, peerSet) + peer, err := getNextPeer(peerSet) if err != nil { log.Println("No new peers found.")