diff --git a/.travis.yml b/.travis.yml index d8dc0551..f6adfee8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ os: osx_image: xcode8 +dist: trusty + sudo: false env: @@ -34,6 +36,7 @@ addons: - libgmp-dev - libjansson-dev - libevent-dev + - liblmdb-dev - lcov - valgrind - libtool @@ -53,6 +56,7 @@ before_install: install: - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install jansson; fi - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install libevent; fi + - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install lmdb; fi - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install argp-standalone; fi - if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "$ENABLE_COVERAGE" = "yes" ]; then brew install lcov; fi - if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "$ENABLE_VALGRIND" = "yes" ]; then brew install valgrind; fi diff --git a/configure.ac b/configure.ac index ab8724a6..64ba7b02 100644 --- a/configure.ac +++ b/configure.ac @@ -75,12 +75,14 @@ AC_CHECK_LIB(event_core, event_base_new, EVENT_LIBS=-levent_core, [AC_MSG_ERROR([Missing required libevent])]) AC_CHECK_LIB(jansson, json_loads, JANSSON_LIBS=-ljansson, [AC_MSG_ERROR([Missing required libjansson])]) -AC_CHECK_LIB(argp, argp_parse, ARGP_LIBS=-largp) +AC_CHECK_LIB(argp, argp_parse, ARGP_LIBS=-lARGP) +AC_CHECK_LIB(lmdb, mdb_dbi_open, LMDB_LIBS=-llmdb, + [AC_MSG_ERROR([Missing required liblmdb])]) dnl ------------------------------------- dnl Checks for optional library functions dnl ------------------------------------- -AC_CHECK_FUNCS(fdatasync memmem strndup mkstemp) +AC_CHECK_FUNCS(memmem strndup mkstemp) dnl ----------------- dnl Configure options @@ -152,6 +154,7 @@ AC_SUBST(GMP_LIBS) AC_SUBST(EVENT_LIBS) AC_SUBST(JANSSON_LIBS) AC_SUBST(ARGP_LIBS) +AC_SUBST(LMDB_LIBS) AC_CONFIG_SUBDIRS([external/secp256k1]) AC_CONFIG_FILES([ diff --git a/contrib/debian/control b/contrib/debian/control index 3f9d56db..5bb32713 100644 --- a/contrib/debian/control +++ b/contrib/debian/control @@ -1,7 +1,7 @@ Source: picocoin Priority: optional Maintainer: Joel DeJesus -Build-Depends: debhelper (>= 8.0.0), autotools-dev, libgmp-dev, libevent-dev, libjansson-dev, libc6-dev +Build-Depends: debhelper (>= 8.0.0), autotools-dev, libgmp-dev, libevent-dev, libjansson-dev, liblmdb-dev, libc6-dev Standards-Version: 3.9.4 Section: libs #Homepage: diff --git a/doc/build-osx.md b/doc/build-osx.md index 85b3730d..08521fe0 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -16,11 +16,11 @@ and `jansson` to build *picocoin*. Install these packages. It will take a few minutes. - brew install autoconf automake libtool argp-standalone jansson libevent gmp + brew install autoconf automake libtool argp-standalone jansson libevent liblmdb gmp or - sudo port install autoconf automake libtool argp-standalone jansson libevent pkgconfig gmp + sudo port install autoconf automake libtool argp-standalone jansson libevent liblmdb pkgconfig gmp Building @@ -61,7 +61,6 @@ You should see output formatted in JSON, "chain": "bitcoin", "net.connect.timeout": "11", "peers": "picocoin.peers", - "blkdb": "picocoin.blkdb" } If that works, `picocoin` is ready for use. diff --git a/include/ccoin/Makefile.am b/include/ccoin/Makefile.am index e955e88c..be91b0e8 100644 --- a/include/ccoin/Makefile.am +++ b/include/ccoin/Makefile.am @@ -13,7 +13,6 @@ ccoininclude_HEADERS = \ address.h \ addr_match.h \ base58.h \ - blkdb.h \ bloom.h \ buffer.h \ buint.h \ @@ -37,6 +36,12 @@ ccoininclude_HEADERS = \ util.h \ wallet.h +ccoindbincludedir=$(includedir)/ccoin/db + +ccoindbinclude_HEADERS = \ + db/chaindb.h \ + db/db.h + ccoinnetincludedir=$(includedir)/ccoin/net ccoinnetinclude_HEADERS = \ diff --git a/include/ccoin/buffer.h b/include/ccoin/buffer.h index 33791ba9..62056ab5 100644 --- a/include/ccoin/buffer.h +++ b/include/ccoin/buffer.h @@ -5,7 +5,8 @@ * file COPYING or http://www.opensource.org/licenses/mit-license.php. */ -#include +#include // for bool +#include // for size_t #ifdef __cplusplus extern "C" { diff --git a/include/ccoin/compat.h b/include/ccoin/compat.h index 9c9b4fac..36dfbb7a 100644 --- a/include/ccoin/compat.h +++ b/include/ccoin/compat.h @@ -7,23 +7,12 @@ * alongside all the other API headers */ -#include +#include // for size_t #ifdef __cplusplus extern "C" { #endif -#if defined(__APPLE__) || !defined(HAVE_FDATASYNC) -static inline int fdatasync(int fd) -{ -#ifdef WIN32 - return _commit(fd); -#else - return fsync(fd); -#endif -} -#endif /* !HAVE_FDATASYNC */ - #ifndef HAVE_MEMMEM extern void *memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); diff --git a/include/ccoin/blkdb.h b/include/ccoin/db/chaindb.h similarity index 51% rename from include/ccoin/blkdb.h rename to include/ccoin/db/chaindb.h index c59b5936..86cd2e4b 100644 --- a/include/ccoin/blkdb.h +++ b/include/ccoin/db/chaindb.h @@ -1,14 +1,17 @@ -#ifndef __LIBCCOIN_BLKDB_H__ -#define __LIBCCOIN_BLKDB_H__ +#ifndef __LIBCCOIN_CHAINDB_H__ +#define __LIBCCOIN_CHAINDB_H__ /* Copyright 2012 exMULTI, Inc. * Distributed under the MIT/X11 software license, see the accompanying * file COPYING or http://www.opensource.org/licenses/mit-license.php. */ -#include -#include -#include -#include +#include // for bu256_t +#include // for bp_block +#include // for bp_hashtab_get + +#include // for mpz_t + +#include // for bool #ifdef __cplusplus extern "C" { @@ -23,24 +26,16 @@ struct blkinfo { mpz_t work; int height; - int32_t n_file; /* uninitialized == -1 */ - int64_t n_pos; /* uninitialized == -1 */ - struct blkinfo *prev; }; -struct blkdb_reorg { +struct chaindb_reorg { struct blkinfo *old_best; /* previous best_chain */ unsigned int conn; /* # blocks connected (normally 1) */ unsigned int disconn; /* # blocks disconnected (normally 0) */ }; -struct blkdb { - int fd; - bool datasync_fd; - bool close_fd; - - unsigned char netmagic[4]; +struct chaindb { bu256_t block0; struct bp_hashtab *blocks; @@ -51,16 +46,15 @@ struct blkdb { extern struct blkinfo *bi_new(void); extern void bi_free(struct blkinfo *bi); -extern bool blkdb_init(struct blkdb *db, const unsigned char *netmagic, +extern bool chaindb_init(struct chaindb *db, const unsigned char *netmagic, const bu256_t *genesis_block); -extern void blkdb_free(struct blkdb *db); -extern bool blkdb_read(struct blkdb *db, const char *idx_fn); -extern bool blkdb_add(struct blkdb *db, struct blkinfo *bi, - struct blkdb_reorg *reorg_info); -extern void blkdb_locator(struct blkdb *db, struct blkinfo *bi, +extern void chaindb_free(struct chaindb *db); +extern bool chaindb_add(struct chaindb *db, struct blkinfo *bi, + struct chaindb_reorg *reorg_info); +extern void chaindb_locator(struct chaindb *db, struct blkinfo *bi, struct bp_locator *locator); -static inline struct blkinfo *blkdb_lookup(struct blkdb *db,const bu256_t *hash) +static inline struct blkinfo *chaindb_lookup(struct chaindb *db,const bu256_t *hash) { return (struct blkinfo *)bp_hashtab_get(db->blocks, hash); } @@ -69,4 +63,4 @@ static inline struct blkinfo *blkdb_lookup(struct blkdb *db,const bu256_t *hash) } #endif -#endif /* __LIBCCOIN_BLKDB_H__ */ +#endif /* __LIBCCOIN_CHAINDB_H__ */ diff --git a/include/ccoin/db/db.h b/include/ccoin/db/db.h new file mode 100644 index 00000000..94e4d4c6 --- /dev/null +++ b/include/ccoin/db/db.h @@ -0,0 +1,67 @@ +#ifndef __LIBCCOIN_DB_H__ +#define __LIBCCOIN_DB_H__ +/* Copyright 2012 exMULTI, Inc. + * Distributed under the MIT/X11 software license, see the accompanying + * file COPYING or http://www.opensource.org/licenses/mit-license.php. + */ + +#include // for const_buffer +#include // for bu256_t + +#include // for MDB_dbi, MDB_env + +#include // for bool +#include // for size_t + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + NETMAGIC_LEN = 4 +}; + + enum { + MAX_DB_SIZE = 17179869184 // Maximum database size in bytes + }; + +enum db_list { + METADB, + BLOCKDB, + BLOCKHEIGHTDB, + MAX_NUM_DBS, +}; + +enum metadb_key { + NETMAGIC_KEY, + GENESIS_KEY, +}; + +struct db_handle { + const char *name; + MDB_dbi dbi; + bool open; +}; + +struct db_info { + MDB_env *env; + struct db_handle handle[MAX_NUM_DBS]; +}; + +extern bool metadb_init(const unsigned char *netmagic, + const bu256_t *genesis_block); + +extern bool blockdb_init(void); +extern bool blockdb_add(bu256_t *hash, struct const_buffer *buf); + +extern bool blockheightdb_init(void); +extern bool blockheightdb_add(int height, bu256_t *hash); +extern bool blockheightdb_getall(bool (*read_block)(void *p, size_t len)); + +extern void db_close(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __LIBCCOIN_DB_H__ */ diff --git a/include/ccoin/net/net.h b/include/ccoin/net/net.h index 1b3d804f..a3b9c3a5 100644 --- a/include/ccoin/net/net.h +++ b/include/ccoin/net/net.h @@ -43,7 +43,7 @@ struct net_child_info { int write_fd; struct peer_manager *peers; - struct blkdb *db; + struct chaindb *db; parr *conns; struct event_base *eb; @@ -57,7 +57,6 @@ struct net_child_info { bool (*inv_block_process)(bu256_t *hash); bool (*block_process)(struct bp_block *block, - struct p2p_message_hdr *hdr, struct const_buffer *buf); }; diff --git a/lib/Makefile.am b/lib/Makefile.am index 707c73b7..800da374 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -22,7 +22,6 @@ libccoin_la_SOURCES= \ addr_match.c \ base58.c \ bignum.c \ - blkdb.c \ block.c \ blockfile.c \ bloom.c \ @@ -54,7 +53,11 @@ libccoin_la_SOURCES= \ utxo.c \ wallet.c -noinst_LTLIBRARIES= libccoinnet.la libccoinaes.la +noinst_LTLIBRARIES= libccoindb.la libccoinnet.la libccoinaes.la + +libccoindb_la_SOURCES= \ + db/chaindb.c \ + db/db.c libccoinnet_la_SOURCES= \ net/dns.c \ diff --git a/lib/blkdb.c b/lib/blkdb.c deleted file mode 100644 index 79bc5c73..00000000 --- a/lib/blkdb.c +++ /dev/null @@ -1,295 +0,0 @@ -/* Copyright 2012 exMULTI, Inc. - * Distributed under the MIT/X11 software license, see the accompanying - * file COPYING or http://www.opensource.org/licenses/mit-license.php. - */ -#include "picocoin-config.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include /* for fdatasync */ - -struct blkinfo *bi_new(void) -{ - struct blkinfo *bi; - - bi = calloc(1, sizeof(*bi)); - mpz_init(bi->work); - bi->height = -1; - bi->n_file = -1; - bi->n_pos = -1LL; - - bp_block_init(&bi->hdr); - - return bi; -} - -void bi_free(struct blkinfo *bi) -{ - if (!bi) - return; - - mpz_clear(bi->work); - - bp_block_free(&bi->hdr); - - memset(bi, 0, sizeof(*bi)); - free(bi); -} - -bool blkdb_init(struct blkdb *db, const unsigned char *netmagic, - const bu256_t *genesis_block) -{ - memset(db, 0, sizeof(*db)); - - db->fd = -1; - - bu256_copy(&db->block0, genesis_block); - - memcpy(db->netmagic, netmagic, sizeof(db->netmagic)); - db->blocks = bp_hashtab_new_ext(bu256_hash, bu256_equal_, - NULL, (bp_freefunc) bi_free); - - return true; -} - -static bool blkdb_connect(struct blkdb *db, struct blkinfo *bi, - struct blkdb_reorg *reorg_info) -{ - memset(reorg_info, 0, sizeof(*reorg_info)); - - if (blkdb_lookup(db, &bi->hash)) - return false; - - bool rc = false; - mpz_t cur_work; - mpz_init(cur_work); - - u256_from_compact(cur_work, bi->hdr.nBits); - - bool best_chain = false; - - /* verify genesis block matches first record */ - if (bp_hashtab_size(db->blocks) == 0) { - if (!bu256_equal(&bi->hdr.sha256, &db->block0)) - goto out; - - /* bi->prev = NULL; */ - bi->height = 0; - - mpz_set(bi->work, cur_work); - - best_chain = true; - } - - /* lookup and verify previous block */ - else { - struct blkinfo *prev = blkdb_lookup(db, &bi->hdr.hashPrevBlock); - if (!prev) - goto out; - - bi->prev = prev; - bi->height = prev->height + 1; - - mpz_add(bi->work, cur_work, prev->work); - - if (mpz_cmp(bi->work, db->best_chain->work) > 0) - best_chain = true; - } - - /* add to block map */ - bp_hashtab_put(db->blocks, &bi->hash, bi); - - /* if new best chain found, update pointers */ - if (best_chain) { - struct blkinfo *old_best = db->best_chain; - struct blkinfo *new_best = bi; - - reorg_info->old_best = old_best; - - /* likely case: new best chain has greater height */ - if (!old_best) { - while (new_best) { - new_best = new_best->prev; - reorg_info->conn++; - } - } else { - while (new_best && - (new_best->height > old_best->height)) { - new_best = new_best->prev; - reorg_info->conn++; - } - } - - /* unlikely case: old best chain has greater height */ - while (old_best && new_best && - (old_best->height > new_best->height)) { - old_best = old_best->prev; - reorg_info->disconn++; - } - - /* height matches, but we are still walking parallel chains */ - while (old_best && new_best && (old_best != new_best)) { - new_best = new_best->prev; - reorg_info->conn++; - - old_best = old_best->prev; - reorg_info->disconn++; - } - - /* reorg analyzed. update database's best-chain pointer */ - db->best_chain = bi; - } - - rc = true; - -out: - mpz_clear(cur_work); - return rc; -} - -static bool blkdb_read_rec(struct blkdb *db, const struct p2p_message *msg) -{ - struct blkinfo *bi; - struct const_buffer buf = { msg->data, msg->hdr.data_len }; - - if (strncmp(msg->hdr.command, "rec", 12)) - return false; - - bi = bi_new(); - if (!bi) - return false; - - /* deserialize record */ - if (!deser_u256(&bi->hash, &buf)) - goto err_out; - if (!deser_bp_block(&bi->hdr, &buf)) - goto err_out; - - /* verify that provided hash matches block header, as an additional - * self-verification step - */ - bp_block_calc_sha256(&bi->hdr); - if (!bu256_equal(&bi->hash, &bi->hdr.sha256)) - goto err_out; - - /* verify block may be added to chain, then add it */ - struct blkdb_reorg dummy; - if (!blkdb_connect(db, bi, &dummy)) - goto err_out; - - return true; - -err_out: - bi_free(bi); - return false; -} - -static cstring *ser_blkinfo(const struct blkinfo *bi) -{ - cstring *rs = cstr_new_sz(sizeof(*bi)); - - ser_u256(rs, &bi->hash); - ser_bp_block(rs, &bi->hdr); - - return rs; -} - -static cstring *blkdb_ser_rec(struct blkdb *db, const struct blkinfo *bi) -{ - cstring *data = ser_blkinfo(bi); - - cstring *rs = message_str(db->netmagic, "rec", data->str, data->len); - - cstr_free(data, true); - - return rs; -} - -bool blkdb_read(struct blkdb *db, const char *idx_fn) -{ - bool rc = true; - int fd = file_seq_open(idx_fn); - if (fd < 0) - return false; - - struct p2p_message msg; - memset(&msg, 0, sizeof(msg)); - bool read_ok = true; - - while (fread_message(fd, &msg, &read_ok)) { - rc = blkdb_read_rec(db, &msg); - if (!rc) - break; - } - - close(fd); - - free(msg.data); - - return read_ok && rc; -} - -bool blkdb_add(struct blkdb *db, struct blkinfo *bi, - struct blkdb_reorg *reorg_info) -{ - if (db->fd >= 0) { - cstring *data = blkdb_ser_rec(db, bi); - if (!data) - return false; - - /* assume either at EOF, or O_APPEND */ - size_t data_len = data->len; - ssize_t wrc = write(db->fd, data->str, data_len); - - cstr_free(data, true); - - if (wrc != data_len) - return false; - - if (db->datasync_fd && (fdatasync(db->fd) < 0)) - return false; - } - - /* verify block may be added to chain, then add it */ - return blkdb_connect(db, bi, reorg_info); -} - -void blkdb_free(struct blkdb *db) -{ - if (db->close_fd && (db->fd >= 0)) - close(db->fd); - - bp_hashtab_unref(db->blocks); -} - -void blkdb_locator(struct blkdb *db, struct blkinfo *bi, - struct bp_locator *locator) -{ - if (!bi) - bi = db->best_chain; - - int step = 1; - while (bi) { - bp_locator_push(locator, &bi->hash); - - unsigned int i; - for (i = 0; bi && i < step; i++) - bi = bi->prev; - if (locator->vHave->len > 10) - step *= 2; - } - - bp_locator_push(locator, &db->block0); -} - diff --git a/lib/db/chaindb.c b/lib/db/chaindb.c new file mode 100644 index 00000000..0a355696 --- /dev/null +++ b/lib/db/chaindb.c @@ -0,0 +1,189 @@ +/* Copyright 2012 exMULTI, Inc. + * Distributed under the MIT/X11 software license, see the accompanying + * file COPYING or http://www.opensource.org/licenses/mit-license.php. + */ + +#include // for bu256_hex, bu256_copy, etc +#include // for bp_block, bp_locator_push, etc +#include // for blkinfo, chaindb, etc +#include // for blockheightdb_add +#include // for bp_hashtab_new_ext, etc +#include // for log_debug, log_info +#include // for parr +#include // for u256_from_compact + +#include // for mpz_clear, mpz_init, etc + +#include // for NULL +#include // for calloc, free +#include // for memset +#include // for bool, true, false + +struct logging *log_state; + +struct blkinfo *bi_new(void) +{ + struct blkinfo *bi; + + bi = calloc(1, sizeof(*bi)); + mpz_init(bi->work); + bi->height = -1; + + bp_block_init(&bi->hdr); + + return bi; +} + +void bi_free(struct blkinfo *bi) +{ + if (!bi) + return; + + mpz_clear(bi->work); + + bp_block_free(&bi->hdr); + + memset(bi, 0, sizeof(*bi)); + free(bi); +} + +bool chaindb_init(struct chaindb *db, const unsigned char *netmagic, + const bu256_t *genesis_block) +{ + memset(db, 0, sizeof(*db)); + + bu256_copy(&db->block0, genesis_block); + + db->blocks = bp_hashtab_new_ext(bu256_hash, bu256_equal_, + NULL, (bp_freefunc) bi_free); + + return true; +} + +bool chaindb_add(struct chaindb *db, struct blkinfo *bi, + struct chaindb_reorg *reorg_info) +{ + memset(reorg_info, 0, sizeof(*reorg_info)); + + if (chaindb_lookup(db, &bi->hash)) + return false; + + bool rc = false; + char hexstr[BU256_STRSZ]; + mpz_t cur_work; + mpz_init(cur_work); + + u256_from_compact(cur_work, bi->hdr.nBits); + + bool best_chain = false; + + /* verify genesis block matches first record */ + if (bp_hashtab_size(db->blocks) == 0) { + if (!bu256_equal(&bi->hdr.sha256, &db->block0)) + goto out; + + /* bi->prev = NULL; */ + bi->height = 0; + + mpz_set(bi->work, cur_work); + + best_chain = true; + } + + /* lookup and verify previous block */ + else { + struct blkinfo *prev = chaindb_lookup(db, &bi->hdr.hashPrevBlock); + if (!prev) + goto out; + + bi->prev = prev; + bi->height = prev->height + 1; + + mpz_add(bi->work, cur_work, prev->work); + + if (mpz_cmp(bi->work, db->best_chain->work) > 0) + best_chain = true; + } + + /* add to block map */ + bp_hashtab_put(db->blocks, &bi->hash, bi); + blockheightdb_add(bi->height, &bi->hash); + + /* if new best chain found, update pointers */ + if (best_chain) { + struct blkinfo *old_best = db->best_chain; + struct blkinfo *new_best = bi; + + reorg_info->old_best = old_best; + + /* likely case: new best chain has greater height */ + if (!old_best) { + while (new_best) { + new_best = new_best->prev; + reorg_info->conn++; + } + } else { + while (new_best && + (new_best->height > old_best->height)) { + new_best = new_best->prev; + reorg_info->conn++; + } + } + + /* unlikely case: old best chain has greater height */ + while (old_best && new_best && + (old_best->height > new_best->height)) { + old_best = old_best->prev; + reorg_info->disconn++; + } + + /* height matches, but we are still walking parallel chains */ + while (old_best && new_best && (old_best != new_best)) { + new_best = new_best->prev; + reorg_info->conn++; + + old_best = old_best->prev; + reorg_info->disconn++; + } + + /* reorg analyzed. update database's best-chain pointer */ + db->best_chain = bi; + + bu256_hex(hexstr, &db->best_chain->hdr.sha256); + log_info("chaindb: New best = %s Height = %i",hexstr, bi->height); + } + rc = true; + bu256_hex(hexstr, &bi->hdr.sha256); + log_debug("chaindb: Adding block %s to chaindb successful", hexstr); + +out: + mpz_clear(cur_work); + return rc; +} + + +void chaindb_free(struct chaindb *db) +{ + bp_hashtab_unref(db->blocks); +} + +void chaindb_locator(struct chaindb *db, struct blkinfo *bi, + struct bp_locator *locator) +{ + if (!bi) + bi = db->best_chain; + + int step = 1; + while (bi) { + bp_locator_push(locator, &bi->hash); + + unsigned int i; + for (i = 0; bi && i < step; i++) + bi = bi->prev; + if (locator->vHave->len > 10) + step *= 2; + } + + bp_locator_push(locator, &db->block0); +} + diff --git a/lib/db/db.c b/lib/db/db.c new file mode 100644 index 00000000..4888fdd0 --- /dev/null +++ b/lib/db/db.c @@ -0,0 +1,265 @@ +/* Copyright 2012 exMULTI, Inc. + * Distributed under the MIT/X11 software license, see the accompanying + * file COPYING or http://www.opensource.org/licenses/mit-license.php. + */ + +#include // for db_handle, db_info, etc +#include // for chain_find_by_netmagic, etc +#include // for log_error, log_info, etc + +#include // for uint8_t +#include // for snprintf +#include // for memcmp, strlen +#include // for sysconf, _SC_PAGESIZE + +struct db_info dbinfo = {NULL, + {[METADB] = {"metadb", (MDB_dbi) 0, false}, + [BLOCKDB] = {"blockdb", (MDB_dbi) 0, false}, + [BLOCKHEIGHTDB] = {"blockheightdb", (MDB_dbi) 0, false},} +}; + +long get_pagesize() +{ +#ifdef PAGESIZE + return PAGESIZE; +#else + long sz = sysconf(_SC_PAGESIZE); + return sz; +#endif +} + +bool metadb_init(const unsigned char *netmagic, + const bu256_t *genesis_block) +{ + const struct chain_info *db_chain = chain_find_by_netmagic(netmagic); + + if (!db_chain) + return false; + + int mdb_rc; + MDB_txn *txn; + MDB_val key_nm, key_gen, data_nm, data_gen; + char db_filename[strlen(db_chain->name) + 4 + 1]; + enum metadb_key key_netmagic = NETMAGIC_KEY; + enum metadb_key key_genesis = GENESIS_KEY; + + key_nm.mv_size = sizeof(enum metadb_key); + key_nm.mv_data = &key_netmagic; + key_gen.mv_size = sizeof(enum metadb_key); + key_gen.mv_data = &key_genesis; + + snprintf(db_filename, sizeof(db_filename), "%s.mdb", db_chain->name); + + if ((mdb_rc = mdb_env_create(&dbinfo.env)) != MDB_SUCCESS) goto err_out; + if ((mdb_rc = mdb_env_set_mapsize(dbinfo.env,(size_t)(((MAX_DB_SIZE - 1) | (get_pagesize() - 1)) + 1))) != MDB_SUCCESS) goto err_out; + if ((mdb_rc = mdb_env_set_maxdbs(dbinfo.env, (MDB_dbi) MAX_NUM_DBS)) != MDB_SUCCESS) goto err_out; + log_debug("db: Opening database file '%s'", db_filename); + if ((mdb_rc = mdb_env_open(dbinfo.env, db_filename, MDB_NOSUBDIR, 0664)) != MDB_SUCCESS) goto err_out; + if ((mdb_rc = mdb_txn_begin(dbinfo.env, NULL, 0, &txn)) != MDB_SUCCESS) goto err_out; + + if ((mdb_rc = mdb_dbi_open(txn, dbinfo.handle[METADB].name, MDB_INTEGERKEY, &dbinfo.handle[METADB].dbi)) == MDB_SUCCESS) { + dbinfo.handle[METADB].open = true; + log_info("db: Opening %s database", dbinfo.handle[METADB].name); + + if ((mdb_rc = mdb_get(txn, dbinfo.handle[METADB].dbi, &key_nm, &data_nm)) != MDB_SUCCESS) goto err_abort; + if ((mdb_rc = mdb_get(txn, dbinfo.handle[METADB].dbi, &key_gen, &data_gen)) != MDB_SUCCESS) goto err_abort; + if ((mdb_rc = mdb_txn_commit(txn)) != MDB_SUCCESS) goto err_close; + + if ((data_nm.mv_size != NETMAGIC_LEN) || (memcmp(data_nm.mv_data, netmagic, NETMAGIC_LEN))) { + log_error("db: invalid netmagic"); + goto close_out; + } + + if ((data_gen.mv_size != sizeof(bu256_t)) || (!bu256_equal(data_gen.mv_data, genesis_block))) { + log_error("db: invalid genesis block"); + goto close_out; + } + + } else if (mdb_rc == MDB_NOTFOUND) { + log_info("db: Creating %s database", dbinfo.handle[METADB].name); + + data_nm.mv_size = NETMAGIC_LEN; + data_nm.mv_data = (unsigned char *) netmagic; + + data_gen.mv_size = sizeof(bu256_t); + data_gen.mv_data = (bu256_t *) genesis_block; + + if ((mdb_rc = mdb_dbi_open(txn, dbinfo.handle[METADB].name, MDB_CREATE, &dbinfo.handle[METADB].dbi)) != MDB_SUCCESS) goto err_abort; + dbinfo.handle[METADB].open = true; + + if ((mdb_rc = mdb_put(txn, dbinfo.handle[METADB].dbi, &key_nm, &data_nm, MDB_NOOVERWRITE | MDB_APPEND)) != MDB_SUCCESS) goto err_abort; + if ((mdb_rc = mdb_put(txn, dbinfo.handle[METADB].dbi, &key_gen, &data_gen, MDB_NOOVERWRITE | MDB_APPEND)) != MDB_SUCCESS) goto err_abort; + if ((mdb_rc = mdb_txn_commit(txn)) != MDB_SUCCESS) goto err_close; + } else goto err_abort; + + return true; + +close_out: + db_close(); + return false; +err_abort: + mdb_txn_abort(txn); +err_close: + db_close(); +err_out: + log_error("db: Database %s error '%s'", dbinfo.handle[METADB].name, mdb_strerror(mdb_rc)); + return false; +} + +bool blockdb_init(void) +{ + int mdb_rc; + MDB_txn *txn; + + if ((mdb_rc = mdb_txn_begin(dbinfo.env, NULL, 0, &txn)) != MDB_SUCCESS) goto err_out; + + log_info("db: Opening %s database", dbinfo.handle[BLOCKDB].name); + if ((mdb_rc = mdb_dbi_open(txn, dbinfo.handle[BLOCKDB].name, MDB_CREATE, &dbinfo.handle[BLOCKDB].dbi)) != MDB_SUCCESS) goto err_abort; + dbinfo.handle[BLOCKDB].open = true; + + if ((mdb_rc = mdb_txn_commit(txn)) != MDB_SUCCESS) goto err_close; + + return true; + +err_abort: + mdb_txn_abort(txn); +err_close: + db_close(); +err_out: + log_error("db: Database %s error '%s'", dbinfo.handle[BLOCKDB].name, mdb_strerror(mdb_rc)); + return false; +} + +bool blockdb_add(bu256_t *hash, struct const_buffer *buf) +{ + int mdb_rc; + MDB_txn *txn; + MDB_val key_hash, data_block; + char hexstr[BU256_STRSZ]; + + key_hash.mv_size = sizeof(bu256_t); + key_hash.mv_data = hash; + data_block.mv_size = buf->len; + data_block.mv_data = (void *)buf->p; + + if ((mdb_rc = mdb_txn_begin(dbinfo.env, NULL, 0, &txn)) != MDB_SUCCESS) goto err_out; + if (((mdb_rc = mdb_put(txn, dbinfo.handle[BLOCKDB].dbi, &key_hash, &data_block, MDB_NOOVERWRITE)) != MDB_SUCCESS) && (mdb_rc != MDB_KEYEXIST)) goto err_abort; + bu256_hex(hexstr, key_hash.mv_data); + if (mdb_rc == MDB_SUCCESS) { + log_info("db: Adding block %s to %s database", hexstr, dbinfo.handle[BLOCKDB].name); + } else if (mdb_rc == MDB_KEYEXIST) { + log_debug("db: Block %s already exists in %s database", hexstr, dbinfo.handle[BLOCKDB].name); + } + + if ((mdb_rc = mdb_txn_commit(txn)) != MDB_SUCCESS) goto err_out; + + return true; + +err_abort: + mdb_txn_abort(txn); +err_out: + log_error("db: Database %s error '%s'", dbinfo.handle[BLOCKDB].name, mdb_strerror(mdb_rc)); + return false; +} + +bool blockheightdb_init(void) +{ + int mdb_rc; + MDB_txn *txn; + + if ((mdb_rc = mdb_txn_begin(dbinfo.env, NULL, 0, &txn)) != MDB_SUCCESS) goto err_out; + + log_info("db: Opening %s database", dbinfo.handle[BLOCKHEIGHTDB].name); + if ((mdb_rc = mdb_dbi_open(txn, dbinfo.handle[BLOCKHEIGHTDB].name, MDB_CREATE | MDB_INTEGERKEY, &dbinfo.handle[BLOCKHEIGHTDB].dbi)) != MDB_SUCCESS) goto err_abort; + dbinfo.handle[BLOCKHEIGHTDB].open = true; + + if ((mdb_rc = mdb_txn_commit(txn)) != MDB_SUCCESS) goto err_close; + + return true; + +err_abort: + mdb_txn_abort(txn); +err_close: + db_close(); +err_out: + log_error("db: Database %s error '%s'", dbinfo.handle[BLOCKHEIGHTDB].name, mdb_strerror(mdb_rc)); + return false; +} + +bool blockheightdb_add(int height, bu256_t *hash) +{ + int mdb_rc; + MDB_txn *txn; + MDB_val key_height, data_hash; + char hexstr[BU256_STRSZ]; + bu256_hex(hexstr, hash); + + key_height.mv_size = sizeof(int); + key_height.mv_data = &height; + data_hash.mv_size = sizeof(bu256_t); + data_hash.mv_data = hash; + + if ((mdb_rc = mdb_txn_begin(dbinfo.env, NULL, 0, &txn)) != MDB_SUCCESS) goto err_out; + if (((mdb_rc = mdb_put(txn, dbinfo.handle[BLOCKHEIGHTDB].dbi, &key_height, &data_hash, MDB_APPEND)) != MDB_SUCCESS) && (mdb_rc != MDB_KEYEXIST)) goto err_abort; + if (mdb_rc == MDB_SUCCESS) { + log_debug("db: Adding %s with height %i to %s database", hexstr, *(int *)key_height.mv_data, dbinfo.handle[BLOCKHEIGHTDB].name); + } else if (mdb_rc == MDB_KEYEXIST) { + log_debug("db: Updating block height %i with hash %s in %s database", *(int *)key_height.mv_data, hexstr, dbinfo.handle[BLOCKHEIGHTDB].name); + } + if ((mdb_rc = mdb_txn_commit(txn)) != MDB_SUCCESS) goto err_out; + + return true; + +err_abort: + mdb_txn_abort(txn); +err_out: + log_error("db: Database %s error '%s'", dbinfo.handle[BLOCKHEIGHTDB].name, mdb_strerror(mdb_rc)); + return false; +} + +bool blockheightdb_getall(bool (*read_block)(void *p, size_t len)) +{ + int mdb_rc; + MDB_txn *txn; + MDB_cursor *cursorheight; + MDB_cursor_op op = MDB_FIRST; + MDB_val key_height, data_hash, data_block; + + if ((mdb_rc = mdb_txn_begin(dbinfo.env, NULL, MDB_RDONLY, &txn)) != MDB_SUCCESS) goto err_out; + if ((mdb_rc = mdb_cursor_open(txn, dbinfo.handle[BLOCKHEIGHTDB].dbi, &cursorheight)) != MDB_SUCCESS) goto err_abort; + + log_info("db: Reading %s database", dbinfo.handle[BLOCKHEIGHTDB].name); + while ((mdb_rc = mdb_cursor_get(cursorheight, &key_height, &data_hash, op)) == MDB_SUCCESS) { + if ((mdb_rc = mdb_get(txn, dbinfo.handle[BLOCKDB].dbi, &data_hash, &data_block)) != MDB_SUCCESS) goto err_abort; + read_block(data_block.mv_data, data_block.mv_size); + if (op != MDB_NEXT) op = MDB_NEXT; + } + + mdb_cursor_close(cursorheight); + mdb_txn_abort(txn); + return true; + +err_abort: + mdb_txn_abort(txn); +err_out: + log_error("db: Database %s error '%s'", dbinfo.handle[BLOCKHEIGHTDB].name, mdb_strerror(mdb_rc)); + return false; +} + +void db_close(void) { + + uint8_t i; + log_info("db: Closing databases"); + + for(i=METADB; i < MAX_NUM_DBS; i++) { + if (dbinfo.handle[i].open) { + mdb_dbi_close(dbinfo.env, dbinfo.handle[i].dbi); + dbinfo.handle[i].open = false; + log_debug("db: Closed %s database", dbinfo.handle[i].name); + } + } + + mdb_env_close(dbinfo.env); + + return; +} diff --git a/lib/net/net.c b/lib/net/net.c index 12532f36..adccfbe4 100644 --- a/lib/net/net.c +++ b/lib/net/net.c @@ -6,7 +6,7 @@ #include "ccoin/net/net.h" // for nc_conn, net_child_info, etc #include // for bn_address_str, etc -#include // for blkdb, blkdb_locator, etc +#include // for chaindb, chaindb_locator, etc #include // for buffer, const_buffer #include // for bp_address, bp_inv, etc #include // for ::CADDR_TIME_VERSION, etc @@ -317,7 +317,7 @@ static bool nc_msg_verack(struct nc_conn *conn) if (conn->nci->last_getblocks < cutoff) { struct msg_getblocks gb; msg_getblocks_init(&gb); - blkdb_locator(conn->nci->db, NULL, &gb.locator); + chaindb_locator(conn->nci->db, NULL, &gb.locator); cstring *s = ser_msg_getblocks(&gb); rc = nc_conn_send(conn, "getblocks", s->str, s->len); @@ -423,7 +423,9 @@ static bool nc_msg_block(struct nc_conn *conn) goto out; } - if (!conn->nci->block_process(&block, &conn->msg.hdr, &buf)) + struct const_buffer ser_data = { conn->msg.data, conn->msg.hdr.data_len }; + + if (!conn->nci->block_process(&block, &ser_data)) goto out; rc = true; diff --git a/src/.gitignore b/src/.gitignore index 59b1bae5..13fe4c23 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -1,14 +1,10 @@ - blkscan blkstats rawtx - brd -brd.peers -brd.blocks - picocoin -picocoin.peers -picocoin.blkdb -*.wallet +*.peers +*.wallet +*.mdb +*.mdb-lock diff --git a/src/Makefile.am b/src/Makefile.am index e1238a74..842fe1a2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -9,8 +9,9 @@ brd_SOURCES= \ brd.h brd_LDADD = $(top_builddir)/lib/libccoin.la \ + $(top_builddir)/lib/libccoindb.la \ $(top_builddir)/lib/libccoinnet.la \ - @GMP_LIBS@ @EVENT_LIBS@ + @GMP_LIBS@ @EVENT_LIBS@ @LMDB_LIBS@ picocoin_SOURCES= \ main.c \ @@ -18,10 +19,11 @@ picocoin_SOURCES= \ wallet.c wallet.h picocoin_LDADD = $(top_builddir)/lib/libccoin.la \ + $(top_builddir)/lib/libccoindb.la \ $(top_builddir)/lib/libccoinnet.la \ $(top_builddir)/lib/libccoinaes.la \ - @GMP_LIBS@ @ARGP_LIBS@ \ - @EVENT_LIBS@ @JANSSON_LIBS@ + @GMP_LIBS@ @ARGP_LIBS@ @EVENT_LIBS@ \ + @JANSSON_LIBS@ @LMDB_LIBS@ blkscan_LDADD = $(top_builddir)/lib/libccoin.la \ @GMP_LIBS@ @ARGP_LIBS@ diff --git a/src/brd.c b/src/brd.c index e9addda7..63a1bf9e 100644 --- a/src/brd.c +++ b/src/brd.c @@ -5,7 +5,8 @@ #include "picocoin-config.h" // for VERSION, _LARGE_FILES, etc #include "brd.h" -#include // for blkinfo, blkdb, etc +#include // for blkinfo, chaindb, etc +#include // for blockdb_add, blockdb_init, etc #include // for const_buffer, buffer_copy, etc #include // for clist_length #include // for bp_block, bp_utxo, bp_tx, etc @@ -22,27 +23,19 @@ #include // for bp_verify_sig #include // for ARRAY_SIZE, czstr_equal, etc - #include // for assert -#include // for bool +#include // for false, bool, true #include // for isspace #include // for errno #include // for event_base_dispatch, etc #include // for open #include // for signal, SIG_IGN, SIGHUP, etc -#include // for size_t -#include // for fprintf, NULL, fclose, etc +#include // for NULL, size_t +#include // for fclose, fopen, ferror, etc #include // for exit, free, calloc -#include // for strerror, strcmp, strlen, etc +#include // for strcmp, strlen, strdup, etc #include // for iovec, writev -#include // for lseek64, access, lseek, etc - -#if defined(__APPLE__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(__OpenBSD__) -# define off64_t off_t -# define lseek64 lseek -#endif - +#include // for for access, F_OK #if defined(__GNUC__) /* For add_orphan */ @@ -57,10 +50,10 @@ uint64_t instance_nonce; struct logging *log_state; bool debugging = false; -static struct blkdb db; +static char *peer_filename = NULL; +static struct chaindb db; static struct bp_hashtab *orphans; static struct bp_utxo_set uset; -static int blocks_fd = -1; static bool script_verf = false; static unsigned int net_conn_timeout = 11; struct net_child_info global_nci; @@ -68,13 +61,10 @@ struct net_child_info global_nci; static const char *const_settings[] = { "net.connect.timeout=11", "chain=bitcoin", - "peers=brd.peers", - /* "blkdb=brd.blkdb", */ - "blocks=brd.blocks", "log=-", /* "log=brd.log", */ }; -static bool block_process(const struct bp_block *block, int64_t fpos); +static bool block_process(const struct bp_block *block); static bool have_orphan(const bu256_t *v); static bool add_orphan(const bu256_t *hash_in, struct const_buffer *buf_in); @@ -211,33 +201,29 @@ static void init_log(void) } -static void init_blkdb(void) +static void init_chaindb(void) { - if (!blkdb_init(&db, chain->netmagic, &chain_genesis)) { - log_info("%s: blkdb init failed", prog_name); - exit(1); - } - - char *blkdb_fn = setting("blkdb"); - if (!blkdb_fn) - return; + char hexstr[BU256_STRSZ]; - if ((access(blkdb_fn, F_OK) == 0) && - !blkdb_read(&db, blkdb_fn)) { - log_info("%s: blkdb read failed", prog_name); + if (!chaindb_init(&db, chain->netmagic, &chain_genesis)) { + log_info("%s: chaindb initialisation failed", prog_name); exit(1); } - db.fd = open(blkdb_fn, - O_WRONLY | O_APPEND | O_CREAT | O_LARGEFILE, 0666); - if (db.fd < 0) { - log_info("%s: blkdb file open failed: %s", prog_name, strerror(errno)); + log_debug("%s: Initialised chaindb", prog_name); +} + +static void init_db(void) +{ + if (!metadb_init(chain->netmagic, &chain_genesis) || + !blockdb_init() || + !blockheightdb_init()) + { + log_error("%s: db initialisation failed", prog_name); exit(1); } - log_debug("%s: blkdb opened", prog_name); } - static const char *genesis_bitcoin = "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000"; static const char *genesis_testnet = @@ -264,47 +250,17 @@ static void init_block0(void) size_t genesis_rawlen = strlen(genesis_hex) / 2; char genesis_raw[genesis_rawlen]; if (!decode_hex(genesis_raw, sizeof(genesis_raw), genesis_hex, &olen)) { - log_info("%s: chain hex decode fail", prog_name); + log_error("%s: chain hex decode fail", prog_name); exit(1); } - cstring *msg0 = message_str(chain->netmagic, "block", - genesis_raw, genesis_rawlen); - ssize_t bwritten = write(blocks_fd, msg0->str, msg0->len); - if (bwritten != msg0->len) { - log_info("%s: blocks write0 failed: %s", prog_name, strerror(errno)); - exit(1); - } - cstr_free(msg0, true); + init_db(); - off64_t fpos64 = lseek64(blocks_fd, 0, SEEK_SET); - if (fpos64 == (off64_t)-1) { - log_info("%s: blocks lseek0 failed: %s", prog_name, strerror(errno)); - exit(1); + struct const_buffer buf0 = { genesis_raw, genesis_rawlen }; + if (blockdb_add(&chain_genesis, &buf0)) { + log_info("%s: Genesis block written to block database", prog_name); + blockheightdb_add(0, &chain_genesis); } - log_info("blocks: genesis block written"); -} - -static void init_blocks(void) -{ - char *blocks_fn = setting("blocks"); - if (!blocks_fn) - return; - - blocks_fd = open(blocks_fn, O_RDWR | O_CREAT | O_LARGEFILE, 0666); - if (blocks_fd < 0) { - log_info("%s: blocks file open failed: %s", prog_name, strerror(errno)); - exit(1); - } - - off64_t flen = lseek64(blocks_fd, 0, SEEK_END); - if (flen == (off64_t)-1) { - log_info("%s: blocks file lseek64 failed: %s", prog_name, strerror(errno)); - exit(1); - } - - if (flen == 0) - init_block0(); } static bool spend_tx(struct bp_utxo_set *uset, const struct bp_tx *tx, @@ -389,7 +345,7 @@ static bool spend_block(struct bp_utxo_set *uset, const struct bp_block *block, if (!spend_tx(uset, tx, i, height)) { char hexstr[BU256_STRSZ]; bu256_hex(hexstr, &tx->sha256); - log_info("%s: spent_block tx fail %s", prog_name, hexstr); + log_error("%s: spent_block tx fail %s", prog_name, hexstr); return false; } } @@ -397,18 +353,18 @@ static bool spend_block(struct bp_utxo_set *uset, const struct bp_block *block, return true; } -static bool block_process(const struct bp_block *block, int64_t fpos) +static bool block_process(const struct bp_block *block) { struct blkinfo *bi = bi_new(); bu256_copy(&bi->hash, &block->sha256); bp_block_copy_hdr(&bi->hdr, block); - bi->n_file = 0; - bi->n_pos = fpos; + char hexstr[BU256_STRSZ]; + bu256_hex(hexstr, &bi->hash); - struct blkdb_reorg reorg; + struct chaindb_reorg reorg; - if (!blkdb_add(&db, bi, &reorg)) { - log_info("%s: blkdb add fail", prog_name); + if (!chaindb_add(&db, bi, &reorg)) { + log_debug("%s: Adding block %s to chaindb failed", prog_name, hexstr); goto err_out; } @@ -419,13 +375,11 @@ static bool block_process(const struct bp_block *block, int64_t fpos) /* if best chain, mark TX's as spent */ if (bu256_equal(&db.best_chain->hash, &bi->hdr.sha256)) { if (!spend_block(&uset, block, bi->height)) { - char hexstr[BU256_STRSZ]; bu256_hex(hexstr, &bi->hdr.sha256); log_info("%s: block spend fail %u %s", prog_name, bi->height, hexstr); - /* FIXME: bad record is now in blkdb */ - + /* FIXME: bad record is now in chaindb */ goto err_out; } } @@ -437,21 +391,15 @@ static bool block_process(const struct bp_block *block, int64_t fpos) return false; } -static bool read_block_msg(struct p2p_message *msg, int64_t fpos) +static bool read_block(void *p, size_t len) { - /* unknown records are invalid */ - if (strncmp(msg->hdr.command, "block", - sizeof(msg->hdr.command))) - return false; - bool rc = false; struct bp_block block; bp_block_init(&block); - - struct const_buffer buf = { msg->data, msg->hdr.data_len }; + struct const_buffer buf = { p, len }; if (!deser_bp_block(&block, &buf)) { - log_info("%s: block deser fail", prog_name); + log_error("%s: block deser fail", prog_name); goto out; } bp_block_calc_sha256(&block); @@ -461,62 +409,16 @@ static bool read_block_msg(struct p2p_message *msg, int64_t fpos) goto out; } - rc = block_process(&block, fpos); + /* used at runtime */ + bool sha256_valid; + bu256_t sha256; + rc = block_process(&block); out: bp_block_free(&block); return rc; } -static void read_blocks(void) -{ - int fd = blocks_fd; - - struct p2p_message msg = {}; - bool read_ok = true; - int64_t fpos = 0; - while (fread_message(fd, &msg, &read_ok)) { - if (memcmp(msg.hdr.netmagic, chain->netmagic, 4)) { - log_info("blocks file: invalid network magic"); - exit(1); - } - - if (!read_block_msg(&msg, fpos)) - exit(1); - - fpos += P2P_HDR_SZ; - fpos += msg.hdr.data_len; - } - - if (!read_ok) { - log_info("blocks file: read failed"); - exit(1); - } - - free(msg.data); -} - -static void readprep_blocks_file(void) -{ - /* if no blk index, but blocks are present, read and index - * all block data (several gigabytes) - */ - if (blocks_fd >= 0) { - if (db.fd < 0) - read_blocks(); - else { - /* TODO: verify that blocks file offsets are - * present in blkdb */ - - if (lseek(blocks_fd, 0, SEEK_END) == (off_t)-1) { - log_info("blocks file: seek failed: %s", - strerror(errno)); - exit(1); - } - } - } -} - static void init_orphans(void) { orphans = bp_hashtab_new_ext(bu256_hash, bu256_equal_, @@ -558,12 +460,12 @@ static void init_peers(struct net_child_info *nci) */ struct peer_manager *peers; - peers = peerman_read(setting("peers")); + peers = peerman_read(peer_filename); if (!peers) { log_info("%s: initializing empty peer list", prog_name); peers = peerman_seed(setting("no_dns") == NULL ? true : false); - if (!peerman_write(peers, setting("peers"), chain)) { + if (!peerman_write(peers, peer_filename, chain)) { log_info("%s: failed to write peer list", prog_name); exit(1); } @@ -585,44 +487,21 @@ static void init_peers(struct net_child_info *nci) static bool inv_block_process(bu256_t *hash) { - return (!blkdb_lookup(&db, hash) && + return (!chaindb_lookup(&db, hash) && !have_orphan(hash)); } -static bool add_block(struct bp_block *block, struct p2p_message_hdr *hdr, struct const_buffer *buf) +static bool add_block(struct bp_block *block, struct const_buffer *buf) { /* check for duplicate block */ - if (blkdb_lookup(&db, &block->sha256) || + if (chaindb_lookup(&db, &block->sha256) || have_orphan(&block->sha256)) return true; - struct iovec iov[2]; - iov[0].iov_base = &hdr; // TODO: endian bug? - iov[0].iov_len = sizeof(hdr); - iov[1].iov_base = (void *) buf->p; // cast away 'const' - iov[1].iov_len = buf->len; - size_t total_write = iov[0].iov_len + iov[1].iov_len; - - /* store current file position */ - off64_t fpos64 = lseek64(blocks_fd, 0, SEEK_CUR); - if (fpos64 == (off64_t)-1) { - log_info("blocks: lseek64 failed %s", strerror(errno)); - return false; - } - - /* write new block to disk */ - errno = 0; - ssize_t bwritten = writev(blocks_fd, iov, ARRAY_SIZE(iov)); - if (bwritten != total_write) { - log_info("blocks: write failed %s", strerror(errno)); - return false; - } - + blockdb_add(&block->sha256, buf); /* process block */ - if (!block_process(block, fpos64)) { - log_info("blocks: process-block failed"); + if (!block_process(block)) return false; - } return true; @@ -647,11 +526,11 @@ static void init_nci(struct net_child_info *nci) static void init_daemon(struct net_child_info *nci) { - init_blkdb(); + init_chaindb(); bp_utxo_set_init(&uset); - init_blocks(); + init_block0(); init_orphans(); - readprep_blocks_file(); + blockheightdb_getall(read_block); init_nci(nci); } @@ -675,12 +554,14 @@ static void shutdown_nci(struct net_child_info *nci) static void shutdown_daemon(struct net_child_info *nci) { - bool rc = peerman_write(nci->peers, setting("peers"), chain); - log_info("blocks: %s %u/%zu peers", + bool rc = peerman_write(nci->peers, peer_filename, chain); + log_info("%s: %s %u/%zu peers", prog_name, rc ? "wrote" : "failed to write", bp_hashtab_size(nci->peers->map_addr), clist_length(nci->peers->addrlist)); + db_close(); + if (log_state->logtofile) { fclose(log_state->stream); log_state->stream = NULL; @@ -691,7 +572,7 @@ static void shutdown_daemon(struct net_child_info *nci) shutdown_nci(nci); bp_hashtab_unref(orphans); bp_hashtab_unref(settings); - blkdb_free(&db); + chaindb_free(&db); bp_utxo_set_free(&uset); } } @@ -725,6 +606,10 @@ int main (int argc, char *argv[]) init_log(); chain_set(); + char peer_filename_tmp[strlen(chain->name) + 6 + 1]; + snprintf(peer_filename_tmp, sizeof(peer_filename_tmp), "%s.peers", chain->name); + peer_filename = peer_filename_tmp; + /* * properly capture TERM and other signals */ diff --git a/src/main.c b/src/main.c index 3887b68e..7cbd1e30 100644 --- a/src/main.c +++ b/src/main.c @@ -2,10 +2,10 @@ * Distributed under the MIT/X11 software license, see the accompanying * file COPYING or http://www.opensource.org/licenses/mit-license.php. */ -#include "picocoin-config.h" // for VERSION +#include "picocoin-config.h" // for PACKAGE_BUGREPORT, etc -#include "picocoin.h" // for network_sync, setting -#include // for blkinfo, blkdb, etc +#include "picocoin.h" // for setting +#include // for blkinfo, chaindb, etc #include // for clist, clist_free_ext, etc #include // for strndup #include // for bp_address @@ -16,21 +16,22 @@ #include // for net_child_info, nc_conns_gc, etc #include // for bn_address_str, etc #include // for peer_manager, peerman_write, etc +#include // for parr_free, parr_new, parr #include // for ARRAY_SIZE, czstr_equal, etc - #include "wallet.h" // for cur_wallet_addresses, etc +#include + #include #include // for assert #include // for bool #include // for isspace -#include // for errno #include // for event_free, event_base_new, etc -#include // for open #include // for fprintf, printf, NULL, etc +#include // for uint64_t #include // for free, exit #include // for strcmp, strdup, strlen, etc -#include +#include // for sleep enum command_type { CMD_CHAIN_SET, @@ -55,8 +56,9 @@ struct logging *log_state; bool debugging = false; static enum command_type opt_command = CMD_WALLET_INFO; static const char *opt_arg1 = NULL; +static char *peer_filename = NULL; -static struct blkdb db; +static struct chaindb db; static unsigned int net_conn_timeout = 60; struct wallet *cur_wallet; @@ -66,8 +68,6 @@ static const char *const_settings[] = { "net.connect.timeout=11", "wallet=picocoin.wallet", "chain=bitcoin", - "peers=picocoin.peers", - "blkdb=picocoin.blkdb", }; /* Command line arguments and processing */ @@ -490,12 +490,12 @@ static void init_peers(struct net_child_info *nci) */ struct peer_manager *peers; - peers = peerman_read(setting("peers")); + peers = peerman_read(peer_filename); if (!peers) { log_info("%s: initializing empty peer list", prog_name); peers = peerman_seed(setting("no_dns") == NULL ? true : false); - if (!peerman_write(peers, setting("peers"), chain)) { + if (!peerman_write(peers, peer_filename, chain)) { log_info("%s: failed to write peer list", prog_name); exit(1); } @@ -515,31 +515,14 @@ static void init_peers(struct net_child_info *nci) nci->peers = peers; } -static void init_blkdb(void) +static void init_chaindb(void) { - if (!blkdb_init(&db, chain->netmagic, &chain_genesis)) { - log_info("%s: blkdb init failed", prog_name); - exit(1); - } - - char *blkdb_fn = setting("blkdb"); - if (!blkdb_fn) - return; - - if ((access(blkdb_fn, F_OK) == 0) && - !blkdb_read(&db, blkdb_fn)) { - log_info("%s: blkdb read failed", prog_name); - exit(1); - } - - db.fd = open(blkdb_fn, - O_WRONLY | O_APPEND | O_CREAT | O_LARGEFILE, 0666); - if (db.fd < 0) { - log_info("%s: blkdb file open failed: %s", prog_name, strerror(errno)); + if (!chaindb_init(&db, chain->netmagic, &chain_genesis)) { + log_info("%s: chaindb init failed", prog_name); exit(1); } - log_debug("%s: blkdb opened", prog_name); + log_debug("%s: chaindb opened", prog_name); } static void shutdown_nci(struct net_child_info *nci) @@ -591,7 +574,7 @@ static void network_child(int read_fd, int write_fd) event_base_dispatch(nci.eb); } - init_blkdb(); + init_chaindb(); init_peers(&nci); @@ -602,8 +585,8 @@ static void network_child(int read_fd, int write_fd) } while (nci.running); /* cleanup: just the minimum for file I/O correctness */ - peerman_write(nci.peers, setting("peers"), nci.chain); - blkdb_free(nci.db); + peerman_write(nci.peers, peer_filename, nci.chain); + chaindb_free(nci.db); shutdown_nci(&nci); exit(0); } @@ -651,6 +634,10 @@ int main (int argc, char *argv[]) init_log(); chain_set(); + char peer_filename_tmp[strlen(chain->name) + 6 + 1]; + snprintf(peer_filename_tmp, sizeof(peer_filename_tmp), "%s.peers", chain->name); + peer_filename = peer_filename_tmp; + switch (opt_command) { case CMD_CHAIN_SET: chain_set(); break; case CMD_DNS_SEEDS: list_dns_seeds(); break; diff --git a/test/.gitignore b/test/.gitignore index b7c100c6..9c5c76f6 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -6,7 +6,7 @@ base58 block blockfile bloom -blkdb +chaindb chain-verf clist cstr @@ -38,3 +38,5 @@ wallet-basics *.trs *.log aes_util.dat +*.mdb +*.mdb-lock diff --git a/test/Makefile.am b/test/Makefile.am index 7247128a..a86120c8 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -1,5 +1,5 @@ if ENABLE_VALGRIND - TESTS_ENVIRONMENT=$(LIBTOOL) --mode=execute valgrind --leak-check=full \ + TESTS_ENVIRONMENT = $(LIBTOOL) --mode=execute valgrind --leak-check=full \ --leak-resolution=high --num-callers=40 \ --error-exitcode=1 endif @@ -12,28 +12,30 @@ AM_CPPFLAGS = -DTEST_SRCDIR=\"$(top_srcdir)/test\" \ noinst_LIBRARIES= libtest.a -libtest_a_SOURCES= libtest.h libtest.c chisq.c randtest.c +libtest_a_SOURCES = libtest.h libtest.c chisq.c randtest.c noinst_PROGRAMS = clist cstr coredefs hex hdkeys hashtab base58 fileio util \ crypto keystore keyset bloom mbr misc net sighash \ - message parr prng script-parse tx block blockfile blkdb script \ + message parr prng script-parse tx block blockfile chaindb script \ tx-valid wallet wallet-basics chain-verf hash ctaes aes-util TESTS = clist cstr coredefs hex hdkeys hashtab base58 fileio util \ crypto keystore keyset bloom mbr misc net sighash \ - message parr prng script-parse tx block blockfile blkdb script \ + message parr prng script-parse tx block blockfile chaindb script \ tx-valid wallet wallet-basics chain-verf hash ctaes aes-util +CLEANFILES = *.mdb *.mdb-lock + COMMON_LDADD = libtest.a $(top_builddir)/lib/libccoin.la \ $(top_builddir)/external/secp256k1/libsecp256k1.la \ @GMP_LIBS@ @JANSSON_LIBS@ @MATH_LIBS@ base58_LDADD = $(COMMON_LDADD) -blkdb_LDADD = $(COMMON_LDADD) block_LDADD = $(COMMON_LDADD) blockfile_LDADD = $(COMMON_LDADD) bloom_LDADD = $(COMMON_LDADD) -chain_verf_LDADD = $(COMMON_LDADD) +chaindb_LDADD = $(top_builddir)/lib/libccoindb.la @LMDB_LIBS@ $(COMMON_LDADD) +chain_verf_LDADD = $(top_builddir)/lib/libccoindb.la @LMDB_LIBS@ $(COMMON_LDADD) clist_LDADD = $(COMMON_LDADD) cstr_LDADD = $(COMMON_LDADD) coredefs_LDADD = $(COMMON_LDADD) diff --git a/test/blkdb.c b/test/blkdb.c deleted file mode 100644 index 50bd8b2b..00000000 --- a/test/blkdb.c +++ /dev/null @@ -1,107 +0,0 @@ -/* Copyright 2012 exMULTI, Inc. - * Distributed under the MIT/X11 software license, see the accompanying - * file COPYING or http://www.opensource.org/licenses/mit-license.php. - */ -#include "picocoin-config.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "libtest.h" - -static void add_header(struct blkdb *db, char *raw) -{ - struct const_buffer buf = { raw, 80 }; - - struct blkinfo *bi = bi_new(); - assert(bi != NULL); - - assert(deser_bp_block(&bi->hdr, &buf) == true); - - bp_block_calc_sha256(&bi->hdr); - - bu256_copy(&bi->hash, &bi->hdr.sha256); - - struct blkdb_reorg reorg; - - assert(blkdb_add(db, bi, &reorg) == true); - - assert(reorg.conn == 1); - assert(reorg.disconn == 0); -} - -static void read_headers(const char *ser_base_fn, struct blkdb *db) -{ - char *filename = test_filename(ser_base_fn); - int fd = file_seq_open(filename); - assert(fd >= 0); - - char hdrbuf[80]; - - while (read(fd, hdrbuf, 80) == 80) { - add_header(db, hdrbuf); - } - - close(fd); - free(filename); -} - -static void test_blkinfo_prev(struct blkdb *db) -{ - struct blkinfo *tmp = db->best_chain; - int height = db->best_chain->height; - - while (tmp) { - assert(height == tmp->height); - - height--; - tmp = tmp->prev; - } - - assert(height == -1); -} - -static void runtest(const char *ser_base_fn, const struct chain_info *chain, - unsigned int check_height, const char *check_hash) -{ - struct blkdb db; - - bu256_t block0; - bool rc = hex_bu256(&block0, chain->genesis_hash); - assert(rc); - - rc = blkdb_init(&db, chain->netmagic, &block0); - assert(rc); - - read_headers(ser_base_fn, &db); - - assert(db.best_chain->height == check_height); - - bu256_t best_block; - rc = hex_bu256(&best_block, check_hash); - - assert(bu256_equal(&db.best_chain->hash, &best_block)); - - test_blkinfo_prev(&db); - - blkdb_free(&db); -} - -int main (int argc, char *argv[]) -{ - runtest("data/hdr193000.ser", &chain_metadata[CHAIN_BITCOIN], 193000, - "000000000000059f452a5f7340de6682a977387c17010ff6e6c3bd83ca8b1317"); - runtest("data/tn_hdr35141.ser", &chain_metadata[CHAIN_TESTNET3], 35141, - "0000000000dde6ce4b9ad1e2a5be59f1b7ace6ef8d077d846263b0bfbc984f7f"); - - bp_key_static_shutdown(); - return 0; -} diff --git a/test/chain-verf.c b/test/chain-verf.c index a946434b..c7c9b8cc 100644 --- a/test/chain-verf.c +++ b/test/chain-verf.c @@ -1,24 +1,34 @@ -#include "picocoin-config.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "libtest.h" +/* Copyright 2012 exMULTI, Inc. + * Distributed under the MIT/X11 software license, see the accompanying + * file COPYING or http://www.opensource.org/licenses/mit-license.php. + */ + +#include // for const_buffer +#include // for bu256_hex, BU256_STRSZ, etc +#include // for bp_ckpt_last +#include // for bp_block, bp_tx, bp_utxo, etc +#include // for chain_info, etc +#include // for blkinfo, chaindb_reorg, etc +#include // for blockdb_init, etc +#include // for bp_key_static_shutdown +#include // for logging +#include // for fread_block +#include // for p2p_message, etc +#include // for parr, parr_id +#include // for bp_verify_sig +#include // for file_seq_open + +#include // for assert +#include // for true, false, bool +#include // for int64_t +#include // for fprintf, stderr, perror, etc +#include // for getenv, calloc, free +#include // for memcmp, strncmp +#include // for close static bool no_script_verf = false; static bool force_script_verf = false; +struct logging *log_state; static bool spend_tx(struct bp_utxo_set *uset, const struct bp_tx *tx, unsigned int tx_idx, unsigned int height, @@ -124,7 +134,7 @@ static bool spend_block(struct bp_utxo_set *uset, const struct bp_block *block, return true; } -static void read_test_msg(struct blkdb *db, struct bp_utxo_set *uset, +static void read_test_msg(struct chaindb *db, struct bp_utxo_set *uset, const struct p2p_message *msg, int64_t fpos, unsigned int ckpt_height) { @@ -143,12 +153,10 @@ static void read_test_msg(struct blkdb *db, struct bp_utxo_set *uset, struct blkinfo *bi = bi_new(); bu256_copy(&bi->hash, &block.sha256); bp_block_copy_hdr(&bi->hdr, &block); - bi->n_file = 0; - bi->n_pos = fpos + P2P_HDR_SZ; - struct blkdb_reorg reorg; + struct chaindb_reorg reorg; - assert(blkdb_add(db, bi, &reorg) == true); + assert(chaindb_add(db, bi, &reorg) == true); assert(reorg.conn == 1); assert(reorg.disconn == 0); @@ -175,11 +183,15 @@ static void runtest(bool use_testnet, const char *blocks_fn) unsigned int ckpt_height = bp_ckpt_last(chain_id); - struct blkdb blkdb; + struct chaindb chaindb; bu256_t blk0; hex_bu256(&blk0, chain->genesis_hash); - assert(blkdb_init(&blkdb, chain->netmagic, &blk0) == true); + assert(metadb_init(chain_metadata[chain_id].netmagic, (const bu256_t *)chain_metadata[chain_id].genesis_hash)); + assert(blockdb_init()); + assert(blockheightdb_init()); + + assert(chaindb_init(&chaindb, chain->netmagic, &blk0) == true); struct bp_utxo_set uset; bp_utxo_set_init(&uset); @@ -203,7 +215,7 @@ static void runtest(bool use_testnet, const char *blocks_fn) while (fread_block(fd, &msg, &read_ok)) { assert(memcmp(msg.hdr.netmagic, chain->netmagic, 4) == 0); - read_test_msg(&blkdb, &uset, &msg, fpos, ckpt_height); + read_test_msg(&chaindb, &uset, &msg, fpos, ckpt_height); fpos += P2P_HDR_SZ; fpos += msg.hdr.data_len; @@ -215,7 +227,7 @@ static void runtest(bool use_testnet, const char *blocks_fn) close(fd); free(msg.data); - blkdb_free(&blkdb); + chaindb_free(&chaindb); bp_utxo_set_free(&uset); fprintf(stderr, "chain-verf: %u records validated\n", records); @@ -226,6 +238,12 @@ int main (int argc, char *argv[]) char *fn; unsigned int verfd = 0; + log_state = calloc(0, sizeof(struct logging)); + + log_state->stream = stderr; + log_state->logtofile = false; + log_state->debug = true; + if (getenv("NO_SCRIPT_VERF")) no_script_verf = true; if (getenv("FORCE_SCRIPT_VERF")) { diff --git a/test/chaindb.c b/test/chaindb.c new file mode 100644 index 00000000..4c32f8fb --- /dev/null +++ b/test/chaindb.c @@ -0,0 +1,126 @@ +/* Copyright 2012 exMULTI, Inc. + * Distributed under the MIT/X11 software license, see the accompanying + * file COPYING or http://www.opensource.org/licenses/mit-license.php. + */ + +#include // for const_buffer +#include // for hex_bu256, bu256_copy, etc +#include // for bp_block_calc_sha256, etc +#include // for chain_info, chain_metadata, etc +#include // for blkinfo, chaindb, chaindb_reorg, etc +#include // for blockdb_init, etc +#include // for bp_key_static_shutdown +#include // for logging +#include // for file_seq_open + +#include // for assert +#include // for stderr +#include // for free, calloc, NULL +#include // for close, read +#include // for true, bool, false +#include "libtest.h" // for test_filename + +struct logging *log_state; + +static void add_header(struct chaindb *db, char *raw) +{ + struct const_buffer buf = { raw, 80 }; + + struct blkinfo *bi = bi_new(); + assert(bi != NULL); + + assert(deser_bp_block(&bi->hdr, &buf) == true); + + bp_block_calc_sha256(&bi->hdr); + + bu256_copy(&bi->hash, &bi->hdr.sha256); + + struct chaindb_reorg reorg; + + assert(chaindb_add(db, bi, &reorg) == true); + + assert(reorg.conn == 1); + assert(reorg.disconn == 0); +} + +static void read_headers(const char *ser_base_fn, struct chaindb *db) +{ + char *filename = test_filename(ser_base_fn); + int fd = file_seq_open(filename); + assert(fd >= 0); + + char hdrbuf[80]; + + while (read(fd, hdrbuf, 80) == 80) { + add_header(db, hdrbuf); + } + + close(fd); + free(filename); +} + +static void test_blkinfo_prev(struct chaindb *db) +{ + struct blkinfo *tmp = db->best_chain; + int height = db->best_chain->height; + + while (tmp) { + assert(height == tmp->height); + + height--; + tmp = tmp->prev; + } + + assert(height == -1); +} + +static void runtest(const char *ser_base_fn, const struct chain_info *chain, + unsigned int check_height, const char *check_hash) +{ + struct chaindb db; + + bu256_t block0; + bool rc = hex_bu256(&block0, chain->genesis_hash); + assert(rc); + + rc = chaindb_init(&db, chain->netmagic, &block0); + assert(rc); + + read_headers(ser_base_fn, &db); + + assert(db.best_chain->height == check_height); + + bu256_t best_block; + rc = hex_bu256(&best_block, check_hash); + + assert(bu256_equal(&db.best_chain->hash, &best_block)); + + test_blkinfo_prev(&db); + + chaindb_free(&db); +} + +int main (int argc, char *argv[]) +{ + log_state = calloc(0, sizeof(struct logging)); + + log_state->stream = stderr; + log_state->logtofile = false; + log_state->debug = true; + + assert(metadb_init(chain_metadata[CHAIN_BITCOIN].netmagic, (const bu256_t *)chain_metadata[CHAIN_BITCOIN].genesis_hash)); + assert(blockdb_init()); + assert(blockheightdb_init()); + runtest("data/hdr50000.ser", &chain_metadata[CHAIN_BITCOIN], 50000, + "000000001aeae195809d120b5d66a39c83eb48792e068f8ea1fea19d84a4278a"); + + assert(metadb_init(chain_metadata[CHAIN_TESTNET3].netmagic, (const bu256_t *)chain_metadata[CHAIN_TESTNET3].genesis_hash)); + assert(blockdb_init()); + assert(blockheightdb_init()); + runtest("data/tn_hdr25000.ser", &chain_metadata[CHAIN_TESTNET3], 25000, + "0000000022b23de294af24d922fb3f1ed21521a8b3bd7716861dcb5310b1b525"); + + bp_key_static_shutdown(); + free(log_state); + return 0; +} diff --git a/test/data/Makefile.am b/test/data/Makefile.am index 3ca6793b..623d9a2a 100644 --- a/test/data/Makefile.am +++ b/test/data/Makefile.am @@ -4,7 +4,7 @@ EXTRA_DIST = \ base58_keys_invalid.json base58_keys_valid.json \ blk0.json blk0.ser blks10.ser \ blk120383.json blk120383.ser \ - hdr193000.ser tn_hdr35141.ser \ + hdr50000.ser tn_hdr25000.ser \ random.data script_tests.json sighash.json \ tn_blk35133.ser tx3e0dc3da.json tx3e0dc3da.ser \ tx_invalid.json tx_valid.json \ diff --git a/test/data/hdr193000.ser b/test/data/hdr193000.ser deleted file mode 100644 index c49fad13..00000000 Binary files a/test/data/hdr193000.ser and /dev/null differ diff --git a/test/data/hdr50000.ser b/test/data/hdr50000.ser new file mode 100644 index 00000000..bd2ca23d Binary files /dev/null and b/test/data/hdr50000.ser differ diff --git a/test/data/tn_hdr35141.ser b/test/data/tn_hdr25000.ser similarity index 71% rename from test/data/tn_hdr35141.ser rename to test/data/tn_hdr25000.ser index fe6db771..dee95681 100644 Binary files a/test/data/tn_hdr35141.ser and b/test/data/tn_hdr25000.ser differ