Use lmdb database in picocoin

.travis.yml

@@ -11,6 +11,8 @@ os:
 
 osx_image: xcode8
 
+dist: trusty
+
 sudo: false
 
 env:
@@ -34,6 +36,7 @@ addons:
       - libgmp-dev
       - libjansson-dev
       - libevent-dev
+      - liblmdb-dev
       - lcov
       - valgrind
       - libtool
@@ -53,6 +56,7 @@ before_install:
 install:
   - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install jansson; fi
   - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install libevent; fi
+  - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install lmdb; fi
   - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install argp-standalone; fi
   - if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "$ENABLE_COVERAGE" = "yes" ]; then brew install lcov; fi
   - if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "$ENABLE_VALGRIND" = "yes" ]; then brew install valgrind; fi

configure.ac

@@ -75,12 +75,14 @@ AC_CHECK_LIB(event_core, event_base_new, EVENT_LIBS=-levent_core,
   [AC_MSG_ERROR([Missing required libevent])])
 AC_CHECK_LIB(jansson, json_loads, JANSSON_LIBS=-ljansson,
   [AC_MSG_ERROR([Missing required libjansson])])
-AC_CHECK_LIB(argp, argp_parse, ARGP_LIBS=-largp)
+AC_CHECK_LIB(argp, argp_parse, ARGP_LIBS=-lARGP)
+AC_CHECK_LIB(lmdb, mdb_dbi_open, LMDB_LIBS=-llmdb,
+  [AC_MSG_ERROR([Missing required liblmdb])])
 
 dnl -------------------------------------
 dnl Checks for optional library functions
 dnl -------------------------------------
-AC_CHECK_FUNCS(fdatasync memmem strndup mkstemp)
+AC_CHECK_FUNCS(memmem strndup mkstemp)
 
 dnl -----------------
 dnl Configure options
@@ -152,6 +154,7 @@ AC_SUBST(GMP_LIBS)
 AC_SUBST(EVENT_LIBS)
 AC_SUBST(JANSSON_LIBS)
 AC_SUBST(ARGP_LIBS)
+AC_SUBST(LMDB_LIBS)
 
 AC_CONFIG_SUBDIRS([external/secp256k1])
 AC_CONFIG_FILES([

contrib/debian/control

@@ -1,7 +1,7 @@
 Source: picocoin
 Priority: optional
 Maintainer: Joel DeJesus <dejesus.joel@e-flamingo.jp>
-Build-Depends: debhelper (>= 8.0.0), autotools-dev, libgmp-dev, libevent-dev, libjansson-dev, libc6-dev
+Build-Depends: debhelper (>= 8.0.0), autotools-dev, libgmp-dev, libevent-dev, libjansson-dev, liblmdb-dev, libc6-dev
 Standards-Version: 3.9.4
 Section: libs
 #Homepage: <insert the upstream URL, if relevant>

doc/build-osx.md

@@ -16,11 +16,11 @@ and `jansson` to build *picocoin*.
 
 Install these packages. It will take a few minutes.
 
-    brew install autoconf automake libtool argp-standalone jansson libevent gmp
+    brew install autoconf automake libtool argp-standalone jansson libevent liblmdb gmp
 
 or
 
-    sudo port install autoconf automake libtool argp-standalone jansson libevent pkgconfig gmp
+    sudo port install autoconf automake libtool argp-standalone jansson libevent liblmdb pkgconfig gmp
 
 
 Building
@@ -61,7 +61,6 @@ You should see output formatted in JSON,
       "chain": "bitcoin",
       "net.connect.timeout": "11",
       "peers": "picocoin.peers",
-      "blkdb": "picocoin.blkdb"
     }
 
 If that works, `picocoin` is ready for use.

include/ccoin/Makefile.am

@@ -13,7 +13,6 @@ ccoininclude_HEADERS = \
 	address.h	\
 	addr_match.h	\
 	base58.h	\
-	blkdb.h		\
 	bloom.h		\
 	buffer.h	\
 	buint.h		\
@@ -37,6 +36,12 @@ ccoininclude_HEADERS = \
 	util.h		\
 	wallet.h
 
+ccoindbincludedir=$(includedir)/ccoin/db
+
+ccoindbinclude_HEADERS = \
+    db/chaindb.h  \
+    db/db.h
+
 ccoinnetincludedir=$(includedir)/ccoin/net
 
 ccoinnetinclude_HEADERS = \

include/ccoin/buffer.h

@@ -5,7 +5,8 @@
  * file COPYING or http://www.opensource.org/licenses/mit-license.php.
  */
 
-#include <stdbool.h>
+#include <stdbool.h>                    // for bool
+#include <stddef.h>                     // for size_t
 
 #ifdef __cplusplus
 extern "C" {

include/ccoin/compat.h

@@ -7,23 +7,12 @@
  * alongside all the other API headers
  */
 
-#include <unistd.h>
+#include <stddef.h>                     // for size_t
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#if defined(__APPLE__) || !defined(HAVE_FDATASYNC)
-static inline int fdatasync(int fd)
-{
-#ifdef WIN32
-	return _commit(fd);
-#else
-	return fsync(fd);
-#endif
-}
-#endif /* !HAVE_FDATASYNC */
-
 #ifndef HAVE_MEMMEM
 extern void *memmem(const void *haystack, size_t haystacklen,
                     const void *needle, size_t needlelen);

include/ccoin/blkdb.h → include/ccoin/db/chaindb.h

@@ -1,14 +1,17 @@
-#ifndef __LIBCCOIN_BLKDB_H__
-#define __LIBCCOIN_BLKDB_H__
+#ifndef __LIBCCOIN_CHAINDB_H__
+#define __LIBCCOIN_CHAINDB_H__
 /* Copyright 2012 exMULTI, Inc.
  * Distributed under the MIT/X11 software license, see the accompanying
  * file COPYING or http://www.opensource.org/licenses/mit-license.php.
  */
 
-#include <stdbool.h>
-#include <ccoin/core.h>
-#include <ccoin/buint.h>
-#include <ccoin/hashtab.h>
+#include <ccoin/buint.h>                // for bu256_t
+#include <ccoin/core.h>                 // for bp_block
+#include <ccoin/hashtab.h>              // for bp_hashtab_get
+
+#include <gmp.h>                        // for mpz_t
+
+#include <stdbool.h>                    // for bool
 
 #ifdef __cplusplus
 extern "C" {
@@ -23,24 +26,16 @@ struct blkinfo {
 	mpz_t		work;
 	int		height;
 
-	int32_t		n_file;		/* uninitialized == -1 */
-	int64_t		n_pos;		/* uninitialized == -1 */
-
 	struct blkinfo	*prev;
 };
 
-struct blkdb_reorg {
+struct chaindb_reorg {
 	struct blkinfo	*old_best;	/* previous best_chain */
 	unsigned int	conn;		/* # blocks connected (normally 1) */
 	unsigned int	disconn;	/* # blocks disconnected (normally 0) */
 };
 
-struct blkdb {
-	int		fd;
-	bool		datasync_fd;
-	bool		close_fd;
-
-	unsigned char	netmagic[4];
+struct chaindb {
 	bu256_t		block0;
 
 	struct bp_hashtab *blocks;
@@ -51,16 +46,15 @@ struct blkdb {
 extern struct blkinfo *bi_new(void);
 extern void bi_free(struct blkinfo *bi);
 
-extern bool blkdb_init(struct blkdb *db, const unsigned char *netmagic,
+extern bool chaindb_init(struct chaindb *db, const unsigned char *netmagic,
 		       const bu256_t *genesis_block);
-extern void blkdb_free(struct blkdb *db);
-extern bool blkdb_read(struct blkdb *db, const char *idx_fn);
-extern bool blkdb_add(struct blkdb *db, struct blkinfo *bi,
-		      struct blkdb_reorg *reorg_info);
-extern void blkdb_locator(struct blkdb *db, struct blkinfo *bi,
+extern void chaindb_free(struct chaindb *db);
+extern bool chaindb_add(struct chaindb *db, struct blkinfo *bi,
+		      struct chaindb_reorg *reorg_info);
+extern void chaindb_locator(struct chaindb *db, struct blkinfo *bi,
 		   struct bp_locator *locator);
 
-static inline struct blkinfo *blkdb_lookup(struct blkdb *db,const bu256_t *hash)
+static inline struct blkinfo *chaindb_lookup(struct chaindb *db,const bu256_t *hash)
 {
 	return (struct blkinfo *)bp_hashtab_get(db->blocks, hash);
 }
@@ -69,4 +63,4 @@ static inline struct blkinfo *blkdb_lookup(struct blkdb *db,const bu256_t *hash)
 }
 #endif
 
-#endif /* __LIBCCOIN_BLKDB_H__ */
+#endif /* __LIBCCOIN_CHAINDB_H__ */

include/ccoin/db/db.h

@@ -0,0 +1,67 @@
+#ifndef __LIBCCOIN_DB_H__
+#define __LIBCCOIN_DB_H__
+/* Copyright 2012 exMULTI, Inc.
+ * Distributed under the MIT/X11 software license, see the accompanying
+ * file COPYING or http://www.opensource.org/licenses/mit-license.php.
+ */
+
+#include <ccoin/buffer.h>               // for const_buffer
+#include <ccoin/buint.h>                // for bu256_t
+
+#include <lmdb.h>                       // for MDB_dbi, MDB_env
+
+#include <stdbool.h>                    // for bool
+#include <stddef.h>                     // for size_t
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+	NETMAGIC_LEN = 4
+};
+
+ enum {
+ 	MAX_DB_SIZE = 17179869184	// Maximum database size in bytes
+ };
+
+enum db_list {
+	METADB,
+	BLOCKDB,
+	BLOCKHEIGHTDB,
+	MAX_NUM_DBS,
+};
+
+enum metadb_key {
+	NETMAGIC_KEY,
+	GENESIS_KEY,
+};
+
+struct db_handle {
+	const char	*name;
+	MDB_dbi		dbi;
+	bool		open;
+};
+
+struct db_info {
+	MDB_env				*env;
+	struct db_handle	handle[MAX_NUM_DBS];
+};
+
+extern bool metadb_init(const unsigned char *netmagic,
+		       const bu256_t *genesis_block);
+
+extern bool blockdb_init(void);
+extern bool blockdb_add(bu256_t *hash, struct const_buffer *buf);
+
+extern bool blockheightdb_init(void);
+extern bool blockheightdb_add(int height, bu256_t *hash);
+extern bool blockheightdb_getall(bool (*read_block)(void *p, size_t len));
+
+extern void db_close(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __LIBCCOIN_DB_H__ */

include/ccoin/net/net.h

@@ -43,7 +43,7 @@ struct net_child_info {
 	int			write_fd;
 
 	struct peer_manager	*peers;
-	struct blkdb		*db;
+	struct chaindb		*db;
 
 	parr			*conns;
 	struct event_base	*eb;
@@ -57,7 +57,6 @@ struct net_child_info {
 
 	bool (*inv_block_process)(bu256_t *hash);
 	bool (*block_process)(struct bp_block *block,
-                          struct p2p_message_hdr *hdr,
                           struct const_buffer *buf);
 };
 

lib/Makefile.am

@@ -22,7 +22,6 @@ libccoin_la_SOURCES=	\
 	addr_match.c	\
 	base58.c	\
 	bignum.c	\
-	blkdb.c		\
 	block.c		\
 	blockfile.c	\
 	bloom.c		\
@@ -54,7 +53,11 @@ libccoin_la_SOURCES=	\
 	utxo.c		\
 	wallet.c
 
-noinst_LTLIBRARIES= libccoinnet.la libccoinaes.la
+noinst_LTLIBRARIES= libccoindb.la libccoinnet.la libccoinaes.la
+
+libccoindb_la_SOURCES=	\
+	db/chaindb.c  \
+	db/db.c
 
 libccoinnet_la_SOURCES=	\
 	net/dns.c	\