diff --git a/.travis.yml b/.travis.yml index 967754de4b..c327de65f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,20 @@ language: rust +cache: cargo rust: - - stable + - stable +env: + global: + # XXX: begin_test_transaction doesn't play nice over threaded tests + - RUST_TEST_THREADS=1 + - SYNC_DATABASE_URL="mysql://travis@127.0.0.1/syncstorage" -cache: cargo +services: + - mysql before_script: - rustup component add rustfmt-preview + - mysql -e 'CREATE DATABASE IF NOT EXISTS syncstorage;' script: - cargo fmt -- --check diff --git a/Cargo.lock b/Cargo.lock index 0e06708163..1c41870358 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,7 +50,7 @@ dependencies = [ "httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazycell 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -61,7 +61,7 @@ dependencies = [ "parking_lot 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.70 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -103,12 +103,17 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "antidote" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "arrayvec" version = "0.4.7" @@ -117,6 +122,16 @@ dependencies = [ "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "atty" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "backtrace" version = "0.2.3" @@ -258,7 +273,7 @@ name = "config" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "nom 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "rust-ini 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.70 (registry+https://github.com/rust-lang/crates.io-index)", @@ -347,7 +362,7 @@ dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -361,7 +376,7 @@ dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -374,7 +389,7 @@ dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -416,7 +431,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "diesel_derives 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libsqlite3-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mysqlclient-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "r2d2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -429,13 +446,31 @@ dependencies = [ "syn 0.13.11 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "diesel_logger" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "diesel 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "diesel_migrations" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "migrations_internals 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "migrations_macros 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "docopt" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.70 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.70 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -511,6 +546,18 @@ dependencies = [ "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "env_logger" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "error-chain" version = "0.1.12" @@ -652,6 +699,14 @@ name = "httparse" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "humantime" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hyper" version = "0.11.27" @@ -754,8 +809,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lazy_static" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "lazycell" @@ -782,15 +840,6 @@ dependencies = [ "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "libsqlite3-sys" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "pkg-config 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", - "vcpkg 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "linked-hash-map" version = "0.3.0" @@ -853,7 +902,7 @@ dependencies = [ [[package]] name = "memchr" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", @@ -864,6 +913,24 @@ name = "memoffset" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "migrations_internals" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "diesel 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "migrations_macros" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "migrations_internals 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mime" version = "0.3.8" @@ -937,10 +1004,19 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.8.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "mysqlclient-sys" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pkg-config 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "native-tls" version = "0.1.5" @@ -975,7 +1051,7 @@ name = "nom" version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1014,7 +1090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.33 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1154,6 +1230,16 @@ dependencies = [ "proc-macro2 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "r2d2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "scheduled-thread-pool 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rand" version = "0.3.22" @@ -1205,7 +1291,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1215,6 +1301,14 @@ name = "redox_syscall" version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "regex" version = "0.1.80" @@ -1229,14 +1323,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1333,10 +1427,18 @@ name = "schannel" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "scoped-tls" version = "0.1.2" @@ -1527,8 +1629,13 @@ dependencies = [ "chrono 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "config 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "diesel 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "diesel_logger 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "diesel_migrations 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "docopt 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "mozsvc-common 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1569,6 +1676,24 @@ dependencies = [ "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "termcolor" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termion" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "thread-id" version = "2.0.0" @@ -1588,11 +1713,10 @@ dependencies = [ [[package]] name = "thread_local" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1808,7 +1932,7 @@ dependencies = [ "error-chain 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)", "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1831,7 +1955,7 @@ dependencies = [ "error-chain 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)", "ipconfig 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "resolv-conf 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1920,7 +2044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "utf8-ranges" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1987,11 +2111,28 @@ name = "winapi-i686-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "winapi-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "wincolor" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winreg" version = "0.5.1" @@ -2031,8 +2172,10 @@ dependencies = [ "checksum actix_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c4b1dc922654b9aca7a8a31eab875fde804fa9fbd67f220f2e457787b23590f2" "checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c" "checksum aho-corasick 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ca972c2ea5f742bfce5687b9aef75506a764f61d37f8f649047846a9686ddb66" -"checksum aho-corasick 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c1c6d463cbe7ed28720b5b489e7c083eeb8f90d08be2a0d6bb9e1ffea9ce1afa" +"checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a" +"checksum antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f" "checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" "checksum backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)" = "bff67d0c06556c0b8e6b5f090f0eac52d950d9dfd1d35ba04e4ca3543eaf6a7e" @@ -2068,6 +2211,8 @@ dependencies = [ "checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850" "checksum diesel 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e71e7a348ae6064e86c4cf0709f0e4c3ef6f30e8e7d3dc05737164af4ebd3511" "checksum diesel_derives 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "03bcaf77491f53e400d5ee3bdd57142ea4e1c47fe9217b3361ff9a76ca0e3d37" +"checksum diesel_logger 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d73a791d0f0fff4bc6244bd8b70776a45b1140716411efe1364be9a868cbea7" +"checksum diesel_migrations 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b42c35d1ce9e8d57a3e7001b4127f2bc1b073a89708bb7019f5be27c991c28" "checksum docopt 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e67fb750c36fc6fffbd3575cf8f2b46790fc0b05096ae3c03a36cf71b55e1e2b" "checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd" "checksum encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" @@ -2078,6 +2223,7 @@ dependencies = [ "checksum encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" "checksum encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" "checksum encoding_rs 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98fd0f24d1fb71a4a6b9330c8ca04cbd4e7cc5d846b54ca74ff376bc7c9f798d" +"checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" "checksum error-chain 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faa976b4fd2e4c2b2f3f486874b19e61944d3de3de8b61c9fcf835d583871bcc" "checksum error-chain 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6930e04918388a9a2e41d518c25cf679ccafe26733fb4127dbf21993f2575d46" "checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82" @@ -2096,6 +2242,7 @@ dependencies = [ "checksum htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" "checksum http 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "4fbced8864b04c030eebcb7d0dc3a81ba5231ac559f5116a29a8ba83ecee22cd" "checksum httparse 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7b6288d7db100340ca12873fd4d08ad1b8f206a9457798dfb17c018a33fee540" +"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" "checksum hyper 0.11.27 (registry+https://github.com/rust-lang/crates.io-index)" = "34a590ca09d341e94cddf8e5af0bbccde205d5fbc2fa3c09dd67c7f85cea59d7" "checksum hyper-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a5aa51f6ae9842239b0fac14af5f22123b8432b4cc774a44ff059fcba0f675ca" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" @@ -2106,12 +2253,11 @@ dependencies = [ "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" -"checksum lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fb497c35d362b6a331cfd94956a07fc2c78a4604cdbee844a81170386b996dd3" +"checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7" "checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef" "checksum lazycell 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d33a48d0365c96081958cc663eef834975cb1e8d8bea3378513fc72bdbf11e50" "checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1" "checksum libflate 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "7d4b4c7aff5bac19b956f693d0ea0eade8066deb092186ae954fa6ba14daab98" -"checksum libsqlite3-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0e9eb7b8e152b6a01be6a4a2917248381875758250dc3df5d46caf9250341dda" "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum linked-hash-map 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7860ec297f7008ff7a1e3382d7f7e1dcd69efc94751a2284bafc3d013c2aa939" "checksum lock_api 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "949826a5ccf18c1b3a7c3d57692778d21768b79e46eb9dd07bfc4c2160036c54" @@ -2120,8 +2266,10 @@ dependencies = [ "checksum lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4d06ff7ff06f729ce5f4e227876cb88d10bc59cd4ae1e09fbb2bde15c850dc21" "checksum matches 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "835511bab37c34c47da5cb44844bea2cfde0236db0b506f90ea4224482c9774a" "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" -"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" +"checksum memchr 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a3b4142ab8738a78c51896f704f83c11df047ff1bda9a92a661aa6361552d93d" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" +"checksum migrations_internals 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8cf7c8c4f83fa9f47440c0b4af99973502de55e6e7b875f693bd263e03f93e7e" +"checksum migrations_macros 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "79f12499ef7353bdeca2d081bc61edd8351dac09a33af845952009b5a3d68c1a" "checksum mime 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "fe51c8699d2dc522bf8c1ebe26ea2193d151fb54bcdfd7d0318750c189994cd9" "checksum mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "30de2e4613efcba1ec63d8133f344076952090c122992a903359be5a4f99c3ed" "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" @@ -2129,6 +2277,7 @@ dependencies = [ "checksum mio-uds 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "84c7b5caa3a118a6e34dbac36504503b1e8dc5835e833306b9d6af0e05929f79" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum mozsvc-common 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "732386d576b01d7e011eedcf3d6373322ee4699bf5c46585ef416959a7f567fa" +"checksum mysqlclient-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "879ce08e38739c54d87b7f8332a476004fe2a095f40a142a36f889779d9942b7" "checksum native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f74dbadc8b43df7864539cedb7bc91345e532fdd913cfdc23ad94f4d2d40fbc0" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" @@ -2155,6 +2304,7 @@ dependencies = [ "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" "checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035" +"checksum r2d2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f9078ca6a8a5568ed142083bb2f7dc9295b69d16f867ddcc9849e51b17d8db46" "checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" "checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" "checksum rand 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "12397506224b2f93e6664ffc4f664b29be8208e5157d3d90b44f09b5fae470ea" @@ -2162,8 +2312,9 @@ dependencies = [ "checksum rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b614fe08b6665cb9a231d07ac1364b0ef3cb3698f1239ee0c4c3a88a524f54c8" "checksum rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" +"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f" -"checksum regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5bbbea44c5490a1e84357ff28b7d518b4619a159fed5d25f6c1de2d19cc42814" +"checksum regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2069749032ea3ec200ca51e4a31df41759190a88edca0d2d86ee8bedf7073341" "checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957" "checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d" "checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a" @@ -2175,6 +2326,7 @@ dependencies = [ "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum schannel 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "dc1fabf2a7b6483a141426e1afd09ad543520a77ac49bd03c286e7696ccfd77f" +"checksum scheduled-thread-pool 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a2ff3fc5223829be817806c6441279c676e454cc7da608faf03b0ccc09d3889" "checksum scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfa44ee9c54ce5eecc9de7d5acbad112ee58755239381f687e564004ba4a2332" @@ -2203,9 +2355,11 @@ dependencies = [ "checksum synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a761d12e6d8dcb4dcf952a7a89b475e3a9d69e4a69307e01a470977642914bd" "checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +"checksum termcolor 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ff3bac0e465b59f194e7037ed404b0326e56ff234d767edc4c5cc9cd49e7a2c7" +"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9539db560102d1cef46b8b78ce737ff0bb64e7e18d35b2a5688f7d097d0ff03" "checksum thread_local 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8576dbbfcaef9641452d5cf0df9b0e7eeab7694956dd33bb61515fb8f18cfdd5" -"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" +"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" "checksum tokio 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8ee337e5f4e501fc32966fec6fe0ca0cc1c237b0b1b14a335f8bfe3c5f06e286" "checksum tokio-codec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "881e9645b81c2ce95fcb799ded2c29ffb9f25ef5bef909089a420e5961dd8ccb" @@ -2237,7 +2391,7 @@ dependencies = [ "checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae" "checksum url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2a321979c09843d272956e73700d12c4e7d3d92b2ee112b31548aef0d4efc5a6" "checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f" -"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" +"checksum utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd70f467df6810094968e2fce0ee1bd0e87157aceb026a8c083bcf5e25b9efe4" "checksum uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e1436e58182935dcd9ce0add9ea0b558e8a87befe01c1a301e6020aeb0876363" "checksum vcpkg 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cbe533e138811704c0e3cbde65a818b35d3240409b4346256c5ede403e082474" "checksum version_check 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7716c242968ee87e5542f8021178248f267f295a5c4803beae8b8b7fd9bc6051" @@ -2248,7 +2402,9 @@ dependencies = [ "checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" "checksum winreg 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a27a759395c1195c4cc5cda607ef6f8f6498f64e78f7900f5de0a127a424704a" "checksum winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" diff --git a/Cargo.toml b/Cargo.toml index daaf2204af..31ed1322fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,10 +13,15 @@ actix-web = "0.7.1" cadence = "0.15.1" chrono = "0.4" config = "0.9.0" -diesel = { version = "1.3", features = ["sqlite"] } +diesel = { version = "1.3", features = ["mysql", "r2d2"] } +diesel_logger = "0.1.0" +diesel_migrations = { version = "1.3", features = ["mysql"] } docopt = "1.0.0" +env_logger = "0.5.13" +failure = "0.1" futures = "0.1" # hawk = "1.0.2" +lazy_static = "1.1.0" mozsvc-common = "0.1.0" num_cpus = "1.8.0" rand = "0.5.2" diff --git a/migrations/2018-08-28-010336_init/down.sql b/migrations/2018-08-28-010336_init/down.sql new file mode 100644 index 0000000000..591b22ed48 --- /dev/null +++ b/migrations/2018-08-28-010336_init/down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS `bso`; +DROP TABLE IF EXISTS `collections`; +DROP TABLE IF EXISTS `user_collections`; +DROP TABLE IF EXISTS `batches`; diff --git a/migrations/2018-08-28-010336_init/up.sql b/migrations/2018-08-28-010336_init/up.sql new file mode 100644 index 0000000000..ed7087b76d --- /dev/null +++ b/migrations/2018-08-28-010336_init/up.sql @@ -0,0 +1,71 @@ +CREATE DATABASE IF NOT EXISTS `syncstorage` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `syncstorage`; + +-- DROP TABLE IF EXISTS `bso`; +CREATE TABLE `bso` ( + `user_id` INT NOT NULL, + `collection_id` INT NOT NULL, + `id` VARCHAR(64) NOT NULL, + + `sortindex` INT DEFAULT NULL, + + `payload` MEDIUMTEXT NOT NULL, + `payload_size` INT DEFAULT '0' NOT NULL, + + -- last modified time in milliseconds since epoch + `modified` BIGINT NOT NULL, + -- expiration in seconds milliseconds since epoch + `expiry` BIGINT DEFAULT '3153600000000' NOT NULL, + + PRIMARY KEY (`user_id`, `collection_id`, `id`), + KEY `bso_expiry_idx` (`expiry`), + KEY `bso_usr_col_mod_idx` (`user_id`, `collection_id`, `modified`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + + +-- DROP TABLE IF EXISTS `collections`; +CREATE TABLE `collections` ( + `id` INT PRIMARY KEY NOT NULL AUTO_INCREMENT, + `name` VARCHAR(32) UNIQUE NOT NULL +) ENGINE=InnoDB AUTO_INCREMENT=100 DEFAULT CHARSET=latin1; +INSERT INTO collections (id, name) VALUES + ( 1, "clients"), + ( 2, "crypto"), + ( 3, "forms"), + ( 4, "history"), + ( 5, "keys"), + ( 6, "meta"), + ( 7, "bookmarks"), + ( 8, "prefs"), + ( 9, "tabs"), + (10, "passwords"), + (11, "addons"), + (12, "addresses"), + (13, "creditcards"); + + +-- DROP TABLE IF EXISTS `user_collections`; +CREATE TABLE `user_collections` ( + `user_id` INT NOT NULL, + `collection_id` INT NOT NULL, + -- last modified time in milliseconds since epoch + `modified` BIGINT NOT NULL, + PRIMARY KEY (`user_id`, `collection_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + + +-- XXX: based on the go version (bsos is a concatenated blob of BSO jsons separated by newlines) +-- DROP TABLE IF EXISTS `batches`; +CREATE TABLE `batches` ( + `user_id` INT NOT NULL, + `collection_id` INT NOT NULL, + `id` varchar(64) NOT NULL, + + `modified` BIGINT NOT NULL, + + -- XXX: possibly largetext? + `bsos` MEDIUMTEXT NOT NULL, + + PRIMARY KEY (`user_id`, `collection_id`, `id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/src/db/error.rs b/src/db/error.rs new file mode 100644 index 0000000000..afa6eee9d6 --- /dev/null +++ b/src/db/error.rs @@ -0,0 +1,74 @@ +use std::fmt; + +use diesel; +use diesel_migrations; +use failure::{Backtrace, Context, Fail}; + +#[derive(Debug)] +pub struct DbError { + inner: Context, +} + +#[derive(Debug, Fail)] +pub enum DbErrorKind { + #[fail(display = "A database error occurred")] + Query(#[cause] diesel::result::Error), + + #[fail(display = "A database pool error occurred")] + Pool(diesel::r2d2::PoolError), + + #[fail(display = "Error migrating the database")] + Migration(diesel_migrations::RunMigrationsError), +} + +impl DbError { + pub fn kind(&self) -> &DbErrorKind { + self.inner.get_context() + } +} + +impl Fail for DbError { + fn cause(&self) -> Option<&Fail> { + self.inner.cause() + } + + fn backtrace(&self) -> Option<&Backtrace> { + self.inner.backtrace() + } +} + +impl fmt::Display for DbError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.inner, f) + } +} + +impl From for DbError { + fn from(kind: DbErrorKind) -> DbError { + Context::new(kind).into() + } +} + +impl From> for DbError { + fn from(inner: Context) -> DbError { + DbError { inner } + } +} + +impl From for DbError { + fn from(inner: diesel::result::Error) -> DbError { + DbErrorKind::Query(inner).into() + } +} + +impl From for DbError { + fn from(inner: diesel::r2d2::PoolError) -> DbError { + DbErrorKind::Pool(inner).into() + } +} + +impl From for DbError { + fn from(inner: diesel_migrations::RunMigrationsError) -> DbError { + DbErrorKind::Migration(inner).into() + } +} diff --git a/src/db/mock.rs b/src/db/mock.rs index c9f5e2d851..af61ee4cd6 100644 --- a/src/db/mock.rs +++ b/src/db/mock.rs @@ -11,19 +11,22 @@ use super::*; #[derive(Debug)] pub struct MockDb; +impl MockDb { + pub fn new() -> Self { + MockDb + } +} + macro_rules! mock_db_method { ($name:ident, $type:ident) => { - fn $name(&self, _params: params::$type) -> DbFuture { + fn $name(&self, _params: ¶ms::$type) -> DbFuture { Box::new(future::result(Ok(results::$type::default()))) } } } impl Db for MockDb { - fn new() -> Box { - Box::new(MockDb) - } - + mock_db_method!(get_collection_id, GetCollectionId); mock_db_method!(get_collections, GetCollections); mock_db_method!(get_collection_counts, GetCollectionCounts); mock_db_method!(get_collection_usage, GetCollectionUsage); diff --git a/src/db/mod.rs b/src/db/mod.rs index 29cb23e8a7..b66ad3ac0e 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -4,64 +4,100 @@ //! Generic db abstration. +pub mod error; +#[macro_use] pub mod mock; +pub mod mysql; pub mod params; pub mod results; +pub mod util; -use std::{ - error::Error, - fmt::{self, Display, Formatter}, -}; +use std::{collections::HashMap, ops::Deref}; use futures::future::Future; +pub use self::error::DbError; + +lazy_static! { + static ref STD_COLLS: HashMap = { + let mut m = HashMap::new(); + m.insert(1, "clients"); + m.insert(2, "crypto"); + m.insert(3, "forms"); + m.insert(4, "history"); + m.insert(5, "keys"); + m.insert(6, "meta"); + m.insert(7, "bookmarks"); + m.insert(8, "prefs"); + m.insert(9, "tabs"); + m.insert(10, "passwords"); + m.insert(11, "addons"); + m.insert(12, "addresses"); + m.insert(13, "creditcards"); + m + }; + static ref STD_COLLS_IDS: HashMap<&'static str, i32> = + STD_COLLS.iter().map(|(k, v)| (*v, *k)).collect(); +} + +fn get_std_collection_name(id: i32) -> Option<&'static str> { + STD_COLLS.get(&id).map(Deref::deref) +} + +fn get_std_collection_id(name: &str) -> Option { + STD_COLLS_IDS.get(&name).map(|i| *i) +} + type DbFuture = Box>; +// XXX: add a DbPool trait + pub trait Db: Send { - fn new() -> Box - where - Self: Sized; + // XXX: add a generic fn transaction(&self, f) + + fn get_collection_id( + &self, + params: ¶ms::GetCollectionId, + ) -> DbFuture; - fn get_collections(&self, params: params::GetCollections) -> DbFuture; + fn get_collections(&self, params: ¶ms::GetCollections) + -> DbFuture; fn get_collection_counts( &self, - params: params::GetCollectionCounts, + params: ¶ms::GetCollectionCounts, ) -> DbFuture; fn get_collection_usage( &self, - params: params::GetCollectionUsage, + params: ¶ms::GetCollectionUsage, ) -> DbFuture; - fn get_quota(&self, params: params::GetQuota) -> DbFuture; + fn get_quota(&self, params: ¶ms::GetQuota) -> DbFuture; - fn delete_all(&self, params: params::DeleteAll) -> DbFuture; + fn delete_all(&self, params: ¶ms::DeleteAll) -> DbFuture; fn delete_collection( &self, - params: params::DeleteCollection, + params: ¶ms::DeleteCollection, ) -> DbFuture; - fn get_collection(&self, params: params::GetCollection) -> DbFuture; + fn get_collection(&self, params: ¶ms::GetCollection) -> DbFuture; - fn post_collection(&self, params: params::PostCollection) -> DbFuture; + fn post_collection(&self, params: ¶ms::PostCollection) + -> DbFuture; - fn delete_bso(&self, params: params::DeleteBso) -> DbFuture; + fn delete_bso(&self, params: ¶ms::DeleteBso) -> DbFuture; - fn get_bso(&self, params: params::GetBso) -> DbFuture; + fn get_bso(&self, params: ¶ms::GetBso) -> DbFuture; - fn put_bso(&self, params: params::PutBso) -> DbFuture; + fn put_bso(&self, params: ¶ms::PutBso) -> DbFuture; } -// HACK: temporary placeholder until we have proper error-handling #[derive(Debug)] -pub struct DbError(String); - -impl Display for DbError { - fn fmt(&self, formatter: &mut Formatter) -> fmt::Result { - write!(formatter, "db error: {}", &self.0) - } +pub enum Sorting { + None, + Newest, + Oldest, + Index, } - -impl Error for DbError {} diff --git a/src/db/mysql/mod.rs b/src/db/mysql/mod.rs new file mode 100644 index 0000000000..ed0abd470c --- /dev/null +++ b/src/db/mysql/mod.rs @@ -0,0 +1,4 @@ +pub mod models; +mod schema; +#[cfg(test)] +mod test; diff --git a/src/db/mysql/models.rs b/src/db/mysql/models.rs new file mode 100644 index 0000000000..c14fd5f80e --- /dev/null +++ b/src/db/mysql/models.rs @@ -0,0 +1,352 @@ +use std::{self, ops::Deref}; + +use diesel::{ + insert_into, + mysql::MysqlConnection, + r2d2::{ConnectionManager, Pool, PooledConnection}, + sql_types::{BigInt, Integer, Text}, + update, Connection, ExpressionMethods, OptionalExtension, +}; +use diesel::{sql_query, QueryDsl, RunQueryDsl}; +#[cfg(test)] +use diesel_logger::LoggingConnection; +use futures::future; + +use super::schema::{bso, collections, user_collections}; +#[cfg(test)] +use super::test::TestTransactionCustomizer; +use db::{ + error::DbError, get_std_collection_id, get_std_collection_name, params, results, + util::ms_since_epoch, Db, DbFuture, Sorting, +}; +use settings::Settings; + +embed_migrations!(); + +pub type Result = std::result::Result; + +// The ttl to use for rows that are never supposed to expire (in seconds) +pub const DEFAULT_BSO_TTL: u32 = 2100000000; + +no_arg_sql_function!(last_insert_id, Integer); + +/// Run the diesel embedded migrations +/// +/// Mysql DDL statements implicitly commit which could disrupt MysqlPool's +/// begin_test_transaction during tests. So this runs on its own separate conn. +pub fn run_embedded_migrations(settings: &Settings) -> Result<()> { + let conn = MysqlConnection::establish(&settings.database_url).unwrap(); + Ok(embedded_migrations::run(&conn)?) +} + +pub struct MysqlDbPool { + pool: Pool>, +} + +// XXX: to become a db::DbPool trait +impl MysqlDbPool { + pub fn new(settings: &Settings) -> Result { + let manager = ConnectionManager::::new(settings.database_url.as_ref()); + let mut builder = Pool::builder().max_size(settings.database_pool_max_size.unwrap_or(10)); + #[cfg(test)] + { + if settings.database_use_test_transactions { + builder = builder.connection_customizer(Box::new(TestTransactionCustomizer)); + } + } + Ok(Self { + pool: builder.build(manager)?, + }) + } + + pub fn get(&self) -> Result { + Ok(MysqlDb { + #[cfg(not(test))] + conn: self.pool.get()?, + #[cfg(test)] + conn: LoggingConnection::new(self.pool.get()?), + }) + } +} + +pub struct MysqlDb { + #[cfg(not(test))] + pub(super) conn: PooledConnection>, + #[cfg(test)] + pub(super) conn: LoggingConnection>>, +} + +impl MysqlDb { + pub fn get_collection_id_sync( + &self, + params: ¶ms::GetCollectionId, + ) -> Result { + let id = if let Some(id) = get_std_collection_id(params) { + id + } else { + sql_query("SELECT id FROM collections WHERE name = ?") + .bind::(params) + .get_result::(&self.conn)? + .id + }; + Ok(id) + } + + pub fn get_collections_sync( + &self, + params: ¶ms::GetCollections, + ) -> Result { + sql_query("SELECT collection_id, modified FROM user_collections WHERE user_id = ?") + .bind::(params.user_id as i32) + .load::(&self.conn)? + .into_iter() + .map(|cr| { + self.get_collection_name(cr.id) + .map(|name| (name, cr.modified)) + }) + .collect() + } + + pub fn create_collection_sync(&self, name: &str) -> Result { + // XXX: handle concurrent attempts at inserts + let collection_id = self.conn.transaction(|| { + sql_query("INSERT INTO collections (name) VALUES (?)") + .bind::(name) + .execute(&self.conn)?; + collections::table.select(last_insert_id).first(&self.conn) + })?; + Ok(collection_id) + } + + fn get_collection_name(&self, id: i32) -> Result { + // XXX: python caches collection names/ids in memory as they're added + let name = if let Some(name) = get_std_collection_name(id) { + name.to_owned() + } else { + sql_query("SELECT name FROM collections where id = ?") + .bind::(&id) + .get_result::(&self.conn)? + .name + }; + Ok(name) + } + + pub fn put_bso_sync(&self, bso: ¶ms::PutBso) -> Result { + /* + if bso.payload.is_none() && bso.sortindex.is_none() && bso.ttl.is_none() { + // XXX: go returns an error here (ErrNothingToDo), and is treated + // as other errors + return Ok(()); + } + */ + + // XXX: consider mysql ON DUPLICATE KEY UPDATE? + self.conn.transaction(|| { + let q = r#" + SELECT 1 as count FROM bso + WHERE user_id = ? AND collection_id = ? AND id = ? + "#; + let exists = sql_query(q) + .bind::(bso.user_id as i32) // XXX: + .bind::(&bso.collection_id) + .bind::(&bso.id) + .get_result::(&self.conn) + .optional()? + .is_some(); + + if exists { + update(bso::table) + .filter(bso::user_id.eq(bso.user_id as i32)) // XXX: + .filter(bso::collection_id.eq(&bso.collection_id)) + .filter(bso::id.eq(&bso.id)) + .set(put_bso_as_changeset(&bso)) + .execute(&self.conn)?; + } else { + let payload = bso.payload.as_ref().map(Deref::deref).unwrap_or_default(); + let sortindex = bso.sortindex; + let ttl = bso.ttl.map_or(DEFAULT_BSO_TTL, |ttl| ttl); + insert_into(bso::table) + .values(( + bso::user_id.eq(bso.user_id as i32), // XXX: + bso::collection_id.eq(&bso.collection_id), + bso::id.eq(&bso.id), + bso::sortindex.eq(sortindex), + bso::payload.eq(payload), + bso::payload_size.eq(payload.len() as i32), // XXX: + bso::modified.eq(bso.modified), + bso::expiry.eq(bso.modified + ttl as i64), + )) + .execute(&self.conn)?; + } + self.touch_collection(bso.user_id as i32, bso.collection_id, bso.modified)?; + // XXX: + Ok(bso.modified as u64) + }) + } + + // XXX: limit/offset i64? + pub fn get_bsos_sync( + &self, + user_id: u32, + collection_id: i32, + mut ids: &[&str], + older: u64, + newer: u64, + sort: Sorting, + limit: i64, + offset: i64, + ) -> Result { + // XXX: ensure offset/limit/newer are valid + + // XXX: should error out (400 Bad Request) when more than 100 + // are provided (move to validation layer) + if ids.len() > 100 { + // spec says only 100 ids at a time + ids = &ids[0..100]; + } + + // XXX: convert to raw SQL for use by other backends + let mut query = bso::table + //.select(bso::table::all_columns()) + .select((bso::id, bso::modified, bso::payload, bso::sortindex, bso::expiry)) + .filter(bso::user_id.eq(user_id as i32)) // XXX: + .filter(bso::collection_id.eq(collection_id as i32)) // XXX: + .filter(bso::modified.lt(older as i64)) + .filter(bso::modified.gt(newer as i64)) + .filter(bso::expiry.gt(ms_since_epoch())) + .into_boxed(); + + if !ids.is_empty() { + query = query.filter(bso::id.eq_any(ids)); + } + + query = match sort { + Sorting::Index => query.order(bso::sortindex.desc()), + Sorting::Newest => query.order(bso::modified.desc()), + Sorting::Oldest => query.order(bso::modified.asc()), + _ => query, + }; + + // fetch an extra row to detect if there are more rows that + // match the query conditions + query = query.limit(if limit >= 0 { limit + 1 } else { limit }); + if offset != 0 { + // XXX: copy over this optimization: + // https://github.com/mozilla-services/server-syncstorage/blob/a0f8117/syncstorage/storage/sql/__init__.py#L404 + query = query.offset(offset); + } + let mut bsos = query.load::(&self.conn)?; + + let (more, next_offset) = if limit >= 0 && bsos.len() > limit as usize { + bsos.pop(); + (true, limit + offset) + } else { + (false, 0) + }; + + Ok(results::BSOs { + bsos, + more, + offset: next_offset, + }) + } + + pub fn get_bso_sync(&self, params: ¶ms::GetBso) -> Result> { + Ok(sql_query(r#" + SELECT id, modified, payload, sortindex, expiry FROM bso + WHERE user_id = ? AND collection_id = ? AND id = ? AND expiry >= ? + "#) + .bind::(params.user_id as i32) // XXX: + .bind::(¶ms.collection_id) + .bind::(¶ms.id) + .bind::(ms_since_epoch() as i32) // XXX: + .get_result::(&self.conn) + .optional()?) + } + + fn touch_collection(&self, user_id: i32, collection_id: i32, modified: i64) -> Result<()> { + // XXX: ensure transaction + // The common case will be an UPDATE, so try that first + let affected_rows = update(user_collections::table) + .filter(user_collections::user_id.eq(&user_id)) + .filter(user_collections::collection_id.eq(&collection_id)) + .set(user_collections::modified.eq(&modified)) + .execute(&self.conn)?; + if affected_rows != 1 { + insert_into(user_collections::table) + .values(( + user_collections::user_id.eq(&user_id), + user_collections::collection_id.eq(&collection_id), + user_collections::modified.eq(&modified), + )) + .execute(&self.conn)?; + } + Ok(()) + } +} + +impl Db for MysqlDb { + mock_db_method!(get_collection_id, GetCollectionId); + mock_db_method!(get_collections, GetCollections); + mock_db_method!(get_collection_counts, GetCollectionCounts); + mock_db_method!(get_collection_usage, GetCollectionUsage); + mock_db_method!(get_quota, GetQuota); + mock_db_method!(delete_all, DeleteAll); + mock_db_method!(delete_collection, DeleteCollection); + mock_db_method!(get_collection, GetCollection); + mock_db_method!(post_collection, PostCollection); + mock_db_method!(delete_bso, DeleteBso); + mock_db_method!(get_bso, GetBso); + mock_db_method!(put_bso, PutBso); +} + +#[derive(Debug, QueryableByName)] +struct IdResult { + #[sql_type = "Integer"] + id: i32, +} + +#[derive(Debug, QueryableByName)] +struct NameResult { + #[sql_type = "Text"] + name: String, +} + +#[derive(Debug, QueryableByName)] +struct UserCollectionsResult { + #[sql_type = "Integer"] + id: i32, + #[sql_type = "BigInt"] + modified: i64, +} + +#[derive(Debug, QueryableByName)] +struct Count { + #[sql_type = "BigInt"] + count: i64, +} + +/// Formats a BSO for UPDATEs +#[derive(AsChangeset)] +#[table_name = "bso"] +struct UpdateBSO<'a> { + pub sortindex: Option, + pub payload: Option<&'a String>, + pub payload_size: Option, + pub modified: Option, + pub expiry: Option, +} + +fn put_bso_as_changeset<'a>(bso: &'a params::PutBso) -> UpdateBSO<'a> { + UpdateBSO { + sortindex: bso.sortindex, + expiry: bso.ttl.map(|ttl| bso.modified + ttl as i64), + payload: bso.payload.as_ref(), + payload_size: bso.payload.as_ref().map(|payload| payload.len() as i32), // XXX: + modified: if bso.payload.is_some() || bso.sortindex.is_some() { + Some(bso.modified) + } else { + None + }, + } +} diff --git a/src/db/mysql/schema.rs b/src/db/mysql/schema.rs new file mode 100644 index 0000000000..2e06083ab5 --- /dev/null +++ b/src/db/mysql/schema.rs @@ -0,0 +1,39 @@ +table! { + batches (user_id, collection_id, id) { + user_id -> Integer, + collection_id -> Integer, + id -> Varchar, + modified -> Bigint, + bsos -> Mediumtext, + } +} + +table! { + bso (user_id, collection_id, id) { + user_id -> Integer, + collection_id -> Integer, + id -> Varchar, + sortindex -> Nullable, + payload -> Mediumtext, + payload_size -> Integer, + modified -> Bigint, + expiry -> Bigint, + } +} + +table! { + collections (id) { + id -> Integer, + name -> Varchar, + } +} + +table! { + user_collections (user_id, collection_id) { + user_id -> Integer, + collection_id -> Integer, + modified -> Bigint, + } +} + +allow_tables_to_appear_in_same_query!(batches, bso, collections, user_collections,); diff --git a/src/db/mysql/test.rs b/src/db/mysql/test.rs new file mode 100644 index 0000000000..1cc022affc --- /dev/null +++ b/src/db/mysql/test.rs @@ -0,0 +1,421 @@ +use std::collections::HashMap; + +use db::mysql::{ + models::{run_embedded_migrations, MysqlDb, MysqlDbPool, DEFAULT_BSO_TTL}, + schema::collections, +}; +use db::{params, util::ms_since_epoch, Sorting}; +use env_logger; +use settings::Settings; + +use diesel::{ + mysql::MysqlConnection, + r2d2::{CustomizeConnection, Error as PoolError}, + Connection, +}; + +// distant future (year 2099) timestamp for tests +pub const MAX_TIMESTAMP: u64 = 4070937600000; + +#[derive(Debug)] +pub struct TestTransactionCustomizer; + +impl CustomizeConnection for TestTransactionCustomizer { + fn on_acquire(&self, conn: &mut MysqlConnection) -> Result<(), PoolError> { + conn.begin_test_transaction().map_err(PoolError::QueryError) + } +} + +pub fn db() -> MysqlDb { + let _ = env_logger::try_init(); + // inherit SYNC_DATABASE_URL from the env + let settings = Settings::with_env_and_config_file(&None).unwrap(); + let settings = Settings { + debug: true, + port: 8000, + database_url: settings.database_url, + database_pool_max_size: Some(1), + database_use_test_transactions: true, + }; + + run_embedded_migrations(&settings).unwrap(); + let pool = MysqlDbPool::new(&settings).unwrap(); + pool.get().unwrap() +} + +fn pbso( + user_id: u32, + cid: i32, + bid: &str, + payload: Option<&str>, + sortindex: Option, + ttl: Option, +) -> params::PutBso { + params::PutBso { + user_id, + collection_id: cid, + id: bid.to_owned(), + payload: payload.map(&str::to_owned), + sortindex, + ttl, + modified: ms_since_epoch(), + } +} + +fn gbso(user_id: u32, cid: i32, bid: &str) -> params::GetBso { + params::GetBso { + user_id, + collection_id: cid, + id: bid.to_owned(), + } +} + +#[test] +fn static_collection_id() { + let db = db(); + + // ensure DB actually has predefined common collections + let cols: Vec<(i32, _)> = vec![ + (1, "clients"), + (2, "crypto"), + (3, "forms"), + (4, "history"), + (5, "keys"), + (6, "meta"), + (7, "bookmarks"), + (8, "prefs"), + (9, "tabs"), + (10, "passwords"), + (11, "addons"), + (12, "addresses"), + (13, "creditcards"), + ]; + + use diesel::{QueryDsl, RunQueryDsl}; + let results: HashMap = collections::table + .select((collections::id, collections::name)) + .load(&db.conn) + .unwrap() + .into_iter() + .collect(); + assert_eq!(results.len(), cols.len()); + for (id, name) in &cols { + assert_eq!(results.get(id).unwrap(), name); + } + + for (id, name) in &cols { + let result = db.get_collection_id_sync(name).unwrap(); + assert_eq!(result, *id); + } + + let cid = db.create_collection_sync("col1").unwrap(); + assert!(cid >= 100); +} + +#[test] +fn bso_successfully_updates_single_values() { + let db = db(); + + let uid = 1; + let cid = 1; + let bid = "testBSO"; + let sortindex = 1; + let ttl = 3600 * 1000; + let bso1 = pbso( + uid, + cid, + bid, + Some("initial value"), + Some(sortindex), + Some(ttl), + ); + db.put_bso_sync(&bso1).unwrap(); + + let payload = "Updated payload"; + let bso2 = pbso(uid, cid, bid, Some(payload), None, None); + db.put_bso_sync(&bso2).unwrap(); + + let bso = db.get_bso_sync(&gbso(uid, cid, bid)).unwrap().unwrap(); + assert_eq!(bso.modified, bso2.modified); + assert_eq!(bso.payload, payload); + assert_eq!(bso.sortindex, Some(sortindex)); + // XXX: go version assumes ttl was updated here? + //assert_eq!(bso.expiry, modified + ttl); + assert_eq!(bso.expiry, bso1.modified + ttl as i64); + + let sortindex = 2; + let bso2 = pbso(uid, cid, bid, None, Some(sortindex), None); + db.put_bso_sync(&bso2).unwrap(); + let bso = db.get_bso_sync(&gbso(uid, cid, bid)).unwrap().unwrap(); + assert_eq!(bso.modified, bso2.modified); + assert_eq!(bso.payload, payload); + assert_eq!(bso.sortindex, Some(sortindex)); + // XXX: + //assert_eq!(bso.expiry, modified + ttl); + assert_eq!(bso.expiry, bso1.modified + ttl as i64); +} + +#[test] +fn bso_modified_not_changed_on_ttl_touch() { + let db = db(); + + let uid = 1; + let cid = 1; + let bid = "testBSO"; + + let mut bso1 = pbso(uid, cid, bid, Some("hello"), Some(1), Some(10)); + bso1.modified = ms_since_epoch() - 100; + db.put_bso_sync(&bso1).unwrap(); + + let bso2 = pbso(uid, cid, bid, None, None, Some(15)); + db.put_bso_sync(&bso2).unwrap(); + let bso = db.get_bso_sync(&gbso(uid, cid, bid)).unwrap().unwrap(); + // ttl has changed + assert_eq!(bso.expiry, bso2.modified + 15); + // modified has not changed + assert_eq!(bso.modified, bso1.modified); +} + +#[test] +fn put_bso_updates() { + let db = db(); + + let uid = 1; + let cid = 1; + let bid = "1"; + let bso1 = pbso(uid, cid, bid, Some("initial"), None, None); + db.put_bso_sync(&bso1).unwrap(); + + let bso2 = pbso(uid, cid, bid, Some("Updated"), Some(100), None); + db.put_bso_sync(&bso2).unwrap(); + + let bso = db.get_bso_sync(&gbso(uid, cid, bid)).unwrap().unwrap(); + assert_eq!(Some(bso.payload), bso2.payload); + assert_eq!(bso.sortindex, bso2.sortindex); + assert_eq!(bso.modified, bso2.modified); +} + +#[test] +fn get_bsos_limit_offset() { + let db = db(); + + let uid = 1; + let cid = 1; + let size = 12; + for i in 0..size { + let mut bso = pbso( + uid, + cid, + &i.to_string(), + Some(&format!("payload-{}", i)), + Some(i), + Some(DEFAULT_BSO_TTL), + ); + bso.modified += i as i64 * 10; + db.put_bso_sync(&bso).unwrap(); + } + + let bsos = db + .get_bsos_sync(uid, cid, &[], MAX_TIMESTAMP, 0, Sorting::Index, 0, 0) + .unwrap(); + assert!(bsos.bsos.is_empty()); + assert!(bsos.more); + assert_eq!(bsos.offset, 0); + + let bsos = db + .get_bsos_sync(uid, cid, &[], MAX_TIMESTAMP, 0, Sorting::Index, -1, 0) + .unwrap(); + assert_eq!(bsos.bsos.len(), size as usize); + assert!(!bsos.more); + assert_eq!(bsos.offset, 0); + + let newer = 0; + let limit = 5; + let offset = 0; + // XXX: validation? + /* + let bsos = db.get_bsos_sync(uid, cid, &[], MAX_TIMESTAMP, 0, Sorting::Index, -1, 0).unwrap(); + .. etc + */ + + let bsos = + db.get_bsos_sync( + uid, + cid, + &[], + MAX_TIMESTAMP, + newer, + Sorting::Newest, + limit, + offset, + ).unwrap(); + assert_eq!(bsos.bsos.len(), 5 as usize); + assert!(bsos.more); + assert_eq!(bsos.offset, 5); + assert_eq!(bsos.bsos[0].id, "11"); + assert_eq!(bsos.bsos[4].id, "7"); + + let bsos2 = + db.get_bsos_sync( + uid, + cid, + &[], + MAX_TIMESTAMP, + newer, + Sorting::Index, + limit, + bsos.offset, + ).unwrap(); + assert_eq!(bsos2.bsos.len(), 5 as usize); + assert!(bsos2.more); + assert_eq!(bsos2.offset, 10); + assert_eq!(bsos2.bsos[0].id, "6"); + assert_eq!(bsos2.bsos[4].id, "2"); + + let bsos3 = + db.get_bsos_sync( + uid, + cid, + &[], + MAX_TIMESTAMP, + newer, + Sorting::Index, + limit, + bsos2.offset, + ).unwrap(); + assert_eq!(bsos3.bsos.len(), 2 as usize); + assert!(!bsos3.more); + assert_eq!(bsos3.offset, 0); + assert_eq!(bsos3.bsos[0].id, "1"); + assert_eq!(bsos3.bsos[1].id, "0"); +} + +#[test] +fn get_bsos_newer() { + let db = db(); + + let uid = 1; + let cid = 1; + let modified = ms_since_epoch(); + // XXX: validation + //db.get_bsos_sync(uid, cid, &[], MAX_TIMESTAMP, -1, Sorting::None, 10, 0).is_err() + + for i in (0..=2).rev() { + let mut pbso = pbso( + uid, + cid, + &format!("b{}", i), + Some("a"), + Some(1), + Some(DEFAULT_BSO_TTL), + ); + pbso.modified = modified - i; + db.put_bso_sync(&pbso).unwrap(); + } + + let bsos = + db.get_bsos_sync( + uid, + cid, + &[], + MAX_TIMESTAMP, + modified as u64 - 3, + Sorting::Newest, + 10, + 0, + ).unwrap(); + assert_eq!(bsos.bsos.len(), 3); + assert_eq!(bsos.bsos[0].id, "b0"); + assert_eq!(bsos.bsos[1].id, "b1"); + assert_eq!(bsos.bsos[2].id, "b2"); + + let bsos = + db.get_bsos_sync( + uid, + cid, + &[], + MAX_TIMESTAMP, + modified as u64 - 2, + Sorting::Newest, + 10, + 0, + ).unwrap(); + assert_eq!(bsos.bsos.len(), 2); + assert_eq!(bsos.bsos[0].id, "b0"); + assert_eq!(bsos.bsos[1].id, "b1"); + + let bsos = + db.get_bsos_sync( + uid, + cid, + &[], + MAX_TIMESTAMP, + modified as u64 - 1, + Sorting::Newest, + 10, + 0, + ).unwrap(); + assert_eq!(bsos.bsos.len(), 1); + assert_eq!(bsos.bsos[0].id, "b0"); + + let bsos = + db.get_bsos_sync( + uid, + cid, + &[], + MAX_TIMESTAMP, + modified as u64, + Sorting::Newest, + 10, + 0, + ).unwrap(); + assert_eq!(bsos.bsos.len(), 0); +} + +#[test] +fn get_bsos_sort() { + let db = db(); + + let uid = 1; + let cid = 1; + let modified = ms_since_epoch(); + // XXX: validation again + //db.get_bsos_sync(uid, cid, &[], MAX_TIMESTAMP, -1, Sorting::None, 10, 0).is_err() + + for (revi, sortindex) in [1, 0, 2].iter().enumerate().rev() { + let mut pbso = pbso( + uid, + cid, + &format!("b{}", revi), + Some("a"), + Some(*sortindex), + Some(DEFAULT_BSO_TTL), + ); + pbso.modified = modified - revi as i64; + db.put_bso_sync(&pbso).unwrap(); + } + + let bsos = db + .get_bsos_sync(uid, cid, &[], MAX_TIMESTAMP, 0, Sorting::Newest, 10, 0) + .unwrap(); + assert_eq!(bsos.bsos.len(), 3); + assert_eq!(bsos.bsos[0].id, "b0"); + assert_eq!(bsos.bsos[1].id, "b1"); + assert_eq!(bsos.bsos[2].id, "b2"); + + let bsos = db + .get_bsos_sync(uid, cid, &[], MAX_TIMESTAMP, 0, Sorting::Oldest, 10, 0) + .unwrap(); + assert_eq!(bsos.bsos.len(), 3); + assert_eq!(bsos.bsos[0].id, "b2"); + assert_eq!(bsos.bsos[1].id, "b1"); + assert_eq!(bsos.bsos[2].id, "b0"); + + let bsos = db + .get_bsos_sync(uid, cid, &[], MAX_TIMESTAMP, 0, Sorting::Index, 10, 0) + .unwrap(); + assert_eq!(bsos.bsos.len(), 3); + assert_eq!(bsos.bsos[0].id, "b2"); + assert_eq!(bsos.bsos[1].id, "b0"); + assert_eq!(bsos.bsos[2].id, "b1"); +} diff --git a/src/db/params.rs b/src/db/params.rs index 652aa74369..4c6594741b 100644 --- a/src/db/params.rs +++ b/src/db/params.rs @@ -17,7 +17,7 @@ macro_rules! uid_data { ($($name:ident,)+) => ($( data! { $name { - user_id: String, + user_id: u32, } } )+) @@ -27,8 +27,8 @@ macro_rules! collection_data { ($($name:ident {$($property:ident: $type:ty,)*},)+) => ($( data! { $name { - user_id: String, - collection: String, + user_id: u32, + collection_id: i32, $($property: $type,)* } } @@ -39,9 +39,9 @@ macro_rules! bso_data { ($($name:ident {$($property:ident: $type:ty,)*},)+) => ($( data! { $name { - user_id: String, - collection: String, - bso_id: String, + user_id: u32, + collection_id: i32, + id: String, $($property: $type,)* } } @@ -56,6 +56,8 @@ uid_data! { DeleteAll, } +pub type GetCollectionId = str; + collection_data! { DeleteCollection { bso_ids: Vec, @@ -70,6 +72,7 @@ bso_data! { DeleteBso {}, GetBso {}, PutBso { + modified: i64, sortindex: Option, payload: Option, ttl: Option, @@ -78,7 +81,7 @@ bso_data! { #[derive(Debug)] pub struct PostCollectionBso { - pub bso_id: String, + pub id: String, pub sortindex: Option, pub payload: Option, pub ttl: Option, diff --git a/src/db/results.rs b/src/db/results.rs index 16da90ec17..1c86750a03 100644 --- a/src/db/results.rs +++ b/src/db/results.rs @@ -6,9 +6,10 @@ use std::collections::HashMap; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use diesel::sql_types::{BigInt, Integer, Nullable, Text}; -pub type GetCollections = HashMap; +pub type GetCollectionId = i32; +pub type GetCollections = HashMap; pub type GetCollectionCounts = HashMap; pub type GetCollectionUsage = HashMap; pub type GetQuota = Vec; @@ -25,13 +26,28 @@ pub enum GetCollectionItem { Bso(GetBso), } -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Debug, Default, Deserialize, Queryable, QueryableByName, Serialize)] pub struct GetBso { + #[sql_type = "Text"] pub id: String, - pub modified: u64, + #[sql_type = "BigInt"] + pub modified: i64, + #[sql_type = "Text"] pub payload: String, #[serde(skip_serializing_if = "Option::is_none")] + #[sql_type = "Nullable"] pub sortindex: Option, + #[serde(skip_serializing)] + #[serde(skip_deserializing)] + #[sql_type = "BigInt"] + pub expiry: i64, +} + +#[derive(Debug, Default)] +pub struct BSOs { + pub bsos: Vec, // XXX: naming + pub more: bool, + pub offset: i64, // XXX: i64? } #[derive(Debug, Serialize)] diff --git a/src/db/util.rs b/src/db/util.rs index f1fb3782e2..29cb424a2b 100644 --- a/src/db/util.rs +++ b/src/db/util.rs @@ -1,9 +1,6 @@ use chrono::offset::Utc; -use diesel::sql_types::Bigint; /// Get the time since the UNIX epoch in milliseconds pub fn ms_since_epoch() -> i64 { Utc::now().timestamp_millis() } - -no_arg_sql_function!(last_insert_rowid, Bigint); diff --git a/src/handlers.rs b/src/handlers.rs index e44ecec741..d47c6a8d93 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -13,7 +13,7 @@ use futures::future::{self, Future}; //use hawk:: use serde::de::{Deserialize, Deserializer}; -use db::{params, Db, DbError}; +use db::{params, util::ms_since_epoch, Db, DbError}; /// This is the global HTTP state object that will be made available to all /// HTTP API calls. @@ -68,7 +68,7 @@ macro_rules! db_endpoint { ($path, state$(, $param)*): (Path<$path_type>, State$(, $type)*), ) -> FutureResponse { Box::new( - state.db.$handler(params::$data { + state.db.$handler(¶ms::$data { $($property: $value,)* }) .map_err(From::from) @@ -82,7 +82,8 @@ macro_rules! info_endpoints { ($($handler:ident: $data:ident,)+) => ($( db_endpoint! { $handler: $data (params: UidParam) { - user_id: params.uid.clone(), + // XXX: -> HawkPayload::uid + user_id: 1, } } )+) @@ -105,8 +106,9 @@ macro_rules! collection_endpoints { ($($handler:ident: $data:ident ($($param:ident: $type:ty),*) {$($property:ident: $value:expr,)*},)+) => ($( db_endpoint! { $handler: $data (params: CollectionParams $(, $param: $type)*) { - user_id: params.uid.clone(), - collection: params.collection.clone(), + // XXX: -> HawkPayload::uid + user_id: 1, + collection_id: 2, // XXX: get_collection_id(¶ms.collection) $($property: $value,)* } } @@ -151,7 +153,7 @@ pub struct PostCollectionBody { impl From for params::PostCollectionBso { fn from(body: PostCollectionBody) -> params::PostCollectionBso { params::PostCollectionBso { - bso_id: body.id.clone(), + id: body.id.clone(), sortindex: body.sortindex, payload: body.payload.as_ref().map(|payload| payload.clone()), ttl: body.ttl, @@ -169,9 +171,10 @@ macro_rules! bso_endpoints { ($($handler:ident: $data:ident ($($param:ident: $type:ty),*) {$($property:ident: $value:expr,)*},)+) => ($( db_endpoint! { $handler: $data (params: BsoParams $(, $param: $type)*) { - user_id: params.uid.clone(), - collection: params.collection.clone(), - bso_id: params.bso.clone(), + // XXX: -> HawkPayload::uid + user_id: 1, + collection_id: 2, // XXX: get_collection_id(¶ms.collection) + id: params.bso.clone(), $($property: $value,)* } } @@ -182,6 +185,7 @@ bso_endpoints! { delete_bso: DeleteBso () {}, get_bso: GetBso () {}, put_bso: PutBso (body: Json) { + modified: ms_since_epoch(), sortindex: body.sortindex, payload: body.payload.as_ref().map(|payload| payload.clone()), ttl: body.ttl, diff --git a/src/main.rs b/src/main.rs index fe8e218960..d98fe1b581 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,16 +3,24 @@ // file, you can obtain one at https://mozilla.org/MPL/2.0/. //! Main application -#[macro_use] extern crate actix; extern crate actix_web; extern crate chrono; extern crate config; #[macro_use] extern crate diesel; +#[cfg(test)] +extern crate diesel_logger; +#[macro_use] +extern crate diesel_migrations; extern crate docopt; +extern crate env_logger; +#[macro_use] +extern crate failure; extern crate futures; //extern crate hawk; +#[macro_use] +extern crate lazy_static; extern crate mozsvc_common; extern crate num_cpus; extern crate rand; diff --git a/src/server/mod.rs b/src/server/mod.rs index b616a9371e..78214d1c18 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -6,9 +6,9 @@ use actix::{System, SystemRunner}; use actix_web::{http, middleware::cors::Cors, server::HttpServer, App}; -use num_cpus; +//use num_cpus; -use db::{mock::MockDb, Db}; +use db::mock::MockDb; use handlers::{self, ServerState}; use settings::Settings; @@ -65,7 +65,7 @@ impl Server { // Setup the server state let state = ServerState { // TODO: replace MockDb with a real implementation - db: MockDb::new(), + db: Box::new(MockDb::new()), }; App::with_state(state).configure(|app| init_routes!(Cors::for_app(app)).register()) diff --git a/src/server/test.rs b/src/server/test.rs index f6b0713a09..3418634927 100644 --- a/src/server/test.rs +++ b/src/server/test.rs @@ -12,7 +12,9 @@ use db::results::{GetBso, GetCollection, PostCollection, PutBso}; use handlers::{BsoBody, PostCollectionBody}; fn setup() -> TestServer { - TestServer::build_with_state(move || ServerState { db: MockDb::new() }).start(|app| { + TestServer::build_with_state(move || ServerState { + db: Box::new(MockDb::new()), + }).start(|app| { init_routes!(app); }) } diff --git a/src/settings.rs b/src/settings.rs index 62b669e526..cbee1d2c6e 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -5,6 +5,10 @@ use config::{Config, ConfigError, Environment, File}; pub struct Settings { pub debug: bool, pub port: u16, + pub database_url: String, + pub database_pool_max_size: Option, + #[cfg(test)] + pub database_use_test_transactions: bool, } impl Settings { @@ -15,6 +19,8 @@ impl Settings { // https://github.com/mehcode/config-rs/issues/60 s.set_default("debug", false)?; s.set_default("port", 8000)?; + #[cfg(test)] + s.set_default("database_use_test_transactions", false)?; // Merge the config file if supplied if let Some(config_filename) = filename {