From 9e2b04428a918c8eb6113365c738df201139eba4 Mon Sep 17 00:00:00 2001 From: Dmitry Belyaev Date: Wed, 6 Dec 2017 13:43:46 +1100 Subject: [PATCH 1/2] Support Couchbase 5.0 Admin UI and rework data type flags handling * If a modern datatype flag is set, only it is used, legacy is ignored. * Only one datatype may be encoded/decoded --- lib/transcoder.ex | 89 +++++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/lib/transcoder.ex b/lib/transcoder.ex index 87359c7..3143072 100644 --- a/lib/transcoder.ex +++ b/lib/transcoder.ex @@ -1,8 +1,8 @@ defmodule Couchie.Transcoder do use Bitwise - #When editing documents in the administrator flags are changed to 0x01. - #We're going to treat it as JSON data. Better ideas welcome. + # When editing documents in the Couchbase Web UI 4.x flags are changed to 0x01. + # We're going to treat it as JSON data. Better ideas welcome. @gui_legacy 0x01 @json_flag 0x02 <<< 24 @@ -14,64 +14,77 @@ defmodule Couchie.Transcoder do @str_flag 0x04 <<< 24 @str_flag_legacy 0x08 - @flag_mask 0x01 ||| 0x02 <<< 24 ||| 0x02 ||| 0x03 <<< 24 ||| 0x04 ||| 0x04 <<< 24 ||| 0x08 - #API defined by cberl_transcoder.erl + # When a document is created in the Couchbase Web UI 5.x flags are set to 0x02_000006 + # which is @json_flag + @json_flag_legacy + @str_flag_legacy. + # So it seems safe to assume that if a modern type is not 0, legacy part must be ignored. + # + # Also a combination of legacy flags is quite confusing. How can a value be encoded as JSON and + # a string simultaneously? It feels safer to drop support for such possibly erroneous data. + # + # From https://developer.couchbase.com/documentation/server/current/sdk/nonjson.html + # One of the metadata fields is a 32 bit "flag" value. + # all legacy typecodes (regardless of language) are under 24 bits in width + + # Modern flags mask + @flag_mask 0xFF_00_00_00 + + # API defined by cberl_transcoder.erl # Encoder - def encode_value(encoders, value) do - do_encode_value(flag(encoders), value) + # As described earlier there is no sense in combining json and raw or string, + # so only one encoder is supported. + + def encode_value(encoder, value) do + # In parallel to the call to this function cberl uses exported `flag/1` to + # obtain the type descriptor of the encoded value. So to ensure that flag is + # synchronized with the actual encoding `flag/1`should also be used here. + # + # A better way would be if this function returned both the type descriptor + # and the encoded value, but cberl must be upgraded for that. + do_encode_value(flag(encoder), value) end - def do_encode_value(flag, value) when (flag &&& @flag_mask) === @str_flag do - do_encode_value(flag ^^^ @str_flag, value) + defp do_encode_value(flag, value) when flag === @json_flag do + Poison.encode!(value) end - def do_encode_value(flag, value) when (flag &&& @flag_mask) === @json_flag do - do_encode_value(flag ^^^ @json_flag, Poison.encode!(value)) + defp do_encode_value(flag, value) when flag === @str_flag do + value end - def do_encode_value(flag, value) when (flag &&& @flag_mask) === @raw_flag do - do_encode_value(flag ^^^ @raw_flag, :erlang.term_to_binary(value)) + defp do_encode_value(flag, value) when flag === @raw_flag do + :erlang.term_to_binary(value) end - def do_encode_value(_, value), do: value # Decoder - def decode_value(flag, value) when (flag &&& @flag_mask) === @raw_flag do - decode_value(flag ^^^ @raw_flag, :erlang.binary_to_term(value)) - end - def decode_value(flag, value) when (flag &&& @flag_mask) === @raw_flag_legacy do - decode_value(flag ^^^ @raw_flag_legacy, :erlang.binary_to_term(value)) + def decode_value(flag, value) when (flag &&& @flag_mask) === @json_flag + or flag === @json_flag_legacy + or flag === (@json_flag_legacy + @str_flag_legacy) + or flag === @gui_legacy do + Poison.decode!(value) end - - def decode_value(flag, value) when (flag &&& @flag_mask) === @json_flag do - decode_value(flag ^^^ @json_flag, Poison.decode!(value)) - end - def decode_value(flag, value) when (flag &&& @flag_mask) === @json_flag_legacy do - decode_value(flag ^^^ @json_flag_legacy, Poison.decode!(value)) - end - def decode_value(flag, value) when (flag &&& @flag_mask) === @gui_legacy do - decode_value(flag ^^^ @json_flag_legacy, Poison.decode!(value)) + def decode_value(flag, value) when (flag &&& @flag_mask) === @str_flag + or flag === @str_flag_legacy do + value end + def decode_value(flag, value) when (flag &&& @flag_mask) === @raw_flag or flag === @raw_flag_legacy do + # The doc says following on RAW flag: + # Indicates this value is a raw sequence of bytes. It is the simplest encoding form and + # indicates that the application will process and interpret its contents as it sees fit. + # + # I strongly suspect this shall be the same as @raw_str, however this might be needed for + # backwards compatibility. Also term_to_binary is unsafe and can cause atom table overflow, + # so this should be decided on the application level, not in the library. - def decode_value(flag, value) when (flag &&& @flag_mask) === @str_flag do - decode_value(flag ^^^ @str_flag, value) - end - def decode_value(flag, value) when (flag &&& @flag_mask) === @str_flag_legacy do - decode_value(flag ^^^ @str_flag_legacy, value) + :erlang.binary_to_term(value) end - def decode_value(_, value), do: value - - def flag(encoders) when is_list(encoders) do - List.foldr(encoders, 0, &Bitwise.bor(&2, &1)) - end - def flag(encoder) do case encoder do :standard -> @json_flag From 87be5d3bb5bb28456675e51cad8d446180758e38 Mon Sep 17 00:00:00 2001 From: Dmitry Belyaev Date: Thu, 7 Dec 2017 15:24:21 +1100 Subject: [PATCH 2/2] Elixir 1.4 compatibility --- lib/couchie.ex | 2 +- mix.exs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/couchie.ex b/lib/couchie.ex index 138be0c..188a6ff 100755 --- a/lib/couchie.ex +++ b/lib/couchie.ex @@ -263,7 +263,7 @@ defmodule Couchie do """ def query(connection, query) do - query = "statement=#{query}" |> to_char_list + query = "statement=#{query}" |> to_charlist case :cberl.http(connection, '', query, 'application/x-www-form-urlencoded; charset=UTF-8', :post, :n1ql) do {:ok, 200, result} -> results = Poison.decode!(result) diff --git a/mix.exs b/mix.exs index d5547ab..4b9c038 100644 --- a/mix.exs +++ b/mix.exs @@ -9,9 +9,9 @@ defmodule Couchie.Mixfile do @doc "Project Details" def project do [ app: :couchie, - elixir: ">= 1.0.2", + elixir: ">= 1.3.0", version: "0.0.7", - deps: deps ] + deps: deps() ] end def application do