From ef59da16d54354e652589cd7ce3dd843592e608b Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Tue, 21 Oct 2025 11:36:44 -0400 Subject: [PATCH 1/5] Can decode into ordered maps now --- .gitignore | 2 + lib/cbor.ex | 44 ++++++++++++++++++---- lib/cbor/decoder.ex | 77 ++++++++++++++++++++++++-------------- mix.exs | 1 + mix.lock | 1 + test/cbor/decoder_test.exs | 17 +++++++++ 6 files changed, 106 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index c074bbb..0a003ee 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ cbor-*.tar # Temporary files, for example, from tests. /tmp/ + +.tags diff --git a/lib/cbor.ex b/lib/cbor.ex index 0b15a3f..e8ef246 100644 --- a/lib/cbor.ex +++ b/lib/cbor.ex @@ -99,7 +99,7 @@ defmodule CBOR do @doc """ Converts a CBOR encoded binary into native elixir data structures - ## Examples + ## Examples vanilla elixir maps iex> CBOR.decode(<<130, 101, 72, 101, 108, 108, 111, 102, 87, 111, 114, 108, 100, 33>>) {:ok, ["Hello", "World!"], ""} @@ -109,24 +109,52 @@ defmodule CBOR do iex> CBOR.decode(<<162, 97, 97, 1, 97, 98, 130, 2, 3>>) {:ok, %{"a" => 1, "b" => [2, 3]}, ""} + + iex> CBOR.decode(<<130, 101, 72, 101, 108, 108, 111, 102, 87, 111, 114, 108, 100, 33>>, :unordered) + {:ok, ["Hello", "World!"], ""} + + iex> CBOR.decode(<<130, 1, 130, 2, 3>>, :unordered) + {:ok, [1, [2, 3]], ""} + + iex> CBOR.decode(<<162, 97, 97, 1, 97, 98, 130, 2, 3>>, :unordered) + {:ok, %{"a" => 1, "b" => [2, 3]}, ""} + + ## Ordered Maps + + iex> CBOR.decode(<<130, 101, 72, 101, 108, 108, 111, 102, 87, 111, 114, 108, 100, 33>>, :ordered) + {:ok, ["Hello", "World!"], ""} + + iex> CBOR.decode(<<130, 1, 130, 2, 3>>, :ordered) + {:ok, [1, [2, 3]], ""} + + iex> CBOR.decode(<<162, 97, 97, 1, 97, 98, 130, 2, 3>>, :ordered) + {:ok, %OrdMap{tuples: [{"a", 1}, {"b", [2, 3]}]}, ""} """ - @spec decode(binary()) :: {:ok, any(), binary()} | {:error, atom} - def decode(binary) do + @spec decode(binary(), :ordered | :unordered) :: {:ok, any(), binary()} | {:error, atom} + def decode(binary, is_ordered? \\ :unordered) + def decode(binary, is_ordered?) when is_ordered? in [:ordered, :unordered] do try do - perform_decoding(binary) + perform_decoding(binary, is_ordered?) rescue - FunctionClauseError -> {:error, :cbor_function_clause_error} + e in FunctionClauseError -> dbg(e); {:error, :cbor_function_clause_error} MatchError -> {:error, :cbor_match_error} end end - defp perform_decoding(binary) when is_binary(binary) do - case CBOR.Decoder.decode(binary) do + defp perform_decoding(binary, is_ordered?) + when is_binary(binary) and is_ordered? in [:ordered, :unordered] + do + case CBOR.Decoder.decode(binary, is_ordered?) do {value, rest} -> {:ok, value, rest} _other -> {:error, :cbor_decoder_error} end end - defp perform_decoding(_value), do: {:error, :cannot_decode_non_binary_values} + defp perform_decoding(_value, is_ordered?) + when is_ordered? not in [:ordered, :unordered] + do + {:error, :is_ordered_must_be_ordered_or_unordered} + end + defp perform_decoding(_value, _is_ordered?), do: {:error, :cannot_decode_non_binary_values} end diff --git a/lib/cbor/decoder.ex b/lib/cbor/decoder.ex index ed6519c..54fd04b 100644 --- a/lib/cbor/decoder.ex +++ b/lib/cbor/decoder.ex @@ -1,26 +1,32 @@ defmodule CBOR.Decoder do - def decode(binary) do - decode(binary, header(binary)) + def decode(binary, is_ordered? \\ :unordered) + def decode(binary, is_ordered?) do + decode(binary, header(binary), is_ordered?) end - def decode(_binary, {mt, :indefinite, rest}) do + def decode(_binary, {mt, :indefinite, rest}, is_ordered?) do case mt do 2 -> mark_as_bytes(decode_string_indefinite(rest, 2, [])) 3 -> decode_string_indefinite(rest, 3, []) - 4 -> decode_array_indefinite(rest, []) - 5 -> decode_map_indefinite(rest, %{}) + 4 -> decode_array_indefinite(rest, [], is_ordered?) + 5 -> case is_ordered? do + :ordered -> decode_map_indefinite(rest, OrdMap.new([]), is_ordered?) + :unordered -> decode_map_indefinite(rest, %{}, is_ordered?) + end end end - - def decode(bin, {mt, value, rest}) do + def decode(bin, {mt, value, rest}, is_ordered?) do case mt do 0 -> {value, rest} 1 -> {-value - 1, rest} 2 -> mark_as_bytes(decode_string(rest, value)) 3 -> decode_string(rest, value) - 4 -> decode_array(value, rest) - 5 -> decode_map(value, rest) - 6 -> decode_other(value, decode(rest)) + 4 -> decode_array(value, rest, is_ordered?) + 5 -> case is_ordered? do + :ordered -> decode_ordmap(value, rest) + :unordered -> decode_map(value, rest) + end + 6 -> decode_other(value, decode(rest, is_ordered?)) 7 -> decode_float(bin, value, rest) end end @@ -65,38 +71,53 @@ defmodule CBOR.Decoder do end end - defp decode_array(0, rest), do: {[], rest} - defp decode_array(len, rest), do: decode_array(len, [], rest) - defp decode_array(0, acc, bin), do: {Enum.reverse(acc), bin} - defp decode_array(len, acc, bin) do - {value, bin_rest} = decode(bin) - decode_array(len - 1, [value|acc], bin_rest) + defp decode_array(len, rest, is_ordered?) + defp decode_array(0, rest, _is_ordered?), do: {[], rest} + defp decode_array(len, rest, is_ordered?), do: decode_array(len, [], rest, is_ordered?) + + defp decode_array(0, acc, bin, _is_ordered?), do: {Enum.reverse(acc), bin} + defp decode_array(len, acc, bin, is_ordered?) do + {value, bin_rest} = decode(bin, is_ordered?) + decode_array(len - 1, [value|acc], bin_rest, is_ordered?) end - defp decode_array_indefinite(<<0xff, new_rest::binary>>, acc) do + defp decode_array_indefinite(<<0xff, new_rest::binary>>, acc, _is_ordered?) do {Enum.reverse(acc), new_rest} end - - defp decode_array_indefinite(rest, acc) do - {value, new_rest} = decode(rest) - decode_array_indefinite(new_rest, [value | acc]) + defp decode_array_indefinite(rest, acc, is_ordered?) do + {value, new_rest} = decode(rest, is_ordered?) + decode_array_indefinite(new_rest, [value | acc], is_ordered?) end defp decode_map(0, rest), do: {%{}, rest} defp decode_map(len, rest), do: decode_map(len, %{}, rest) defp decode_map(0, acc, bin), do: {acc, bin} defp decode_map(len, acc, bin) do - {key, key_rest} = decode(bin) - {value, bin_rest} = decode(key_rest) + {key, key_rest} = decode(bin, :unordered) + {value, bin_rest} = decode(key_rest, :unordered) decode_map(len - 1, Map.put(acc, key, value), bin_rest) end - defp decode_map_indefinite(<<0xff, new_rest::binary>>, acc), do: {acc, new_rest} - defp decode_map_indefinite(rest, acc) do - {key, key_rest} = decode(rest) - {value, new_rest} = decode(key_rest) - decode_map_indefinite(new_rest, Map.put(acc, key, value)) + defp decode_ordmap(0, rest), do: {OrdMap.new([]), rest} + defp decode_ordmap(len, rest), do: decode_ordmap(len, OrdMap.new([]), rest) + defp decode_ordmap(0, acc, bin), do: {acc, bin} + defp decode_ordmap(len, acc, bin) do + {key, key_rest} = decode(bin, :ordered) + {value, bin_rest} = decode(key_rest, :ordered) + + decode_ordmap(len - 1, OrdMap.put(acc, key, value), bin_rest) + end + + defp decode_map_indefinite(<<0xff, new_rest::binary>>, acc, _is_ordered?), do: {acc, new_rest} + defp decode_map_indefinite(rest, acc, is_ordered?) do + {key, key_rest} = decode(rest, is_ordered?) + {value, new_rest} = decode(key_rest, is_ordered?) + new_map = case is_ordered? do + :ordered -> OrdMap.put(acc, key, value) + :unordered -> Map.put(acc, key, value) + end + decode_map_indefinite(new_rest, new_map, is_ordered?) end defp decode_float(bin, value, rest) do diff --git a/mix.exs b/mix.exs index ca59922..7cd02e1 100644 --- a/mix.exs +++ b/mix.exs @@ -24,6 +24,7 @@ defmodule Cbor.MixProject do defp deps do [ + {:ord_map, "~> 0.1.0"}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} ] end diff --git a/mix.lock b/mix.lock index 72dfa48..624d372 100644 --- a/mix.lock +++ b/mix.lock @@ -7,4 +7,5 @@ "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, + "ord_map": {:hex, :ord_map, "0.1.0", "6c958f53e38934a2f60d4b0050e3bdfcc498071769f93887372ae4db5cb21d3c", [:mix], [], "hexpm", "c3c87eea4f196bf0ab316eb2f089a922fb783c59a2a579bcdea616c06a7faebb"}, } diff --git a/test/cbor/decoder_test.exs b/test/cbor/decoder_test.exs index 937ee9e..f3d195a 100644 --- a/test/cbor/decoder_test.exs +++ b/test/cbor/decoder_test.exs @@ -350,26 +350,36 @@ defmodule CBOR.DecoderTest do test "RFC 7049 Appendix A Example 67" do encoded = <<160>> assert CBOR.decode(encoded) == {:ok, %{}, ""} + + assert CBOR.decode(encoded, :ordered) == {:ok, %OrdMap{tuples: []}, ""} end test "RFC 7049 Appendix A Example 68" do encoded = <<162, 1, 2, 3, 4>> assert CBOR.decode(encoded) == {:ok, %{1 => 2, 3 => 4}, ""} + + assert CBOR.decode(encoded, :ordered) == {:ok, %OrdMap{tuples: [{1, 2}, {3, 4}]}, ""} end test "RFC 7049 Appendix A Example 69" do encoded = <<162, 97, 97, 1, 97, 98, 130, 2, 3>> assert CBOR.decode(encoded) == {:ok, %{"a" => 1, "b" => [2, 3]}, ""} + + assert CBOR.decode(encoded, :ordered) == {:ok, %OrdMap{tuples: [{"a", 1}, {"b", [2,3]}]}, ""} end test "RFC 7049 Appendix A Example 70" do encoded = <<130, 97, 97, 161, 97, 98, 97, 99>> assert CBOR.decode(encoded) == {:ok, ["a", %{"b" => "c"}], ""} + + assert CBOR.decode(encoded, :ordered) == {:ok, ["a", %OrdMap{tuples: [{"b", "c"}]}], ""} end test "RFC 7049 Appendix A Example 71" do encoded = <<165, 97, 97, 97, 65, 97, 98, 97, 66, 97, 99, 97, 67, 97, 100, 97, 68, 97, 101, 97, 69>> assert CBOR.decode(encoded) == {:ok, %{"a" => "A", "b" => "B", "c" => "C", "d" => "D", "e" => "E"}, ""} + + assert CBOR.decode(encoded, :ordered) == {:ok, %OrdMap{tuples: [{"a", "A"}, {"b", "B"}, {"c", "C"}, {"d", "D"}, {"e", "E"}]}, ""} end test "RFC 7049 Appendix A Example 72" do @@ -415,20 +425,27 @@ defmodule CBOR.DecoderTest do test "RFC 7049 Appendix A Example 80" do encoded = <<191, 97, 97, 1, 97, 98, 159, 2, 3, 255, 255>> assert CBOR.decode(encoded) == {:ok, %{"a" => 1, "b" => [2, 3]}, ""} + + assert CBOR.decode(encoded, :ordered) == {:ok, %OrdMap{tuples: [{"a", 1}, {"b", [2,3]}]}, ""} end test "RFC 7049 Appendix A Example 81" do encoded = <<130, 97, 97, 191, 97, 98, 97, 99, 255>> assert CBOR.decode(encoded) == {:ok, ["a", %{"b" => "c"}], ""} + + assert CBOR.decode(encoded, :ordered) == {:ok, ["a", %OrdMap{tuples: [{"b", "c"}]}], ""} end test "RFC 7049 Appendix A Example 82" do encoded = <<191, 99, 70, 117, 110, 245, 99, 65, 109, 116, 33, 255>> assert CBOR.decode(encoded) == {:ok, %{"Fun" => true, "Amt" => -2}, ""} + + assert CBOR.decode(encoded, :ordered) == {:ok, %OrdMap{tuples: [{"Fun", true}, {"Amt", -2}]}, ""} end test "receiving a MatchError" do encoded = "You done goofed" assert CBOR.decode(encoded) == {:error, :cbor_match_error} + assert CBOR.decode(encoded, :ordered) == {:error, :cbor_match_error} end end From 407c318159fddee9381ef2b517386267dcc66783 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Tue, 21 Oct 2025 11:52:06 -0400 Subject: [PATCH 2/5] Encoding now suports ordered Maps --- lib/cbor.ex | 3 +++ lib/cbor/encoder.ex | 16 ++++++++++++++++ test/cbor/encoder_test.exs | 11 +++++++++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/cbor.ex b/lib/cbor.ex index e8ef246..02d390c 100644 --- a/lib/cbor.ex +++ b/lib/cbor.ex @@ -92,6 +92,9 @@ defmodule CBOR do iex> CBOR.encode(%{"a" => 1, "b" => [2, 3]}) <<162, 97, 97, 1, 97, 98, 130, 2, 3>> + iex> CBOR.encode(%OrdMap{tuples: [{"a", 1}, {"b", [2, 3]}]}) + <<162, 97, 97, 1, 97, 98, 130, 2, 3>> + """ @spec encode(any()) :: binary() def encode(value), do: CBOR.Encoder.encode_into(value, <<>>) diff --git a/lib/cbor/encoder.ex b/lib/cbor/encoder.ex index ab9ff20..7bd82b8 100644 --- a/lib/cbor/encoder.ex +++ b/lib/cbor/encoder.ex @@ -119,6 +119,22 @@ defimpl CBOR.Encoder, for: Map do end end +defimpl CBOR.Encoder, for: OrdMap do + def encode_into(%OrdMap{tuples: []}, acc), do: <> + + def encode_into(%OrdMap{tuples: tuples} = map, acc) when length(tuples) < 0x10000000000000000 do + Enum.reduce(map, CBOR.Utils.encode_head(5, length(tuples), acc), fn({k, v}, subacc) -> + CBOR.Encoder.encode_into(v, CBOR.Encoder.encode_into(k, subacc)) + end) + end + + def encode_into(map, acc) do + Enum.reduce(map, <>, fn({k, v}, subacc) -> + CBOR.Encoder.encode_into(v, CBOR.Encoder.encode_into(k, subacc)) + end) <> <<0xff>> + end +end + # We convert MapSets into lists since there is no 'set' representation defimpl CBOR.Encoder, for: MapSet do def encode_into(map_set, acc) do diff --git a/test/cbor/encoder_test.exs b/test/cbor/encoder_test.exs index 5df5e51..d31b179 100644 --- a/test/cbor/encoder_test.exs +++ b/test/cbor/encoder_test.exs @@ -1,8 +1,9 @@ defmodule CBOR.EncoderTest do use ExUnit.Case, async: true - defp reconstruct(value) do - value |> CBOR.encode() |> CBOR.decode() + defp reconstruct(value, ordered \\ :unordered) + defp reconstruct(value, ordered) do + value |> CBOR.encode() |> CBOR.decode(ordered) end test "given the value of true" do @@ -79,14 +80,20 @@ defmodule CBOR.EncoderTest do test "an empty map" do assert reconstruct(%{}) == {:ok, %{}, ""} + assert reconstruct(%OrdMap{tuples: []}, :ordered) == {:ok, %OrdMap{tuples: []}, ""} end test "a map with atom keys and values" do assert reconstruct(%{foo: :bar, baz: :quux}) == {:ok, %{"foo" => "bar", "baz" => "quux"}, ""} + assert reconstruct(%OrdMap{tuples: [{:foo, :bar}, {:baz, :quux}]}, :ordered) == + {:ok, %OrdMap{tuples: [{"foo", "bar"}, {"baz", "quux"}]}, ""} end test "complex maps" do assert reconstruct(%{"a" => 1, "b" => [2, 3]}) == {:ok, %{"a" => 1, "b" => [2, 3]}, ""} + + our_ordmap = %OrdMap{tuples: [{"a", 1}, {"b", [2, 3]}]} + assert reconstruct(our_ordmap, :ordered) == {:ok, our_ordmap, ""} end test "tagged infinity" do From 7feb3610fbc4b6ac78983a737f7d0aa5a2964195 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Tue, 21 Oct 2025 12:01:56 -0400 Subject: [PATCH 3/5] Updated README --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 9acf84b..510b928 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,22 @@ iex(2)> CBOR.decode(<<130, 1, 130, 2, 3>>) {:ok, [1, [2, 3]], ""} ``` +### Insertion Ordered Maps + +OrdMaps from the ord\_map project can be accepted by the encoder + +```elixir +iex(1)> CBOR.encode(OrdMap.new([{"a", 1}, {"b", 2}, {"c", 3}])) +<<163, 97, 97, 1, 97, 98, 2, 97, 99, 3>> +``` + +and decoded by explicitly passing the ":ordered" atom to the decode function + +```elixir +iex(2)> CBOR.decode(<<163, 97, 97, 1, 97, 98, 2, 97, 99, 3>>, :ordered) +{:ok, %OrdMap(tuples: [{"a", 1}, {"b", 2}, {"c", 3}]), {}} +``` + ## Design Notes Given that Elixir has more available data types than are supported in CBOR, decisions were made so that encoding complex data structures succeed without throwing errors. My thoughts are collected below so you can understand why encoding and decoding of a value does not necessarily return exactly the same value. From ad6f9db905bcb74b53f34955cdfc7477bb893c7d Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Tue, 21 Oct 2025 12:05:41 -0400 Subject: [PATCH 4/5] Removed left-over dbg statement --- lib/cbor.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cbor.ex b/lib/cbor.ex index 02d390c..211ccdf 100644 --- a/lib/cbor.ex +++ b/lib/cbor.ex @@ -140,7 +140,7 @@ defmodule CBOR do try do perform_decoding(binary, is_ordered?) rescue - e in FunctionClauseError -> dbg(e); {:error, :cbor_function_clause_error} + FunctionClauseError -> {:error, :cbor_function_clause_error} MatchError -> {:error, :cbor_match_error} end end From 2e795c6a230cd35ecdfc807427f5b48c90404041 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Thu, 6 Nov 2025 09:30:09 -0500 Subject: [PATCH 5/5] Updated README with warning and mix to accomodate publishing to hex.pm --- README.md | 20 +++++++++++++++----- mix.exs | 7 ++++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 510b928..0627c1f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,20 @@ +# IMPORTANT + +**This package is a fork of the much better maintained +https://github.com/scalpel-software/cbor to accomodate serializing and +deserializing to an ordered map representation needed for our cesr +implmentation https://github.com/vLEIDA/cesrixir. Unless you need this +capability its probably better to use that library which is better maintained +and the original work (which was a fork of excbor as mentioned below). +** + # CBOR -[![Module Version](https://img.shields.io/hexpm/v/cbor.svg)](https://hex.pm/packages/cbor) -[![Hex Docs](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/cbor/) -[![Total Download](https://img.shields.io/hexpm/dt/cbor.svg)](https://hex.pm/packages/cbor) -[![License](https://img.shields.io/hexpm/l/cbor.svg)](https://github.com/scalpel-software/cbor/blob/master/LICENSE.md) -[![Last Updated](https://img.shields.io/github/last-commit/scalpel-software/cbor.svg)](https://github.com/scalpel-software/cbor/commits/master) +[![Module Version](https://img.shields.io/hexpm/v/cbor.svg)](https://hex.pm/packages/cbor_ordmap) +[![Hex Docs](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/cbor_ordmap/) +[![Total Download](https://img.shields.io/hexpm/dt/cbor.svg)](https://hex.pm/packages/cbor_ordmap) +[![License](https://img.shields.io/hexpm/l/cbor.svg)](https://github.com/vLEIDA/cbor/blob/master/LICENSE.md) +[![Last Updated](https://img.shields.io/github/last-commit/vLEIDA/cbor.svg)](https://github.com/vLEIDA/cbor/commits/master) Implementation of RFC 7049 [CBOR](http://cbor.io) (Concise Binary Object Representation) for Elixir. diff --git a/mix.exs b/mix.exs index 7cd02e1..3f98376 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Cbor.MixProject do use Mix.Project - @source_url "https://github.com/scalpel-software/cbor" + @source_url "https://github.com/vLEIDA/cbor" @version "1.0.1" def project do @@ -31,8 +31,9 @@ defmodule Cbor.MixProject do defp package do [ - description: "Implementation of RFC 7049 (Concise Binary Object Representation)", - maintainers: ["tomciopp"], + name: "cbor_ordmap", + description: "Implementation of RFC 7049 (Concise Binary Object Representation) with support for serializing/deserializing to an ordered map representation.", + maintainers: ["daidoji", "dc7"], licenses: ["MIT"], links: %{"GitHub" => @source_url} ]