From 346498f6ba683ae10f3a70de390a17de72fc9182 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 2 Dec 2025 18:14:42 +0000 Subject: [PATCH 01/10] wip: Add hb_store_common --- src/hb_cache.erl | 295 +++++----------------------------------- src/hb_store_common.erl | 288 +++++++++++++++++++++++++++++++++++++++ src/hb_store_lmdb.erl | 6 +- 3 files changed, 327 insertions(+), 262 deletions(-) create mode 100644 src/hb_store_common.erl diff --git a/src/hb_cache.erl b/src/hb_cache.erl index a0d0b7a80..c0bdd61d3 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -41,7 +41,7 @@ -export([read_all_commitments/2]). -export([ensure_loaded/1, ensure_loaded/2, ensure_all_loaded/1, ensure_all_loaded/2]). -export([read/2, read_resolved/3, write/2, write_binary/3, write_hashpath/2, link/3]). --export([match/2, list/2, list_numbered/2]). +-export([match/2, list_numbered/2]). -export([test_unsigned/1, test_signed/1]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -178,22 +178,7 @@ ensure_all_loaded(Ref, Msg, Opts) -> %% @doc List all items in a directory, assuming they are numbered. list_numbered(Path, Opts) -> SlotDir = hb_store:path(hb_opts:get(store, no_viable_store, Opts), Path), - [ hb_util:int(Name) || Name <- list(SlotDir, Opts) ]. - -%% @doc List all items under a given path. -list(Path, Opts) when is_map(Opts) and not is_map_key(<<"store-module">>, Opts) -> - case hb_opts:get(store, no_viable_store, Opts) of - not_found -> []; - Store -> - list(Path, Store) - end; -list(Path, Store) -> - ResolvedPath = hb_store:resolve(Store, Path), - case hb_store:list(Store, ResolvedPath) of - {ok, Names} -> Names; - {error, _} -> []; - not_found -> [] - end. + [ hb_util:int(Name) || Name <- hb_store_common:list(SlotDir, Opts) ]. %% @doc Match a template message against the cache, returning a list of IDs %% that match the template. We match on the binary representation of values, @@ -396,280 +381,72 @@ write_binary(Hashpath, Bin, Store, Opts) -> %% richly typed map or a direct binary. read(Path, Opts) -> StoreReadResult = - store_read(Path, hb_opts:get(store, no_viable_store, Opts), Opts), + hb_store_common:store_read(Path, hb_opts:get(store, no_viable_store, Opts), Opts), case StoreReadResult of {ok, Res} -> {ok, hb_message:normalize_commitments(Res, Opts)}; _ -> StoreReadResult end. do_read_commitment(Path, Opts) -> - store_read(Path, hb_opts:get(store, no_viable_store, Opts), Opts). + hb_store_common:store_read(Path, hb_opts:get(store, no_viable_store, Opts), Opts). %% @doc Load all of the commitments for a message into memory. read_all_commitments(Msg, Opts) -> Store = hb_opts:get(store, no_viable_store, Opts), - UncommittedID = hb_message:id(Msg, none, Opts#{ linkify_mode => discard }), CurrentCommitments = hb_maps:get(<<"commitments">>, Msg, #{}, Opts), - AlreadyLoaded = hb_maps:keys(CurrentCommitments, Opts), - CommitmentsPath = - hb_store:resolve( - Store, - hb_store:path(Store, [UncommittedID, <<"commitments">>]) - ), - FoundCommitments = - case hb_store:list(Store, CommitmentsPath) of - {ok, CommitmentIDs} -> - lists:filtermap( - fun(CommitmentID) -> - ShouldLoad = not lists:member(CommitmentID, AlreadyLoaded), - ResolvedCommPath = - hb_store:path( - Store, - [CommitmentsPath, CommitmentID] - ), - case ShouldLoad andalso do_read_commitment(ResolvedCommPath, Opts) of - {ok, Commitment} -> - { - true, - { - CommitmentID, - ensure_all_loaded( - Commitment, - Opts#{ commitment => true } - ) - } - }; - _ -> - false - end - end, - CommitmentIDs - ); - not_found -> - [] - end, + FoundCommitments = read_all_commitments_by_store(Msg, Store, Opts), NewCommitments = hb_maps:merge( CurrentCommitments, maps:from_list(FoundCommitments) ), Msg#{ <<"commitments">> => NewCommitments }. -%% @doc List all of the subpaths of a given path and return a map of keys and -%% links to the subpaths, including their types. -store_read(Path, Store, Opts) -> - store_read(Path, Path, Store, Opts). -store_read(_Target, _Path, no_viable_store, _) -> - not_found; -store_read(Target, Path, Store, Opts) -> - ResolvedFullPath = hb_store:resolve(Store, PathBin = hb_path:to_binary(Path)), - ?event({reading, - {original_path, {string, PathBin}}, - {fully_resolved_path, ResolvedFullPath}, - {store, Store} - }), - case hb_store:type(Store, ResolvedFullPath) of - not_found -> not_found; - simple -> - ?event({reading_data, ResolvedFullPath}), - case hb_store:read(Store, ResolvedFullPath) of - {ok, Bin} -> {ok, Bin}; - not_found -> not_found - end; - composite -> - ?event({reading_composite, ResolvedFullPath}), - case hb_store:list(Store, ResolvedFullPath) of - {ok, RawSubpaths} -> - Subpaths = - lists:map(fun hb_util:bin/1, RawSubpaths), - ?event( - {listed, - {original_path, Path}, - {subpaths, {explicit, Subpaths}} - } - ), - % Generate links for each of the listed keys. We only list - % the target ID given in the case of multiple known - % commitments. - Msg = - prepare_links( - Target, - ResolvedFullPath, - Subpaths, - Store, - Opts - ), - ?event( - {completed_read, - {resolved_path, ResolvedFullPath}, - {explicit, Msg} - } - ), - {ok, Msg}; - _ -> - ?event({empty_composite_message, ResolvedFullPath}), - {ok, #{}} - end - end. -%% @doc Prepare a set of links from a listing of subpaths. -prepare_links(Target, RootPath, Subpaths, Store, Opts) -> - {ok, Implicit, Types} = read_ao_types(RootPath, Subpaths, Store, Opts), - Res = - maps:from_list(lists:filtermap( - fun(<<"ao-types">>) -> false; - (<<"commitments">>) -> - % List the commitments for this message, and load them into - % memory. If there no commitments at the path, we exclude - % commitments from the list of links. - CommPath = - hb_store:resolve( +read_all_commitments_by_store(Msg, Store, Opts) when not is_list(Store) -> + read_all_commitments_by_store(Msg, [Store], Opts); +read_all_commitments_by_store(_Msg, [], _Opts) -> + []; +read_all_commitments_by_store(Msg, [Store | ReaminingStores], Opts) -> + CurrentCommitments = hb_maps:get(<<"commitments">>, Msg, #{}, Opts), + AlreadyLoaded = hb_maps:keys(CurrentCommitments, Opts), + UncommittedID = hb_message:id(Msg, none, Opts#{ linkify_mode => discard }), + CommitmentsPath = + hb_store:resolve( + Store, + hb_store:path(Store, [UncommittedID, <<"commitments">>]) + ), + case hb_store:list(Store, CommitmentsPath) of + {ok, CommitmentIDs} -> + lists:filtermap( + fun(CommitmentID) -> + ShouldLoad = not lists:member(CommitmentID, AlreadyLoaded), + ResolvedCommPath = + hb_store:path( Store, - hb_store:path( - Store, - [ - RootPath, - <<"commitments">>, - Target - ] - ) + [CommitmentsPath, CommitmentID] ), - ?event(read_commitment, - {reading_commitment, - {target, Target}, - {root_path, RootPath}, - {commitments_path, CommPath} - } - ), - case do_read_commitment(CommPath, Opts) of + case ShouldLoad andalso do_read_commitment(ResolvedCommPath, Opts) of {ok, Commitment} -> - LoadedCommitment = - ensure_all_loaded( - Commitment, - Opts#{ commitment => true } - ), - ?event(read_commitment, - {found_target_commitment, - {path, CommPath}, - {commitment, LoadedCommitment} - } - ), - % We have commitments, so we read each commitment - % into memory, and return it as part of the message. { true, { - <<"commitments">>, - #{ Target => LoadedCommitment } + CommitmentID, + ensure_all_loaded( + Commitment, + Opts#{ commitment => true } + ) } }; _ -> false - end; - (Subpath) -> - ?event( - {returning_link, - {subpath, Subpath} - } - ), - SubkeyPath = hb_store:path(Store, [RootPath, Subpath]), - case hb_link:is_link_key(Subpath) of - false -> - % The key is a literal value, not a nested composite - % message. Subsequently, we return a resolvable link - % to the subpath, leaving the key as-is. - {true, - { - Subpath, - {link, - SubkeyPath, - (case Types of - #{ Subpath := Type } -> - % We have an `ao-types' entry for the - % subpath, so we return a link to the - % subpath with `lazy' set to `true' - % because we need to resolve the link - % to get the final value. - #{ - <<"type">> => Type, - <<"lazy">> => true - }; - _ -> - % We do not have an `ao-types' entry for the - % subpath, so we return a link to the - % subpath with `lazy' set to `true', - % because the subpath is a literal - % value. - #{ - <<"lazy">> => true - } - end)#{ store => Store } - } - } - }; - true -> - % The key is an encoded link, so we create a resolvable - % link to the underlying link. This requires that we - % dereference the link twice in order to get the final - % value. Returning the data this way avoids having to - % read each of the link keys themselves, which may be - % a large quantity. - {true, - { - binary:part(Subpath, 0, byte_size(Subpath) - 5), - {link, SubkeyPath, #{ - <<"type">> => <<"link">>, - <<"lazy">> => true - }} - } - } end end, - Subpaths - )), - Merged = maps:merge(Res, Implicit), - % Convert the message to an ordered list if the ao-types indicate that it - % should be so. If it is a message, we ensure that the commitments are - % normalized (have an unsigned comm. ID) and loaded into memory. - case dev_codec_structured:is_list_from_ao_types(Types, Opts) of - true -> - hb_util:message_to_ordered_list(Merged, Opts); - false -> - case hb_opts:get(lazy_loading, true, Opts) of - true -> Merged; - false -> ensure_all_loaded(Merged, Opts) - end - end. - -%% @doc Read and parse the ao-types for a given path if it is in the supplied -%% list of subpaths, returning a map of keys and their types. -read_ao_types(Path, Subpaths, Store, Opts) -> - ?event({reading_ao_types, {path, Path}, {subpaths, {explicit, Subpaths}}}), - case lists:member(<<"ao-types">>, Subpaths) of - true -> - {ok, TypesBin} = - hb_store:read( - Store, - hb_store:path(Store, [Path, <<"ao-types">>]) - ), - Types = dev_codec_structured:decode_ao_types(TypesBin, Opts), - ?event({parsed_ao_types, {types, Types}}), - {ok, types_to_implicit(Types), Types}; - false -> - ?event({no_ao_types_key_found, {path, Path}, {subpaths, Subpaths}}), - {ok, #{}, #{}} + CommitmentIDs + ); + not_found -> + read_all_commitments_by_store(Msg, ReaminingStores, Opts) end. -%% @doc Convert a map of ao-types to an implicit map of types. -types_to_implicit(Types) -> - maps:filtermap( - fun(_K, <<"empty-message">>) -> {true, #{}}; - (_K, <<"empty-list">>) -> {true, []}; - (_K, <<"empty-binary">>) -> {true, <<>>}; - (_, _) -> false - end, - Types - ). - %% @doc Read the result of a computation, using heuristics. The supported %% heuristics are as follows: %% 1. If the base message is an ID, we try to determine if the message has an diff --git a/src/hb_store_common.erl b/src/hb_store_common.erl new file mode 100644 index 000000000..afc568643 --- /dev/null +++ b/src/hb_store_common.erl @@ -0,0 +1,288 @@ +%% @doc Store standard library +%% +%% Common store access patterns should be defined here. +%% +%% TODO: +%% - Find other cases where type/resolve/read are used together in the same function. +%% - Look into type/read cache miss? Jack was working on it, but maybe I can pick it up (at least my understanding from the meeting) +%% - Make tests for stores more generic, avoid individual tests (there are some in S3 that can be applied to LMDB). +%% - But since S3 isn't in edge, this should be another PR. + +-module(hb_store_common). +-export([list/2, store_read/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc List all items under a given path. +list(Path, Opts) when is_map(Opts) and not is_map_key(<<"store-module">>, Opts) -> + case hb_opts:get(store, no_viable_store, Opts) of + not_found -> []; + Store -> + list(Path, Store) + end; +list(Path, Store) when is_map(Store)-> + list(Path, [Store]); +list(_Path, []) -> + []; +list(Path, [Store | RemainingStores]) -> + ResolvedPath = hb_store:resolve(Store, Path), + case hb_store:list(Store, ResolvedPath) of + {ok, Names} -> Names; + _ -> list(Path, RemainingStores) + end. + +%% @doc List all of the subpaths of a given path and return a map of keys and +%% links to the subpaths, including their types. +store_read(Path, Store, Opts) -> + store_read(Path, Path, Store, Opts). +store_read(_Target, _Path, no_viable_store, _) -> + not_found; +store_read(_Target, _Path, [], _) -> + not_found; +store_read(Target, Path, Store, Opts) when is_map(Store) -> + store_read(Target, Path, [Store], Opts); +store_read(Target, Path, [Store | RemainingStores], Opts) -> + ResolvedFullPath = hb_store:resolve(Store, PathBin = hb_path:to_binary(Path)), + ?event({reading, + {original_path, {string, PathBin}}, + {fully_resolved_path, ResolvedFullPath}, + {store, Store} + }), + ResolvedFullPathContent = case hb_store:type(Store, ResolvedFullPath) of + not_found -> not_found; + simple -> + ?event({reading_data, ResolvedFullPath}), + case hb_store:read(Store, ResolvedFullPath) of + {ok, Bin} -> {ok, Bin}; + not_found -> not_found + end; + composite -> + ?event({reading_composite, ResolvedFullPath}), + case hb_store:list(Store, ResolvedFullPath) of + {ok, RawSubpaths} -> + Subpaths = + lists:map(fun hb_util:bin/1, RawSubpaths), + ?event( + {listed, + {original_path, Path}, + {subpaths, {explicit, Subpaths}} + } + ), + % Generate links for each of the listed keys. We only list + % the target ID given in the case of multiple known + % commitments. + Msg = + prepare_links( + Target, + ResolvedFullPath, + Subpaths, + Store, + Opts + ), + ?event( + {completed_read, + {resolved_path, ResolvedFullPath}, + {explicit, Msg} + } + ), + {ok, Msg}; + _ -> + ?event({empty_composite_message, ResolvedFullPath}), + {ok, #{}} + end + end, + case ResolvedFullPathContent of + {ok, _} = Response -> Response; + not_found -> store_read(Target, Path, RemainingStores, Opts) + end. + +%% @doc Prepare a set of links from a listing of subpaths. +prepare_links(Target, RootPath, Subpaths, Store, Opts) -> + {ok, Implicit, Types} = read_ao_types(RootPath, Subpaths, Store, Opts), + Res = + maps:from_list(lists:filtermap( + fun(<<"ao-types">>) -> false; + (<<"commitments">>) -> + % List the commitments for this message, and load them into + % memory. If there no commitments at the path, we exclude + % commitments from the list of links. + CommPath = + hb_store:resolve( + Store, + hb_store:path( + Store, + [ + RootPath, + <<"commitments">>, + Target + ] + ) + ), + ?event(read_commitment, + {reading_commitment, + {target, Target}, + {root_path, RootPath}, + {commitments_path, CommPath} + } + ), + %% TODO: Maybe improve this? This line bellow is called in + %% hb_cache:do_read_commitment + case store_read(CommPath, hb_opts:get(store, no_viable_store, Opts), Opts) of + {ok, Commitment} -> + LoadedCommitment = + hb_cache:ensure_all_loaded( + Commitment, + Opts#{ commitment => true } + ), + ?event(read_commitment, + {found_target_commitment, + {path, CommPath}, + {commitment, LoadedCommitment} + } + ), + % We have commitments, so we read each commitment + % into memory, and return it as part of the message. + { + true, + { + <<"commitments">>, + #{ Target => LoadedCommitment } + } + }; + _ -> + false + end; + (Subpath) -> + ?event( + {returning_link, + {subpath, Subpath} + } + ), + SubkeyPath = hb_store:path(Store, [RootPath, Subpath]), + case hb_link:is_link_key(Subpath) of + false -> + % The key is a literal value, not a nested composite + % message. Subsequently, we return a resolvable link + % to the subpath, leaving the key as-is. + {true, + { + Subpath, + {link, + SubkeyPath, + (case Types of + #{ Subpath := Type } -> + % We have an `ao-types' entry for the + % subpath, so we return a link to the + % subpath with `lazy' set to `true' + % because we need to resolve the link + % to get the final value. + #{ + <<"type">> => Type, + <<"lazy">> => true + }; + _ -> + % We do not have an `ao-types' entry for the + % subpath, so we return a link to the + % subpath with `lazy' set to `true', + % because the subpath is a literal + % value. + #{ + <<"lazy">> => true + } + end)#{ store => Store } + } + } + }; + true -> + % The key is an encoded link, so we create a resolvable + % link to the underlying link. This requires that we + % dereference the link twice in order to get the final + % value. Returning the data this way avoids having to + % read each of the link keys themselves, which may be + % a large quantity. + {true, + { + binary:part(Subpath, 0, byte_size(Subpath) - 5), + {link, SubkeyPath, #{ + <<"type">> => <<"link">>, + <<"lazy">> => true + }} + } + } + end + end, + Subpaths + )), + Merged = maps:merge(Res, Implicit), + % Convert the message to an ordered list if the ao-types indicate that it + % should be so. If it is a message, we ensure that the commitments are + % normalized (have an unsigned comm. ID) and loaded into memory. + case dev_codec_structured:is_list_from_ao_types(Types, Opts) of + true -> + hb_util:message_to_ordered_list(Merged, Opts); + false -> + case hb_opts:get(lazy_loading, true, Opts) of + true -> Merged; + false -> hb_cache:ensure_all_loaded(Merged, Opts) + end + end. + +%% @doc Read and parse the ao-types for a given path if it is in the supplied +%% list of subpaths, returning a map of keys and their types. +read_ao_types(Path, Subpaths, Store, Opts) -> + ?event({reading_ao_types, {path, Path}, {subpaths, {explicit, Subpaths}}}), + case lists:member(<<"ao-types">>, Subpaths) of + true -> + {ok, TypesBin} = + hb_store:read( + Store, + hb_store:path(Store, [Path, <<"ao-types">>]) + ), + Types = dev_codec_structured:decode_ao_types(TypesBin, Opts), + ?event({parsed_ao_types, {types, Types}}), + {ok, types_to_implicit(Types), Types}; + false -> + ?event({no_ao_types_key_found, {path, Path}, {subpaths, Subpaths}}), + {ok, #{}, #{}} + end. + +%% @doc Convert a map of ao-types to an implicit map of types. +types_to_implicit(Types) -> + maps:filtermap( + fun(_K, <<"empty-message">>) -> {true, #{}}; + (_K, <<"empty-list">>) -> {true, []}; + (_K, <<"empty-binary">>) -> {true, <<>>}; + (_, _) -> false + end, + Types + ). + +%% Tests + +%% @doc Initialize multiple stores +get_multiple_stores() -> + Store1 = hb_test_utils:test_store(hb_store_lmdb, <<"store1">>), + Store2 = hb_test_utils:test_store(hb_store_lmdb, <<"store2">>), + [Store1, Store2]. + +%% @doc Shutdown multiple stores +shutdown_stores([]) -> ok; +shutdown_stores([Store | RemainingStores]) -> + hb_store:reset(Store), + hb_store:stop(Store), + shutdown_stores(RemainingStores). + +%% @doc Read value from Store1 and Store2 when is only available in Store2 +multiple_stores_store_read_test() -> + [Store1, Store2] = Stores = get_multiple_stores(), + %% Write test data + hb_store:make_group(Store2, <<"group1">>), + hb_store:write(Store2, <<"data/final_id">>, <<"data">>), + hb_store:make_link(Store2, <<"data/final_id">>, <<"group1/data">>), + hb_store:make_link(Store2, <<"group1">>, <<"random_id">>), + %% Check result + Opts = #{}, + Path = <<"random_id">>, + Content = store_read(Path, [Store1, Store2], Opts), + ?assertMatch({ok, #{<<"data">> := _}}, Content), + shutdown_stores(Stores). diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index 0a970513c..ef31e0dbd 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -367,8 +367,8 @@ list(Opts, Path) -> #{ <<"db">> := DBInstance } = find_env(Opts), case elmdb:list(DBInstance, SearchPath) of {ok, Children} -> {ok, Children}; - {error, not_found} -> {ok, []}; % Normalize new error format - not_found -> {ok, []} % Handle both old and new format + {error, not_found} -> not_found; % Normalize new error format + not_found -> not_found % Handle both old and new format end. %% @doc Match a series of keys and values against the database. Returns @@ -661,7 +661,7 @@ list_test() -> <<"capacity">> => ?DEFAULT_SIZE }, reset(StoreOpts), - ?assertEqual(list(StoreOpts, <<"colors">>), {ok, []}), + ?assertEqual(list(StoreOpts, <<"colors">>), not_found), % Create immediate children under colors/ write(StoreOpts, <<"colors/red">>, <<"1">>), write(StoreOpts, <<"colors/blue">>, <<"2">>), From 7b50d9a91199c278052b1974db091ff869f30c48 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 3 Dec 2025 20:11:05 +0000 Subject: [PATCH 02/10] fix: Abstract resolve type and list functionality --- src/dev_process_cache.erl | 5 +- src/hb_cache.erl | 2 +- src/hb_store_common.erl | 146 ++++++++++++++++++++++++++++++++------ src/hb_store_lmdb.erl | 2 + src/hb_store_lru.erl | 6 +- 5 files changed, 131 insertions(+), 30 deletions(-) diff --git a/src/dev_process_cache.erl b/src/dev_process_cache.erl index 0dd96bbbf..9b77718d4 100644 --- a/src/dev_process_cache.erl +++ b/src/dev_process_cache.erl @@ -136,9 +136,8 @@ first_with_path(_ProcID, _Required, [], _Opts, _Store) -> not_found; first_with_path(ProcID, RequiredPath, [Slot | Rest], Opts, Store) -> RawPath = path(ProcID, Slot, RequiredPath, Opts), - ResolvedPath = hb_store:resolve(Store, RawPath), - ?event({trying_slot, {slot, Slot}, {path, RawPath}, {resolved_path, ResolvedPath}}), - case hb_store:type(Store, ResolvedPath) of + ?event({trying_slot, {slot, Slot}, {path, RawPath}}), + case hb_store_common:resolved_type(Store, RawPath) of not_found -> first_with_path(ProcID, RequiredPath, Rest, Opts, Store); _ -> diff --git a/src/hb_cache.erl b/src/hb_cache.erl index c0bdd61d3..988da686b 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -425,7 +425,7 @@ read_all_commitments_by_store(Msg, [Store | ReaminingStores], Opts) -> Store, [CommitmentsPath, CommitmentID] ), - case ShouldLoad andalso do_read_commitment(ResolvedCommPath, Opts) of + case ShouldLoad andalso do_read_commitment(ResolvedCommPath, Opts#{store => Store}) of {ok, Commitment} -> { true, diff --git a/src/hb_store_common.erl b/src/hb_store_common.erl index afc568643..6fe04bdfe 100644 --- a/src/hb_store_common.erl +++ b/src/hb_store_common.erl @@ -2,14 +2,13 @@ %% %% Common store access patterns should be defined here. %% -%% TODO: -%% - Find other cases where type/resolve/read are used together in the same function. -%% - Look into type/read cache miss? Jack was working on it, but maybe I can pick it up (at least my understanding from the meeting) +%% TODO: +%% - Test list/1 %% - Make tests for stores more generic, avoid individual tests (there are some in S3 that can be applied to LMDB). -%% - But since S3 isn't in edge, this should be another PR. -module(hb_store_common). --export([list/2, store_read/3]). +-export([list/2, resolve/2, store_read/3]). +-export([resolved_list/2, resolved_type/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -31,6 +30,63 @@ list(Path, [Store | RemainingStores]) -> _ -> list(Path, RemainingStores) end. +%% NOTE: resolved_read/2 not needed because stores normally implement +%% resolve/2 inside read/2. + +%% Old version, to be deleted before merge +resolved_list2(Store, Path) -> + ResolvedPath = hb_store:resolve(Store, Path), + hb_store:list(Store, ResolvedPath). + +resolved_list(Stores, Path) when is_list(Stores) -> + do_resolved_list(Stores, Path); +resolved_list(Store, Path) -> + do_resolved_list([Store], Path). + +do_resolved_list([], _Path) -> + not_found; +do_resolved_list([Store|RemainingStores], Path) -> + ResolvedPath = hb_store:resolve(Store, Path), + erlang:display([{path, Path}, {resolved_path, ResolvedPath}]), + ?event({resolved_list, {path, Path}, {resolved_path, ResolvedPath}}), + case hb_store:list(Store, ResolvedPath) of + {ok, _} = Result -> Result; + not_found -> do_resolved_list(RemainingStores, Path) + end. + +%% Old version, to be deleted before merge +resolved_type2(Store, Path) -> + ResolvedPath = hb_store:resolve(Store, Path), + ?event({resolved_type, {path, Path}, {resolved_path, ResolvedPath}}), + hb_store:type(Store, ResolvedPath). + +resolved_type(Stores, Path) when is_list(Stores) -> + do_resolved_type(Stores, Path); +resolved_type(Store, Path) -> + do_resolved_type([Store], Path). + +do_resolved_type([], _Path) -> not_found; +do_resolved_type([Store|RemainingStores], Path) -> + ResolvedPath = hb_store:resolve(Store, Path), + ?event({resolved_type, {path, Path}, {resolved_path, ResolvedPath}}), + case hb_store:type(Store, ResolvedPath) of + Result when Result =/= not_found -> Result; + _ -> do_resolved_type(RemainingStores, Path) + end. + +%% TODO: This should replace the retry logic in the `hb_store:resolve` +resolve(Store, Opts) when not is_list(Store) -> + resolve([Store], Opts); +resolve(Stores, Path) -> + do_resolve(Stores, Path). + +do_resolve([], _Path) -> false; +do_resolve([Store|RemainingStores], Path) -> + case hb_store:type(Store, Path) of + not_found -> false; + _ -> {true, hb_store:resolve(RemainingStores, Path)} + end. + %% @doc List all of the subpaths of a given path and return a map of keys and %% links to the subpaths, including their types. store_read(Path, Store, Opts) -> @@ -259,22 +315,9 @@ types_to_implicit(Types) -> %% Tests -%% @doc Initialize multiple stores -get_multiple_stores() -> - Store1 = hb_test_utils:test_store(hb_store_lmdb, <<"store1">>), - Store2 = hb_test_utils:test_store(hb_store_lmdb, <<"store2">>), - [Store1, Store2]. - -%% @doc Shutdown multiple stores -shutdown_stores([]) -> ok; -shutdown_stores([Store | RemainingStores]) -> - hb_store:reset(Store), - hb_store:stop(Store), - shutdown_stores(RemainingStores). - %% @doc Read value from Store1 and Store2 when is only available in Store2 multiple_stores_store_read_test() -> - [Store1, Store2] = Stores = get_multiple_stores(), + [_Store1, Store2] = Stores = get_multiple_stores(), %% Write test data hb_store:make_group(Store2, <<"group1">>), hb_store:write(Store2, <<"data/final_id">>, <<"data">>), @@ -283,6 +326,65 @@ multiple_stores_store_read_test() -> %% Check result Opts = #{}, Path = <<"random_id">>, - Content = store_read(Path, [Store1, Store2], Opts), - ?assertMatch({ok, #{<<"data">> := _}}, Content), - shutdown_stores(Stores). + Content = store_read(Path, Stores, Opts), + try + ?assertMatch({ok, #{<<"data">> := _}}, Content) + after + shutdown_stores(Stores) + end. + +%% @doc Test that resolve and type must be made in the same store, +%% when multiple stores are provided. +resolved_type_test() -> + [_Store1, Store2] = Stores = get_multiple_stores(), + %% Write test data + hb_store:make_group(Store2, <<"group1">>), + hb_store:write(Store2, <<"data/final_id">>, <<"data">>), + hb_store:make_link(Store2, <<"data/final_id">>, <<"group1/data">>), + hb_store:make_link(Store2, <<"group1">>, <<"random_id">>), + %% Check result + RawPath = <<"random_id/data">>, + Result = resolved_type(Stores, RawPath), + try + ?assertEqual(simple, Result) + after + shutdown_stores(Stores) + end. + +%% @doc Test that resolve and list must be made in the same store, +%% when multiple stores are provided. +resolved_list_test() -> + [_Store1, Store2] = Stores = get_multiple_stores(), + %% Write test data + hb_store:make_group(Store2, <<"group1">>), + hb_store:make_group(Store2, <<"group1/group12">>), + hb_store:write(Store2, <<"data/final_id2">>, <<"7890">>), + %% Link + %% TODO: Not sure if this structure is possible in HB + hb_store:make_link(Store2, <<"data/final_id2">>, <<"group1/group12/data">>), + hb_store:make_link(Store2, <<"group1">>, <<"random_id">>), + %% Check result + RawPath = <<"random_id/group12">>, + Result = resolved_list(Stores, RawPath), + try + ?assertEqual({ok, [<<"data">>]}, Result) + after + shutdown_stores(Stores) + end. + +%% Test utilities + +%% @doc Initialize multiple stores +get_multiple_stores() -> + get_multiple_stores(hb_store_lmdb). +get_multiple_stores(StoreModule) -> + Store1 = hb_test_utils:test_store(StoreModule, <<"store1">>), + Store2 = hb_test_utils:test_store(StoreModule, <<"store2">>), + [Store1, Store2]. + +%% @doc Shutdown multiple stores +shutdown_stores([]) -> ok; +shutdown_stores([Store | RemainingStores]) -> + hb_store:reset(Store), + hb_store:stop(Store), + shutdown_stores(RemainingStores). diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index ef31e0dbd..8a83f5271 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -270,6 +270,8 @@ resolve_path_links(Opts, Path, Depth) -> %% Internal helper that accumulates the resolved path resolve_path_links_acc(_Opts, [], AccPath, _Depth) -> % No more segments to process + % TODO: Should this return not_found? + % Maybe not, not_found whould only be returned by `type`, `read`, etc. {ok, lists:reverse(AccPath)}; resolve_path_links_acc(_, FullPath = [<<"data">>|_], [], _Depth) -> {ok, FullPath}; diff --git a/src/hb_store_lru.erl b/src/hb_store_lru.erl index d68aaa8f7..02acb5817 100644 --- a/src/hb_store_lru.erl +++ b/src/hb_store_lru.erl @@ -242,8 +242,7 @@ list(Opts, Path) -> no_store -> not_found; Store -> - ResolvedPath = hb_store:resolve(Store, Path), - case hb_store:list(Store, ResolvedPath) of + case hb_store_common:resolved_list(Store, Path) of {ok, Keys} -> Keys; not_found -> not_found end @@ -287,8 +286,7 @@ type(Opts, Key) -> no_store -> not_found; Store -> - ResolvedKey = hb_store:resolve(Store, Key), - hb_store:type(Store, ResolvedKey) + hb_store_common:resolved_type(Store, Key) end; {raw, _} -> simple; From 71d5b7634a9f132a5c33d017025a7dc569874cb6 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 4 Dec 2025 17:06:54 +0000 Subject: [PATCH 03/10] fix: Small improvements --- src/dev_query_arweave.erl | 13 ++++++++----- src/hb_cache.erl | 7 ++++++- src/hb_store.erl | 2 +- src/hb_store_common.erl | 2 +- src/hb_store_lmdb.erl | 21 +++++++++++++++++++-- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/dev_query_arweave.erl b/src/dev_query_arweave.erl index 19ded4210..2aae907c7 100644 --- a/src/dev_query_arweave.erl +++ b/src/dev_query_arweave.erl @@ -296,10 +296,12 @@ commitment_id_to_base_id(ID, Opts) -> %% @doc Find all IDs for a message, by any of its other IDs. all_ids(ID, Opts) -> Store = hb_opts:get(store, no_store, Opts), - case hb_store:list(Store, << ID/binary, "/commitments">>) of - {ok, []} -> [ID]; - {ok, CommitmentIDs} -> CommitmentIDs; - _ -> [] + case hb_store_common:resolved_list(Store, << ID/binary, "/commitments">>) of + {ok, CommitmentIDs} -> + CommitmentIDs; + not_found -> + %% Add ID to fetch as a message + [ID] end. %% @doc Scope the stores used for block matching. The searched stores can be @@ -314,7 +316,8 @@ resolve_ids(IDs, Opts) -> lists:map( fun(ID) -> case hb_cache:read(ID, Opts) of - {ok, Msg} -> hb_message:id(Msg, uncommitted, Scoped); + {ok, Msg} -> + hb_message:id(Msg, uncommitted, Scoped); not_found -> ID end end, diff --git a/src/hb_cache.erl b/src/hb_cache.erl index 988da686b..8dd07b4cf 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -40,7 +40,7 @@ -module(hb_cache). -export([read_all_commitments/2]). -export([ensure_loaded/1, ensure_loaded/2, ensure_all_loaded/1, ensure_all_loaded/2]). --export([read/2, read_resolved/3, write/2, write_binary/3, write_hashpath/2, link/3]). +-export([read/2, list/2, read_resolved/3, write/2, write_binary/3, write_hashpath/2, link/3]). -export([match/2, list_numbered/2]). -export([test_unsigned/1, test_signed/1]). -include("include/hb.hrl"). @@ -175,6 +175,10 @@ ensure_all_loaded(Ref, Msg, Opts) when is_list(Msg) -> ensure_all_loaded(Ref, Msg, Opts) -> ensure_loaded(Ref, Msg, Opts). +%% TODO: Remove before final PR, replace calls to `hb_store_common` +list(Path, Opts) -> + hb_store_common:list(Path, Opts). + %% @doc List all items in a directory, assuming they are numbered. list_numbered(Path, Opts) -> SlotDir = hb_store:path(hb_opts:get(store, no_viable_store, Opts), Path), @@ -827,6 +831,7 @@ test_match_typed_message(Store) -> cache_suite_test_() -> hb_store:generate_test_suite([ + {"store ans104 message", fun test_store_ans104_message/1}, {"store unsigned empty message", fun test_store_unsigned_empty_message/1}, {"store binary", fun test_store_binary/1}, diff --git a/src/hb_store.erl b/src/hb_store.erl index 5cb9e5527..33c950495 100644 --- a/src/hb_store.erl +++ b/src/hb_store.erl @@ -384,7 +384,7 @@ apply_store_function(Mod, Store, Function, Args, AttemptsRemaining) -> retry -> retry(Mod, Store, Function, Args, AttemptsRemaining); Other -> Other catch Class:Reason:Stacktrace -> - ?event(store_error, + ?event(error, {store_call_failed_retrying, {store, Store}, {function, Function}, diff --git a/src/hb_store_common.erl b/src/hb_store_common.erl index 6fe04bdfe..13a16efc5 100644 --- a/src/hb_store_common.erl +++ b/src/hb_store_common.erl @@ -24,6 +24,7 @@ list(Path, Store) when is_map(Store)-> list(_Path, []) -> []; list(Path, [Store | RemainingStores]) -> + ?event({list, Path}), ResolvedPath = hb_store:resolve(Store, Path), case hb_store:list(Store, ResolvedPath) of {ok, Names} -> Names; @@ -47,7 +48,6 @@ do_resolved_list([], _Path) -> not_found; do_resolved_list([Store|RemainingStores], Path) -> ResolvedPath = hb_store:resolve(Store, Path), - erlang:display([{path, Path}, {resolved_path, ResolvedPath}]), ?event({resolved_list, {path, Path}, {resolved_path, ResolvedPath}}), case hb_store:list(Store, ResolvedPath) of {ok, _} = Result -> Result; diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index 8a83f5271..e1ac110f9 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -369,10 +369,27 @@ list(Opts, Path) -> #{ <<"db">> := DBInstance } = find_env(Opts), case elmdb:list(DBInstance, SearchPath) of {ok, Children} -> {ok, Children}; - {error, not_found} -> not_found; % Normalize new error format - not_found -> not_found % Handle both old and new format + {error, not_found} -> compatibility_not_found(DBInstance, Path); % Normalize new error format + not_found -> compatibility_not_found(DBInstance, Path) % Handle both old and new format + end. + +compatibility_not_found(DBInstance, Path) -> + case elmdb:get(DBInstance, remove_ending_slash(Path)) of + {ok, <<"group">>} -> {ok, []}; + _ -> not_found end. +remove_ending_slash(Path) -> + list_to_binary( + string:reverse( + do_remove_ending_slash( + string:reverse(Path) + ) + )). + +do_remove_ending_slash(["/" | Path]) -> Path; +do_remove_ending_slash(Path) -> Path. + %% @doc Match a series of keys and values against the database. Returns %% `{ok, Matches}' if the match is successful, or `not_found' if there are no %% messages in the store that feature all of the given key-value pairs. `Matches' From 101961c00dcd2511f609f4f5481c92c148720b82 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 5 Dec 2025 11:21:41 +0000 Subject: [PATCH 04/10] fix: Filter stores to only local --- src/hb_cache.erl | 3 ++- src/hb_store_common.erl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hb_cache.erl b/src/hb_cache.erl index 8dd07b4cf..43afcbeeb 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -396,7 +396,8 @@ do_read_commitment(Path, Opts) -> %% @doc Load all of the commitments for a message into memory. read_all_commitments(Msg, Opts) -> - Store = hb_opts:get(store, no_viable_store, Opts), + %% TODO: Confirm tha this should only look into local stores only + Store = hb_store:scope(hb_opts:get(store, no_viable_store, Opts), local), CurrentCommitments = hb_maps:get(<<"commitments">>, Msg, #{}, Opts), FoundCommitments = read_all_commitments_by_store(Msg, Store, Opts), NewCommitments = diff --git a/src/hb_store_common.erl b/src/hb_store_common.erl index 13a16efc5..800986a8d 100644 --- a/src/hb_store_common.erl +++ b/src/hb_store_common.erl @@ -142,7 +142,7 @@ store_read(Target, Path, [Store | RemainingStores], Opts) -> } ), {ok, Msg}; - _ -> + not_found -> ?event({empty_composite_message, ResolvedFullPath}), {ok, #{}} end From 2ae1845f7dee2a00440b1e0bb0808d61a7914801 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 5 Dec 2025 16:45:18 +0000 Subject: [PATCH 05/10] fix: Clean up code --- src/dev_query_arweave.erl | 5 ++--- src/hb_cache.erl | 24 +++++++++++++++++------- src/hb_store.erl | 2 +- src/hb_store_common.erl | 25 +------------------------ src/hb_store_lmdb.erl | 23 +++++++++++++---------- 5 files changed, 34 insertions(+), 45 deletions(-) diff --git a/src/dev_query_arweave.erl b/src/dev_query_arweave.erl index 2aae907c7..9d2ae1abc 100644 --- a/src/dev_query_arweave.erl +++ b/src/dev_query_arweave.erl @@ -296,7 +296,7 @@ commitment_id_to_base_id(ID, Opts) -> %% @doc Find all IDs for a message, by any of its other IDs. all_ids(ID, Opts) -> Store = hb_opts:get(store, no_store, Opts), - case hb_store_common:resolved_list(Store, << ID/binary, "/commitments">>) of + case hb_store:list(Store, << ID/binary, "/commitments">>) of {ok, CommitmentIDs} -> CommitmentIDs; not_found -> @@ -316,8 +316,7 @@ resolve_ids(IDs, Opts) -> lists:map( fun(ID) -> case hb_cache:read(ID, Opts) of - {ok, Msg} -> - hb_message:id(Msg, uncommitted, Scoped); + {ok, Msg} -> hb_message:id(Msg, uncommitted, Scoped); not_found -> ID end end, diff --git a/src/hb_cache.erl b/src/hb_cache.erl index 43afcbeeb..85f9cad99 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -40,8 +40,8 @@ -module(hb_cache). -export([read_all_commitments/2]). -export([ensure_loaded/1, ensure_loaded/2, ensure_all_loaded/1, ensure_all_loaded/2]). --export([read/2, list/2, read_resolved/3, write/2, write_binary/3, write_hashpath/2, link/3]). --export([match/2, list_numbered/2]). +-export([read/2, read_resolved/3, write/2, write_binary/3, write_hashpath/2, link/3]). +-export([match/2, list/2, list_numbered/2]). -export([test_unsigned/1, test_signed/1]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -175,14 +175,24 @@ ensure_all_loaded(Ref, Msg, Opts) when is_list(Msg) -> ensure_all_loaded(Ref, Msg, Opts) -> ensure_loaded(Ref, Msg, Opts). -%% TODO: Remove before final PR, replace calls to `hb_store_common` -list(Path, Opts) -> - hb_store_common:list(Path, Opts). - %% @doc List all items in a directory, assuming they are numbered. list_numbered(Path, Opts) -> SlotDir = hb_store:path(hb_opts:get(store, no_viable_store, Opts), Path), - [ hb_util:int(Name) || Name <- hb_store_common:list(SlotDir, Opts) ]. + [ hb_util:int(Name) || Name <- list(SlotDir, Opts) ]. + +%% @doc List all items under a given path. +list(Path, Opts) when is_map(Opts) and not is_map_key(<<"store-module">>, Opts) -> + case hb_opts:get(store, no_viable_store, Opts) of + not_found -> []; + Store -> + list(Path, Store) + end; +list(Path, Store) -> + case hb_store_common:resolved_list(Store, Path) of + {ok, Names} -> Names; + {error, _} -> []; + not_found -> [] + end. %% @doc Match a template message against the cache, returning a list of IDs %% that match the template. We match on the binary representation of values, diff --git a/src/hb_store.erl b/src/hb_store.erl index 33c950495..5cb9e5527 100644 --- a/src/hb_store.erl +++ b/src/hb_store.erl @@ -384,7 +384,7 @@ apply_store_function(Mod, Store, Function, Args, AttemptsRemaining) -> retry -> retry(Mod, Store, Function, Args, AttemptsRemaining); Other -> Other catch Class:Reason:Stacktrace -> - ?event(error, + ?event(store_error, {store_call_failed_retrying, {store, Store}, {function, Function}, diff --git a/src/hb_store_common.erl b/src/hb_store_common.erl index 800986a8d..8da988dc9 100644 --- a/src/hb_store_common.erl +++ b/src/hb_store_common.erl @@ -3,37 +3,14 @@ %% Common store access patterns should be defined here. %% %% TODO: -%% - Test list/1 %% - Make tests for stores more generic, avoid individual tests (there are some in S3 that can be applied to LMDB). -module(hb_store_common). --export([list/2, resolve/2, store_read/3]). +-export([resolve/2, store_read/3]). -export([resolved_list/2, resolved_type/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% @doc List all items under a given path. -list(Path, Opts) when is_map(Opts) and not is_map_key(<<"store-module">>, Opts) -> - case hb_opts:get(store, no_viable_store, Opts) of - not_found -> []; - Store -> - list(Path, Store) - end; -list(Path, Store) when is_map(Store)-> - list(Path, [Store]); -list(_Path, []) -> - []; -list(Path, [Store | RemainingStores]) -> - ?event({list, Path}), - ResolvedPath = hb_store:resolve(Store, Path), - case hb_store:list(Store, ResolvedPath) of - {ok, Names} -> Names; - _ -> list(Path, RemainingStores) - end. - -%% NOTE: resolved_read/2 not needed because stores normally implement -%% resolve/2 inside read/2. - %% Old version, to be deleted before merge resolved_list2(Store, Path) -> ResolvedPath = hb_store:resolve(Store, Path), diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index e1ac110f9..3dcf11742 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -270,8 +270,6 @@ resolve_path_links(Opts, Path, Depth) -> %% Internal helper that accumulates the resolved path resolve_path_links_acc(_Opts, [], AccPath, _Depth) -> % No more segments to process - % TODO: Should this return not_found? - % Maybe not, not_found whould only be returned by `type`, `read`, etc. {ok, lists:reverse(AccPath)}; resolve_path_links_acc(_, FullPath = [<<"data">>|_], [], _Depth) -> {ok, FullPath}; @@ -368,11 +366,18 @@ list(Opts, Path) -> % Use native elmdb:list function #{ <<"db">> := DBInstance } = find_env(Opts), case elmdb:list(DBInstance, SearchPath) of - {ok, Children} -> {ok, Children}; - {error, not_found} -> compatibility_not_found(DBInstance, Path); % Normalize new error format - not_found -> compatibility_not_found(DBInstance, Path) % Handle both old and new format + {ok, Children} -> + {ok, Children}; + {error, not_found} -> + % Normalize new error format + compatibility_not_found(DBInstance, Path); + not_found -> + % Handle both old and new format + compatibility_not_found(DBInstance, Path) end. +%% By checking if is a group we can return if there is elements +%% or it wasn't found. compatibility_not_found(DBInstance, Path) -> case elmdb:get(DBInstance, remove_ending_slash(Path)) of {ok, <<"group">>} -> {ok, []}; @@ -381,11 +386,9 @@ compatibility_not_found(DBInstance, Path) -> remove_ending_slash(Path) -> list_to_binary( - string:reverse( - do_remove_ending_slash( - string:reverse(Path) - ) - )). + string:reverse( + do_remove_ending_slash( + string:reverse(Path)))). do_remove_ending_slash(["/" | Path]) -> Path; do_remove_ending_slash(Path) -> Path. From b57ebabcb590246370552dfcfe4deee132505cb5 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 5 Dec 2025 17:42:57 +0000 Subject: [PATCH 06/10] fix: Remove TODO, add as PR comment --- src/hb_cache.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hb_cache.erl b/src/hb_cache.erl index 85f9cad99..90f5369e8 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -406,7 +406,6 @@ do_read_commitment(Path, Opts) -> %% @doc Load all of the commitments for a message into memory. read_all_commitments(Msg, Opts) -> - %% TODO: Confirm tha this should only look into local stores only Store = hb_store:scope(hb_opts:get(store, no_viable_store, Opts), local), CurrentCommitments = hb_maps:get(<<"commitments">>, Msg, #{}, Opts), FoundCommitments = read_all_commitments_by_store(Msg, Store, Opts), From af78376ee53aee477d274f633c03e116e2ef5e9d Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 10 Dec 2025 08:13:27 +0000 Subject: [PATCH 07/10] fix: Improvements to code --- src/hb_store_common.erl | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/src/hb_store_common.erl b/src/hb_store_common.erl index 8da988dc9..61f81a068 100644 --- a/src/hb_store_common.erl +++ b/src/hb_store_common.erl @@ -1,21 +1,14 @@ %% @doc Store standard library %% -%% Common store access patterns should be defined here. -%% -%% TODO: -%% - Make tests for stores more generic, avoid individual tests (there are some in S3 that can be applied to LMDB). +%% Common patterns to access Stores (File System, LMDB, etc). -module(hb_store_common). --export([resolve/2, store_read/3]). +-export([store_read/3]). -export([resolved_list/2, resolved_type/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% Old version, to be deleted before merge -resolved_list2(Store, Path) -> - ResolvedPath = hb_store:resolve(Store, Path), - hb_store:list(Store, ResolvedPath). - +%% @doc Unify resolve and list functions into one call. resolved_list(Stores, Path) when is_list(Stores) -> do_resolved_list(Stores, Path); resolved_list(Store, Path) -> @@ -31,12 +24,7 @@ do_resolved_list([Store|RemainingStores], Path) -> not_found -> do_resolved_list(RemainingStores, Path) end. -%% Old version, to be deleted before merge -resolved_type2(Store, Path) -> - ResolvedPath = hb_store:resolve(Store, Path), - ?event({resolved_type, {path, Path}, {resolved_path, ResolvedPath}}), - hb_store:type(Store, ResolvedPath). - +%% @doc Unify resolve and type functions into one call. resolved_type(Stores, Path) when is_list(Stores) -> do_resolved_type(Stores, Path); resolved_type(Store, Path) -> @@ -51,19 +39,6 @@ do_resolved_type([Store|RemainingStores], Path) -> _ -> do_resolved_type(RemainingStores, Path) end. -%% TODO: This should replace the retry logic in the `hb_store:resolve` -resolve(Store, Opts) when not is_list(Store) -> - resolve([Store], Opts); -resolve(Stores, Path) -> - do_resolve(Stores, Path). - -do_resolve([], _Path) -> false; -do_resolve([Store|RemainingStores], Path) -> - case hb_store:type(Store, Path) of - not_found -> false; - _ -> {true, hb_store:resolve(RemainingStores, Path)} - end. - %% @doc List all of the subpaths of a given path and return a map of keys and %% links to the subpaths, including their types. store_read(Path, Store, Opts) -> From 8a7eedc5ef6211d2f1cbb2e50182c80bcf4842eb Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 11 Dec 2025 13:41:25 +0000 Subject: [PATCH 08/10] fix: Improve removing trailing slash --- src/hb_store_lmdb.erl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index 3dcf11742..8098f2217 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -385,13 +385,7 @@ compatibility_not_found(DBInstance, Path) -> end. remove_ending_slash(Path) -> - list_to_binary( - string:reverse( - do_remove_ending_slash( - string:reverse(Path)))). - -do_remove_ending_slash(["/" | Path]) -> Path; -do_remove_ending_slash(Path) -> Path. + list_to_binary(string:replace(Path, <<"/">>, <<"">>, trailing)). %% @doc Match a series of keys and values against the database. Returns %% `{ok, Matches}' if the match is successful, or `not_found' if there are no From f5a9700982e39f825cb9cf56236b68f6dc7cb1a9 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 11 Dec 2025 18:47:22 +0000 Subject: [PATCH 09/10] fix: Remove store_read from hb_store_common --- src/hb_cache.erl | 247 ++++++++++++++++++++++++++++++++++++++- src/hb_store_common.erl | 248 +--------------------------------------- 2 files changed, 248 insertions(+), 247 deletions(-) diff --git a/src/hb_cache.erl b/src/hb_cache.erl index 90f5369e8..c4c59fa03 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -395,14 +395,14 @@ write_binary(Hashpath, Bin, Store, Opts) -> %% richly typed map or a direct binary. read(Path, Opts) -> StoreReadResult = - hb_store_common:store_read(Path, hb_opts:get(store, no_viable_store, Opts), Opts), + store_read(Path, hb_opts:get(store, no_viable_store, Opts), Opts), case StoreReadResult of {ok, Res} -> {ok, hb_message:normalize_commitments(Res, Opts)}; _ -> StoreReadResult end. do_read_commitment(Path, Opts) -> - hb_store_common:store_read(Path, hb_opts:get(store, no_viable_store, Opts), Opts). + store_read(Path, hb_opts:get(store, no_viable_store, Opts), Opts). %% @doc Load all of the commitments for a message into memory. read_all_commitments(Msg, Opts) -> @@ -461,6 +461,230 @@ read_all_commitments_by_store(Msg, [Store | ReaminingStores], Opts) -> read_all_commitments_by_store(Msg, ReaminingStores, Opts) end. +%% @doc List all of the subpaths of a given path and return a map of keys and +%% links to the subpaths, including their types. +store_read(Path, Store, Opts) -> + store_read(Path, Path, Store, Opts). +store_read(_Target, _Path, no_viable_store, _) -> + not_found; +store_read(_Target, _Path, [], _) -> + not_found; +store_read(Target, Path, Store, Opts) when is_map(Store) -> + store_read(Target, Path, [Store], Opts); +store_read(Target, Path, [Store | RemainingStores], Opts) -> + ResolvedFullPath = hb_store:resolve(Store, PathBin = hb_path:to_binary(Path)), + ?event({reading, + {original_path, {string, PathBin}}, + {fully_resolved_path, ResolvedFullPath}, + {store, Store} + }), + ResolvedFullPathContent = case hb_store:type(Store, ResolvedFullPath) of + not_found -> not_found; + simple -> + ?event({reading_data, ResolvedFullPath}), + case hb_store:read(Store, ResolvedFullPath) of + {ok, Bin} -> {ok, Bin}; + not_found -> not_found + end; + composite -> + ?event({reading_composite, ResolvedFullPath}), + case hb_store:list(Store, ResolvedFullPath) of + {ok, RawSubpaths} -> + Subpaths = + lists:map(fun hb_util:bin/1, RawSubpaths), + ?event( + {listed, + {original_path, Path}, + {subpaths, {explicit, Subpaths}} + } + ), + % Generate links for each of the listed keys. We only list + % the target ID given in the case of multiple known + % commitments. + Msg = + prepare_links( + Target, + ResolvedFullPath, + Subpaths, + Store, + Opts + ), + ?event( + {completed_read, + {resolved_path, ResolvedFullPath}, + {explicit, Msg} + } + ), + {ok, Msg}; + not_found -> + ?event({empty_composite_message, ResolvedFullPath}), + {ok, #{}} + end + end, + case ResolvedFullPathContent of + {ok, _} = Response -> Response; + not_found -> store_read(Target, Path, RemainingStores, Opts) + end. + +%% @doc Prepare a set of links from a listing of subpaths. +prepare_links(Target, RootPath, Subpaths, Store, Opts) -> + {ok, Implicit, Types} = read_ao_types(RootPath, Subpaths, Store, Opts), + Res = + maps:from_list(lists:filtermap( + fun(<<"ao-types">>) -> false; + (<<"commitments">>) -> + % List the commitments for this message, and load them into + % memory. If there no commitments at the path, we exclude + % commitments from the list of links. + CommPath = + hb_store:resolve( + Store, + hb_store:path( + Store, + [ + RootPath, + <<"commitments">>, + Target + ] + ) + ), + ?event(read_commitment, + {reading_commitment, + {target, Target}, + {root_path, RootPath}, + {commitments_path, CommPath} + } + ), + case do_read_commitment(CommPath, Opts) of + {ok, Commitment} -> + LoadedCommitment = + ensure_all_loaded( + Commitment, + Opts#{ commitment => true } + ), + ?event(read_commitment, + {found_target_commitment, + {path, CommPath}, + {commitment, LoadedCommitment} + } + ), + % We have commitments, so we read each commitment + % into memory, and return it as part of the message. + { + true, + { + <<"commitments">>, + #{ Target => LoadedCommitment } + } + }; + _ -> + false + end; + (Subpath) -> + ?event( + {returning_link, + {subpath, Subpath} + } + ), + SubkeyPath = hb_store:path(Store, [RootPath, Subpath]), + case hb_link:is_link_key(Subpath) of + false -> + % The key is a literal value, not a nested composite + % message. Subsequently, we return a resolvable link + % to the subpath, leaving the key as-is. + {true, + { + Subpath, + {link, + SubkeyPath, + (case Types of + #{ Subpath := Type } -> + % We have an `ao-types' entry for the + % subpath, so we return a link to the + % subpath with `lazy' set to `true' + % because we need to resolve the link + % to get the final value. + #{ + <<"type">> => Type, + <<"lazy">> => true + }; + _ -> + % We do not have an `ao-types' entry for the + % subpath, so we return a link to the + % subpath with `lazy' set to `true', + % because the subpath is a literal + % value. + #{ + <<"lazy">> => true + } + end)#{ store => Store } + } + } + }; + true -> + % The key is an encoded link, so we create a resolvable + % link to the underlying link. This requires that we + % dereference the link twice in order to get the final + % value. Returning the data this way avoids having to + % read each of the link keys themselves, which may be + % a large quantity. + {true, + { + binary:part(Subpath, 0, byte_size(Subpath) - 5), + {link, SubkeyPath, #{ + <<"type">> => <<"link">>, + <<"lazy">> => true + }} + } + } + end + end, + Subpaths + )), + Merged = maps:merge(Res, Implicit), + % Convert the message to an ordered list if the ao-types indicate that it + % should be so. If it is a message, we ensure that the commitments are + % normalized (have an unsigned comm. ID) and loaded into memory. + case dev_codec_structured:is_list_from_ao_types(Types, Opts) of + true -> + hb_util:message_to_ordered_list(Merged, Opts); + false -> + case hb_opts:get(lazy_loading, true, Opts) of + true -> Merged; + false -> ensure_all_loaded(Merged, Opts) + end + end. + +%% @doc Read and parse the ao-types for a given path if it is in the supplied +%% list of subpaths, returning a map of keys and their types. +read_ao_types(Path, Subpaths, Store, Opts) -> + ?event({reading_ao_types, {path, Path}, {subpaths, {explicit, Subpaths}}}), + case lists:member(<<"ao-types">>, Subpaths) of + true -> + {ok, TypesBin} = + hb_store:read( + Store, + hb_store:path(Store, [Path, <<"ao-types">>]) + ), + Types = dev_codec_structured:decode_ao_types(TypesBin, Opts), + ?event({parsed_ao_types, {types, Types}}), + {ok, types_to_implicit(Types), Types}; + false -> + ?event({no_ao_types_key_found, {path, Path}, {subpaths, Subpaths}}), + {ok, #{}, #{}} + end. + +%% @doc Convert a map of ao-types to an implicit map of types. +types_to_implicit(Types) -> + maps:filtermap( + fun(_K, <<"empty-message">>) -> {true, #{}}; + (_K, <<"empty-list">>) -> {true, []}; + (_K, <<"empty-binary">>) -> {true, <<>>}; + (_, _) -> false + end, + Types + ). + %% @doc Read the result of a computation, using heuristics. The supported %% heuristics are as follows: %% 1. If the base message is an ID, we try to determine if the message has an @@ -874,3 +1098,22 @@ test_device_map_cannot_be_written_test() -> run_test() -> Store = hb_test_utils:test_store(hb_store_lmdb), test_match_typed_message(Store). + +%% @doc Read value from Store1 and Store2 when is only available in Store2 +multiple_stores_store_read_test() -> + [_Store1, Store2] = Stores = hb_store_common:get_multiple_stores(), + %% Write test data + hb_store:make_group(Store2, <<"group1">>), + hb_store:write(Store2, <<"data/final_id">>, <<"data">>), + hb_store:make_link(Store2, <<"data/final_id">>, <<"group1/data">>), + hb_store:make_link(Store2, <<"group1">>, <<"random_id">>), + %% Check result + Opts = #{}, + Path = <<"random_id">>, + Content = store_read(Path, Stores, Opts), + try + ?assertMatch({ok, #{<<"data">> := _}}, Content) + after + hb_store_common:shutdown_stores(Stores) + end. + diff --git a/src/hb_store_common.erl b/src/hb_store_common.erl index 61f81a068..36e08e8e6 100644 --- a/src/hb_store_common.erl +++ b/src/hb_store_common.erl @@ -3,8 +3,8 @@ %% Common patterns to access Stores (File System, LMDB, etc). -module(hb_store_common). --export([store_read/3]). -export([resolved_list/2, resolved_type/2]). +-export([get_multiple_stores/0, shutdown_stores/1]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -39,252 +39,8 @@ do_resolved_type([Store|RemainingStores], Path) -> _ -> do_resolved_type(RemainingStores, Path) end. -%% @doc List all of the subpaths of a given path and return a map of keys and -%% links to the subpaths, including their types. -store_read(Path, Store, Opts) -> - store_read(Path, Path, Store, Opts). -store_read(_Target, _Path, no_viable_store, _) -> - not_found; -store_read(_Target, _Path, [], _) -> - not_found; -store_read(Target, Path, Store, Opts) when is_map(Store) -> - store_read(Target, Path, [Store], Opts); -store_read(Target, Path, [Store | RemainingStores], Opts) -> - ResolvedFullPath = hb_store:resolve(Store, PathBin = hb_path:to_binary(Path)), - ?event({reading, - {original_path, {string, PathBin}}, - {fully_resolved_path, ResolvedFullPath}, - {store, Store} - }), - ResolvedFullPathContent = case hb_store:type(Store, ResolvedFullPath) of - not_found -> not_found; - simple -> - ?event({reading_data, ResolvedFullPath}), - case hb_store:read(Store, ResolvedFullPath) of - {ok, Bin} -> {ok, Bin}; - not_found -> not_found - end; - composite -> - ?event({reading_composite, ResolvedFullPath}), - case hb_store:list(Store, ResolvedFullPath) of - {ok, RawSubpaths} -> - Subpaths = - lists:map(fun hb_util:bin/1, RawSubpaths), - ?event( - {listed, - {original_path, Path}, - {subpaths, {explicit, Subpaths}} - } - ), - % Generate links for each of the listed keys. We only list - % the target ID given in the case of multiple known - % commitments. - Msg = - prepare_links( - Target, - ResolvedFullPath, - Subpaths, - Store, - Opts - ), - ?event( - {completed_read, - {resolved_path, ResolvedFullPath}, - {explicit, Msg} - } - ), - {ok, Msg}; - not_found -> - ?event({empty_composite_message, ResolvedFullPath}), - {ok, #{}} - end - end, - case ResolvedFullPathContent of - {ok, _} = Response -> Response; - not_found -> store_read(Target, Path, RemainingStores, Opts) - end. - -%% @doc Prepare a set of links from a listing of subpaths. -prepare_links(Target, RootPath, Subpaths, Store, Opts) -> - {ok, Implicit, Types} = read_ao_types(RootPath, Subpaths, Store, Opts), - Res = - maps:from_list(lists:filtermap( - fun(<<"ao-types">>) -> false; - (<<"commitments">>) -> - % List the commitments for this message, and load them into - % memory. If there no commitments at the path, we exclude - % commitments from the list of links. - CommPath = - hb_store:resolve( - Store, - hb_store:path( - Store, - [ - RootPath, - <<"commitments">>, - Target - ] - ) - ), - ?event(read_commitment, - {reading_commitment, - {target, Target}, - {root_path, RootPath}, - {commitments_path, CommPath} - } - ), - %% TODO: Maybe improve this? This line bellow is called in - %% hb_cache:do_read_commitment - case store_read(CommPath, hb_opts:get(store, no_viable_store, Opts), Opts) of - {ok, Commitment} -> - LoadedCommitment = - hb_cache:ensure_all_loaded( - Commitment, - Opts#{ commitment => true } - ), - ?event(read_commitment, - {found_target_commitment, - {path, CommPath}, - {commitment, LoadedCommitment} - } - ), - % We have commitments, so we read each commitment - % into memory, and return it as part of the message. - { - true, - { - <<"commitments">>, - #{ Target => LoadedCommitment } - } - }; - _ -> - false - end; - (Subpath) -> - ?event( - {returning_link, - {subpath, Subpath} - } - ), - SubkeyPath = hb_store:path(Store, [RootPath, Subpath]), - case hb_link:is_link_key(Subpath) of - false -> - % The key is a literal value, not a nested composite - % message. Subsequently, we return a resolvable link - % to the subpath, leaving the key as-is. - {true, - { - Subpath, - {link, - SubkeyPath, - (case Types of - #{ Subpath := Type } -> - % We have an `ao-types' entry for the - % subpath, so we return a link to the - % subpath with `lazy' set to `true' - % because we need to resolve the link - % to get the final value. - #{ - <<"type">> => Type, - <<"lazy">> => true - }; - _ -> - % We do not have an `ao-types' entry for the - % subpath, so we return a link to the - % subpath with `lazy' set to `true', - % because the subpath is a literal - % value. - #{ - <<"lazy">> => true - } - end)#{ store => Store } - } - } - }; - true -> - % The key is an encoded link, so we create a resolvable - % link to the underlying link. This requires that we - % dereference the link twice in order to get the final - % value. Returning the data this way avoids having to - % read each of the link keys themselves, which may be - % a large quantity. - {true, - { - binary:part(Subpath, 0, byte_size(Subpath) - 5), - {link, SubkeyPath, #{ - <<"type">> => <<"link">>, - <<"lazy">> => true - }} - } - } - end - end, - Subpaths - )), - Merged = maps:merge(Res, Implicit), - % Convert the message to an ordered list if the ao-types indicate that it - % should be so. If it is a message, we ensure that the commitments are - % normalized (have an unsigned comm. ID) and loaded into memory. - case dev_codec_structured:is_list_from_ao_types(Types, Opts) of - true -> - hb_util:message_to_ordered_list(Merged, Opts); - false -> - case hb_opts:get(lazy_loading, true, Opts) of - true -> Merged; - false -> hb_cache:ensure_all_loaded(Merged, Opts) - end - end. - -%% @doc Read and parse the ao-types for a given path if it is in the supplied -%% list of subpaths, returning a map of keys and their types. -read_ao_types(Path, Subpaths, Store, Opts) -> - ?event({reading_ao_types, {path, Path}, {subpaths, {explicit, Subpaths}}}), - case lists:member(<<"ao-types">>, Subpaths) of - true -> - {ok, TypesBin} = - hb_store:read( - Store, - hb_store:path(Store, [Path, <<"ao-types">>]) - ), - Types = dev_codec_structured:decode_ao_types(TypesBin, Opts), - ?event({parsed_ao_types, {types, Types}}), - {ok, types_to_implicit(Types), Types}; - false -> - ?event({no_ao_types_key_found, {path, Path}, {subpaths, Subpaths}}), - {ok, #{}, #{}} - end. - -%% @doc Convert a map of ao-types to an implicit map of types. -types_to_implicit(Types) -> - maps:filtermap( - fun(_K, <<"empty-message">>) -> {true, #{}}; - (_K, <<"empty-list">>) -> {true, []}; - (_K, <<"empty-binary">>) -> {true, <<>>}; - (_, _) -> false - end, - Types - ). - %% Tests -%% @doc Read value from Store1 and Store2 when is only available in Store2 -multiple_stores_store_read_test() -> - [_Store1, Store2] = Stores = get_multiple_stores(), - %% Write test data - hb_store:make_group(Store2, <<"group1">>), - hb_store:write(Store2, <<"data/final_id">>, <<"data">>), - hb_store:make_link(Store2, <<"data/final_id">>, <<"group1/data">>), - hb_store:make_link(Store2, <<"group1">>, <<"random_id">>), - %% Check result - Opts = #{}, - Path = <<"random_id">>, - Content = store_read(Path, Stores, Opts), - try - ?assertMatch({ok, #{<<"data">> := _}}, Content) - after - shutdown_stores(Stores) - end. - %% @doc Test that resolve and type must be made in the same store, %% when multiple stores are provided. resolved_type_test() -> @@ -332,6 +88,8 @@ get_multiple_stores() -> get_multiple_stores(StoreModule) -> Store1 = hb_test_utils:test_store(StoreModule, <<"store1">>), Store2 = hb_test_utils:test_store(StoreModule, <<"store2">>), + hb_store:reset(Store1), + hb_store:reset(Store2), [Store1, Store2]. %% @doc Shutdown multiple stores From 9e8da396747c95edb2b11ef7c6a1214f2f37c833 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 26 Dec 2025 16:24:50 +0000 Subject: [PATCH 10/10] impr: Remove TODO --- src/hb_store_common.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hb_store_common.erl b/src/hb_store_common.erl index 36e08e8e6..9903ab193 100644 --- a/src/hb_store_common.erl +++ b/src/hb_store_common.erl @@ -68,7 +68,6 @@ resolved_list_test() -> hb_store:make_group(Store2, <<"group1/group12">>), hb_store:write(Store2, <<"data/final_id2">>, <<"7890">>), %% Link - %% TODO: Not sure if this structure is possible in HB hb_store:make_link(Store2, <<"data/final_id2">>, <<"group1/group12/data">>), hb_store:make_link(Store2, <<"group1">>, <<"random_id">>), %% Check result