diff --git a/src/dev_process_cache.erl b/src/dev_process_cache.erl index 0dd96bbbf..9b77718d4 100644 --- a/src/dev_process_cache.erl +++ b/src/dev_process_cache.erl @@ -136,9 +136,8 @@ first_with_path(_ProcID, _Required, [], _Opts, _Store) -> not_found; first_with_path(ProcID, RequiredPath, [Slot | Rest], Opts, Store) -> RawPath = path(ProcID, Slot, RequiredPath, Opts), - ResolvedPath = hb_store:resolve(Store, RawPath), - ?event({trying_slot, {slot, Slot}, {path, RawPath}, {resolved_path, ResolvedPath}}), - case hb_store:type(Store, ResolvedPath) of + ?event({trying_slot, {slot, Slot}, {path, RawPath}}), + case hb_store_common:resolved_type(Store, RawPath) of not_found -> first_with_path(ProcID, RequiredPath, Rest, Opts, Store); _ -> diff --git a/src/dev_query_arweave.erl b/src/dev_query_arweave.erl index 19ded4210..9d2ae1abc 100644 --- a/src/dev_query_arweave.erl +++ b/src/dev_query_arweave.erl @@ -297,9 +297,11 @@ commitment_id_to_base_id(ID, Opts) -> all_ids(ID, Opts) -> Store = hb_opts:get(store, no_store, Opts), case hb_store:list(Store, << ID/binary, "/commitments">>) of - {ok, []} -> [ID]; - {ok, CommitmentIDs} -> CommitmentIDs; - _ -> [] + {ok, CommitmentIDs} -> + CommitmentIDs; + not_found -> + %% Add ID to fetch as a message + [ID] end. %% @doc Scope the stores used for block matching. The searched stores can be diff --git a/src/hb_cache.erl b/src/hb_cache.erl index a0d0b7a80..c4c59fa03 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -188,8 +188,7 @@ list(Path, Opts) when is_map(Opts) and not is_map_key(<<"store-module">>, Opts) list(Path, Store) end; list(Path, Store) -> - ResolvedPath = hb_store:resolve(Store, Path), - case hb_store:list(Store, ResolvedPath) of + case hb_store_common:resolved_list(Store, Path) of {ok, Names} -> Names; {error, _} -> []; not_found -> [] @@ -407,67 +406,79 @@ do_read_commitment(Path, Opts) -> %% @doc Load all of the commitments for a message into memory. read_all_commitments(Msg, Opts) -> - Store = hb_opts:get(store, no_viable_store, Opts), - UncommittedID = hb_message:id(Msg, none, Opts#{ linkify_mode => discard }), + Store = hb_store:scope(hb_opts:get(store, no_viable_store, Opts), local), + CurrentCommitments = hb_maps:get(<<"commitments">>, Msg, #{}, Opts), + FoundCommitments = read_all_commitments_by_store(Msg, Store, Opts), + NewCommitments = + hb_maps:merge( + CurrentCommitments, + maps:from_list(FoundCommitments) + ), + Msg#{ <<"commitments">> => NewCommitments }. + +read_all_commitments_by_store(Msg, Store, Opts) when not is_list(Store) -> + read_all_commitments_by_store(Msg, [Store], Opts); +read_all_commitments_by_store(_Msg, [], _Opts) -> + []; +read_all_commitments_by_store(Msg, [Store | ReaminingStores], Opts) -> CurrentCommitments = hb_maps:get(<<"commitments">>, Msg, #{}, Opts), AlreadyLoaded = hb_maps:keys(CurrentCommitments, Opts), + UncommittedID = hb_message:id(Msg, none, Opts#{ linkify_mode => discard }), CommitmentsPath = hb_store:resolve( Store, hb_store:path(Store, [UncommittedID, <<"commitments">>]) ), - FoundCommitments = - case hb_store:list(Store, CommitmentsPath) of - {ok, CommitmentIDs} -> - lists:filtermap( - fun(CommitmentID) -> - ShouldLoad = not lists:member(CommitmentID, AlreadyLoaded), - ResolvedCommPath = - hb_store:path( - Store, - [CommitmentsPath, CommitmentID] - ), - case ShouldLoad andalso do_read_commitment(ResolvedCommPath, Opts) of - {ok, Commitment} -> + case hb_store:list(Store, CommitmentsPath) of + {ok, CommitmentIDs} -> + lists:filtermap( + fun(CommitmentID) -> + ShouldLoad = not lists:member(CommitmentID, AlreadyLoaded), + ResolvedCommPath = + hb_store:path( + Store, + [CommitmentsPath, CommitmentID] + ), + case ShouldLoad andalso do_read_commitment(ResolvedCommPath, Opts#{store => Store}) of + {ok, Commitment} -> + { + true, { - true, - { - CommitmentID, - ensure_all_loaded( - Commitment, - Opts#{ commitment => true } - ) - } - }; - _ -> - false - end - end, - CommitmentIDs - ); - not_found -> - [] - end, - NewCommitments = - hb_maps:merge( - CurrentCommitments, - maps:from_list(FoundCommitments) - ), - Msg#{ <<"commitments">> => NewCommitments }. + CommitmentID, + ensure_all_loaded( + Commitment, + Opts#{ commitment => true } + ) + } + }; + _ -> + false + end + end, + CommitmentIDs + ); + not_found -> + read_all_commitments_by_store(Msg, ReaminingStores, Opts) + end. + %% @doc List all of the subpaths of a given path and return a map of keys and %% links to the subpaths, including their types. store_read(Path, Store, Opts) -> store_read(Path, Path, Store, Opts). store_read(_Target, _Path, no_viable_store, _) -> not_found; -store_read(Target, Path, Store, Opts) -> +store_read(_Target, _Path, [], _) -> + not_found; +store_read(Target, Path, Store, Opts) when is_map(Store) -> + store_read(Target, Path, [Store], Opts); +store_read(Target, Path, [Store | RemainingStores], Opts) -> ResolvedFullPath = hb_store:resolve(Store, PathBin = hb_path:to_binary(Path)), ?event({reading, {original_path, {string, PathBin}}, {fully_resolved_path, ResolvedFullPath}, {store, Store} }), - case hb_store:type(Store, ResolvedFullPath) of + ResolvedFullPathContent = case hb_store:type(Store, ResolvedFullPath) of not_found -> not_found; simple -> ?event({reading_data, ResolvedFullPath}), @@ -505,10 +516,14 @@ store_read(Target, Path, Store, Opts) -> } ), {ok, Msg}; - _ -> + not_found -> ?event({empty_composite_message, ResolvedFullPath}), {ok, #{}} end + end, + case ResolvedFullPathContent of + {ok, _} = Response -> Response; + not_found -> store_read(Target, Path, RemainingStores, Opts) end. %% @doc Prepare a set of links from a listing of subpaths. @@ -1050,6 +1065,7 @@ test_match_typed_message(Store) -> cache_suite_test_() -> hb_store:generate_test_suite([ + {"store ans104 message", fun test_store_ans104_message/1}, {"store unsigned empty message", fun test_store_unsigned_empty_message/1}, {"store binary", fun test_store_binary/1}, @@ -1082,3 +1098,22 @@ test_device_map_cannot_be_written_test() -> run_test() -> Store = hb_test_utils:test_store(hb_store_lmdb), test_match_typed_message(Store). + +%% @doc Read value from Store1 and Store2 when is only available in Store2 +multiple_stores_store_read_test() -> + [_Store1, Store2] = Stores = hb_store_common:get_multiple_stores(), + %% Write test data + hb_store:make_group(Store2, <<"group1">>), + hb_store:write(Store2, <<"data/final_id">>, <<"data">>), + hb_store:make_link(Store2, <<"data/final_id">>, <<"group1/data">>), + hb_store:make_link(Store2, <<"group1">>, <<"random_id">>), + %% Check result + Opts = #{}, + Path = <<"random_id">>, + Content = store_read(Path, Stores, Opts), + try + ?assertMatch({ok, #{<<"data">> := _}}, Content) + after + hb_store_common:shutdown_stores(Stores) + end. + diff --git a/src/hb_store_common.erl b/src/hb_store_common.erl new file mode 100644 index 000000000..9903ab193 --- /dev/null +++ b/src/hb_store_common.erl @@ -0,0 +1,99 @@ +%% @doc Store standard library +%% +%% Common patterns to access Stores (File System, LMDB, etc). + +-module(hb_store_common). +-export([resolved_list/2, resolved_type/2]). +-export([get_multiple_stores/0, shutdown_stores/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Unify resolve and list functions into one call. +resolved_list(Stores, Path) when is_list(Stores) -> + do_resolved_list(Stores, Path); +resolved_list(Store, Path) -> + do_resolved_list([Store], Path). + +do_resolved_list([], _Path) -> + not_found; +do_resolved_list([Store|RemainingStores], Path) -> + ResolvedPath = hb_store:resolve(Store, Path), + ?event({resolved_list, {path, Path}, {resolved_path, ResolvedPath}}), + case hb_store:list(Store, ResolvedPath) of + {ok, _} = Result -> Result; + not_found -> do_resolved_list(RemainingStores, Path) + end. + +%% @doc Unify resolve and type functions into one call. +resolved_type(Stores, Path) when is_list(Stores) -> + do_resolved_type(Stores, Path); +resolved_type(Store, Path) -> + do_resolved_type([Store], Path). + +do_resolved_type([], _Path) -> not_found; +do_resolved_type([Store|RemainingStores], Path) -> + ResolvedPath = hb_store:resolve(Store, Path), + ?event({resolved_type, {path, Path}, {resolved_path, ResolvedPath}}), + case hb_store:type(Store, ResolvedPath) of + Result when Result =/= not_found -> Result; + _ -> do_resolved_type(RemainingStores, Path) + end. + +%% Tests + +%% @doc Test that resolve and type must be made in the same store, +%% when multiple stores are provided. +resolved_type_test() -> + [_Store1, Store2] = Stores = get_multiple_stores(), + %% Write test data + hb_store:make_group(Store2, <<"group1">>), + hb_store:write(Store2, <<"data/final_id">>, <<"data">>), + hb_store:make_link(Store2, <<"data/final_id">>, <<"group1/data">>), + hb_store:make_link(Store2, <<"group1">>, <<"random_id">>), + %% Check result + RawPath = <<"random_id/data">>, + Result = resolved_type(Stores, RawPath), + try + ?assertEqual(simple, Result) + after + shutdown_stores(Stores) + end. + +%% @doc Test that resolve and list must be made in the same store, +%% when multiple stores are provided. +resolved_list_test() -> + [_Store1, Store2] = Stores = get_multiple_stores(), + %% Write test data + hb_store:make_group(Store2, <<"group1">>), + hb_store:make_group(Store2, <<"group1/group12">>), + hb_store:write(Store2, <<"data/final_id2">>, <<"7890">>), + %% Link + hb_store:make_link(Store2, <<"data/final_id2">>, <<"group1/group12/data">>), + hb_store:make_link(Store2, <<"group1">>, <<"random_id">>), + %% Check result + RawPath = <<"random_id/group12">>, + Result = resolved_list(Stores, RawPath), + try + ?assertEqual({ok, [<<"data">>]}, Result) + after + shutdown_stores(Stores) + end. + +%% Test utilities + +%% @doc Initialize multiple stores +get_multiple_stores() -> + get_multiple_stores(hb_store_lmdb). +get_multiple_stores(StoreModule) -> + Store1 = hb_test_utils:test_store(StoreModule, <<"store1">>), + Store2 = hb_test_utils:test_store(StoreModule, <<"store2">>), + hb_store:reset(Store1), + hb_store:reset(Store2), + [Store1, Store2]. + +%% @doc Shutdown multiple stores +shutdown_stores([]) -> ok; +shutdown_stores([Store | RemainingStores]) -> + hb_store:reset(Store), + hb_store:stop(Store), + shutdown_stores(RemainingStores). diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index 0a970513c..8098f2217 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -366,11 +366,27 @@ list(Opts, Path) -> % Use native elmdb:list function #{ <<"db">> := DBInstance } = find_env(Opts), case elmdb:list(DBInstance, SearchPath) of - {ok, Children} -> {ok, Children}; - {error, not_found} -> {ok, []}; % Normalize new error format - not_found -> {ok, []} % Handle both old and new format + {ok, Children} -> + {ok, Children}; + {error, not_found} -> + % Normalize new error format + compatibility_not_found(DBInstance, Path); + not_found -> + % Handle both old and new format + compatibility_not_found(DBInstance, Path) end. +%% By checking if is a group we can return if there is elements +%% or it wasn't found. +compatibility_not_found(DBInstance, Path) -> + case elmdb:get(DBInstance, remove_ending_slash(Path)) of + {ok, <<"group">>} -> {ok, []}; + _ -> not_found + end. + +remove_ending_slash(Path) -> + list_to_binary(string:replace(Path, <<"/">>, <<"">>, trailing)). + %% @doc Match a series of keys and values against the database. Returns %% `{ok, Matches}' if the match is successful, or `not_found' if there are no %% messages in the store that feature all of the given key-value pairs. `Matches' @@ -661,7 +677,7 @@ list_test() -> <<"capacity">> => ?DEFAULT_SIZE }, reset(StoreOpts), - ?assertEqual(list(StoreOpts, <<"colors">>), {ok, []}), + ?assertEqual(list(StoreOpts, <<"colors">>), not_found), % Create immediate children under colors/ write(StoreOpts, <<"colors/red">>, <<"1">>), write(StoreOpts, <<"colors/blue">>, <<"2">>), diff --git a/src/hb_store_lru.erl b/src/hb_store_lru.erl index d68aaa8f7..02acb5817 100644 --- a/src/hb_store_lru.erl +++ b/src/hb_store_lru.erl @@ -242,8 +242,7 @@ list(Opts, Path) -> no_store -> not_found; Store -> - ResolvedPath = hb_store:resolve(Store, Path), - case hb_store:list(Store, ResolvedPath) of + case hb_store_common:resolved_list(Store, Path) of {ok, Keys} -> Keys; not_found -> not_found end @@ -287,8 +286,7 @@ type(Opts, Key) -> no_store -> not_found; Store -> - ResolvedKey = hb_store:resolve(Store, Key), - hb_store:type(Store, ResolvedKey) + hb_store_common:resolved_type(Store, Key) end; {raw, _} -> simple;