Add a whole ncurses TUI and fix a delete logic bug

apps/revault/src/revault_data_wrapper.erl

@@ -11,7 +11,8 @@
 -module(revault_data_wrapper).
 -export([peer/1, peer/2, new/0, ask/0, ok/0, error/1, fork/2]).
 -export([manifest/0, manifest/1,
-         send_file/4, send_multipart_file/6, send_deleted/2,
+         send_file/4, send_multipart_file/6,
+         send_deleted/2, send_conflict_deleted/3,
          send_conflict_file/5, send_conflict_multipart_file/7, fetch_file/1,
          sync_complete/0]).
 
@@ -73,6 +74,9 @@ send_multipart_file(Path, Vsn, Hash, M, N, Bin) when M >= 1, M =< N ->
 send_deleted(Path, Vsn) ->
     {deleted_file, ?VSN, Path, {Vsn, deleted}}.
 
+send_conflict_deleted(WorkPath, ConflictsLeft, Meta) ->
+    {conflict_file, ?VSN, WorkPath, deleted, ConflictsLeft, Meta}.
+
 send_conflict_file(WorkPath, Path, ConflictsLeft, Meta, Bin) ->
     {conflict_file, ?VSN, WorkPath, Path, ConflictsLeft, Meta, Bin}.
 

apps/revault/src/revault_data_wrapper.hrl

@@ -1,4 +1,4 @@
 % VSN 1: initial protocol
 % VSN 2: adds multipart file transfers
 % TODO: add test about protocol compatibility
--define(VSN, 2).
+-define(VSN, 3).

apps/revault/src/revault_dirmon_tracker.erl

@@ -151,6 +151,14 @@ handle_call({conflict, Work, {NewStamp, deleted}}, _From,
             %% but note the deletion stamp as part of the conflict.
             CStamp = conflict_stamp(Id, Stamp, NewStamp),
             {CStamp, {conflict, ConflictHashes, WorkingHash}};
+        #{Work := {Stamp, deleted}} ->
+            %% This is a special case similar to having both files diverging
+            %% in stamps but having the same "hash" or value by virtue of being
+            %% deleted. Create an empty conflict file, assume further files might
+            %% come in as part of the sync or that this will properly carry
+            %% the state moving forward.
+            CStamp = conflict_stamp(Id, Stamp, NewStamp),
+            {CStamp, {conflict, [], deleted}};
         #{Work := {Stamp, WorkingHash}} ->
             %% No conflict, create it
             ConflictingWork = revault_conflict_file:conflicting(Work, WorkingHash),
@@ -374,8 +382,10 @@ conflict_marker(Dir, WorkingFile) ->
 write_conflict_marker(Dir, WorkingFile, {_, {conflict, Hashes, _}}) ->
     %% We don't care about the rename trick here, it's informational
     %% but all the critical data is tracked in the snapshot
+    F = conflict_marker(Dir, WorkingFile),
+    revault_file:ensure_dir(F),
     revault_file:write_file(
-        conflict_marker(Dir, WorkingFile),
+        F,
         lists:join($\n, [revault_conflict_file:hex(Hash) || Hash <- Hashes])
     ).
 

apps/revault/src/revault_disterl.erl

@@ -83,6 +83,8 @@ unpack({file, ?VSN, Path, Meta, PartNum, PartTotal, Bin}) -> {file, Path, Meta,
 unpack({fetch, ?VSN, Path}) -> {fetch, Path};
 unpack({sync_complete, ?VSN}) -> sync_complete;
 unpack({deleted_file, ?VSN, Path, Meta}) -> {deleted_file, Path, Meta};
+unpack({conflict_file, ?VSN, WorkPath, deleted, Count, Meta}) ->
+    {conflict_file, WorkPath, deleted, Count, Meta};
 unpack({conflict_file, ?VSN, WorkPath, Path, Count, Meta, Bin}) ->
     {conflict_file, WorkPath, Path, Count, Meta, Bin};
 unpack({conflict_multipart_file, ?VSN, WorkPath, Path, Count, Meta, PartNum, PartTotal, Bin}) ->

apps/revault/src/revault_fsm.erl

@@ -637,6 +637,25 @@ client_sync_files(info, {revault, _Marker, {deleted_file, F, Meta}}, Data) ->
     NewQ = send_next_scheduled(Q),
     NewAcc = Acc -- [F],
     {keep_state, Data#data{scan=true, sub=S#client_sync{queue=NewQ, acc=NewAcc}}};
+client_sync_files(info, {revault, _Marker, {conflict_file, WorkF, deleted, CountLeft, Meta}}, Data) ->
+        #data{name=Name, sub=S=#client_sync{queue=Q, acc=Acc}} = Data,
+        ?with_span(
+            <<"conflict">>,
+            #{attributes => [{<<"path">>, WorkF}, {<<"meta">>, ?str(Meta)},
+                            {<<"count">>, CountLeft} | ?attrs(Data)]},
+            fun(_SpanCtx) ->
+                %% TODO: handle the file being corrupted vs its own hash
+                revault_dirmon_tracker:conflict(Name, WorkF, Meta)
+            end
+        ),
+        case CountLeft =:= 0 andalso Acc -- [WorkF] of
+            false ->
+                %% more of the same conflict file to come
+                {keep_state, Data#data{scan=true}};
+            NewAcc ->
+                NewQ = send_next_scheduled(Q),
+                {keep_state, Data#data{scan=true, sub=S#client_sync{queue=NewQ, acc=NewAcc}}}
+        end;
 client_sync_files(info, {revault, _Marker, {conflict_file, WorkF, F, CountLeft, Meta, Bin}}, Data) ->
     #data{name=Name, sub=S=#client_sync{queue=Q, acc=Acc}} = Data,
     ?with_span(
@@ -854,6 +873,16 @@ server_sync_files(info, {revault, _Marker, {deleted_file, F, Meta}},
                                                | ?attrs(Data)]},
                fun(_SpanCtx) -> handle_delete_sync(Name, Id, F, Meta) end),
     {keep_state, Data#data{scan=true}};
+server_sync_files(info, {revault, _M, {conflict_file, WorkF, deleted, CountLeft, Meta}}, Data) ->
+    ?with_span(
+        <<"conflict">>,
+        #{attributes => [{<<"path">>, WorkF}, {<<"meta">>, ?str(Meta)},
+                        {<<"count">>, CountLeft} | ?attrs(Data)]},
+        fun(_SpanCtx) ->
+            revault_dirmon_tracker:conflict(Data#data.name, WorkF, Meta)
+        end
+    ),
+    {keep_state, Data#data{scan=true}};
 server_sync_files(info, {revault, _M, {conflict_file, WorkF, F, CountLeft, Meta, Bin}}, Data) ->
     %% TODO: handle the file being corrupted vs its own hash
     ?with_span(
@@ -1202,6 +1231,11 @@ file_transfer_schedule(Name, Path, File) ->
     case revault_dirmon_tracker:file(Name, File) of
         {Vsn, deleted} ->
             [{deleted, File, Vsn}];
+        {Vsn, {conflict, [], deleted}} ->
+            %% Special deletion case where clashing deleted files
+            %% exist; there's no hash to send here, and no FHash;
+            %% explicitly call it as deleted.
+            [{conflict_file, File, deleted, 0, {Vsn, deleted}}];
         {Vsn, {conflict, Hashes, _}} ->
             {List, _} = lists:foldl(
                 fun(Hash, {Acc, Ct}) ->
@@ -1231,6 +1265,11 @@ wrap(_Path, {deleted, File, Vsn}) ->
     ?set_attribute(<<"path">>, File),
     ?set_attribute(<<"transfer_type">>, <<"deleted">>),
     revault_data_wrapper:send_deleted(File, Vsn);
+wrap(_Path, {conflict_file, File, deleted, Ct, Meta}) ->
+    ?set_attribute(<<"path">>, deleted),
+    ?set_attribute(<<"transfer_type">>, <<"conflict_file">>),
+    ?set_attribute(<<"conflict.ct">>, Ct),
+    revault_data_wrapper:send_conflict_deleted(File, Ct, Meta);
 wrap(Path, {conflict_file, File, FHash, Ct, Meta}) ->
     ?set_attribute(<<"path">>, FHash),
     ?set_attribute(<<"transfer_type">>, <<"conflict_file">>),
@@ -1303,4 +1342,3 @@ pid_attrs() ->
        proplists:get_value(minor_gcs,
                            proplists:get_value(garbage_collection, PidInfo))}
     ].
-

apps/revault/src/revault_tcp.erl

@@ -134,6 +134,8 @@ unpack({file, ?VSN, Path, Meta, PartNum, PartTotal, Bin}) -> {file, Path, Meta,
 unpack({fetch, ?VSN, Path}) -> {fetch, Path};
 unpack({sync_complete, ?VSN}) -> sync_complete;
 unpack({deleted_file, ?VSN, Path, Meta}) -> {deleted_file, Path, Meta};
+unpack({conflict_file, ?VSN, WorkPath, deleted, Count, Meta}) ->
+    {conflict_file, WorkPath, deleted, Count, Meta};
 unpack({conflict_file, ?VSN, WorkPath, Path, Count, Meta, Bin}) ->
     {conflict_file, WorkPath, Path, Count, Meta, Bin};
 unpack({conflict_multipart_file, ?VSN, WorkPath, Path, Count, Meta, PartNum, PartTotal, Bin}) ->

apps/revault/src/revault_tls.erl

@@ -138,6 +138,8 @@ unpack({file, ?VSN, Path, Meta, PartNum, PartTotal, Bin}) -> {file, Path, Meta,
 unpack({fetch, ?VSN, Path}) -> {fetch, Path};
 unpack({sync_complete, ?VSN}) -> sync_complete;
 unpack({deleted_file, ?VSN, Path, Meta}) -> {deleted_file, Path, Meta};
+unpack({conflict_file, ?VSN, WorkPath, deleted, Count, Meta}) ->
+    {conflict_file, WorkPath, deleted, Count, Meta};
 unpack({conflict_file, ?VSN, WorkPath, Path, Count, Meta, Bin}) ->
     {conflict_file, WorkPath, Path, Count, Meta, Bin};
 unpack({conflict_multipart_file, ?VSN, WorkPath, Path, Count, Meta, PartNum, PartTotal, Bin}) ->

apps/revault/test/revault_fsm_SUITE.erl

@@ -22,6 +22,7 @@ groups() ->
                   fork_server_save, seed_fork, basic_sync,
                   delete_sync, too_many_clients,
                   overwrite_sync_clash, conflict_sync,
+                  delete_sync_conflict,
                   prevent_server_clash,
                   multipart, double_conflict]}].
 
@@ -714,6 +715,70 @@ conflict_sync(Config) ->
     ?assertEqual({ok, <<"sh2">>}, file:read_file(filename:join([ClientPath2, "shared.1C56416E"]))),
     ok.
 
+delete_sync_conflict() ->
+    [{doc, "A deletion conflict can be sync'd to a third party"},
+        {timetrap, timer:seconds(5)}].
+delete_sync_conflict(Config) ->
+    Client = ?config(name, Config),
+    Server=?config(server, Config),
+    Remote = (?config(peer, Config))(Server),
+    ClientPath = ?config(path, Config),
+    ServerPath = ?config(server_path, Config),
+    {ok, _ServId1} = revault_fsm:id(Server),
+    {ok, _} = revault_fsm_sup:start_fsm(
+        ?config(db_dir, Config),
+        Client,
+        ClientPath,
+        ?config(ignore, Config),
+        ?config(interval, Config),
+        (?config(callback, Config))(Client)
+    ),
+    ok = revault_fsm:client(Client),
+    {ok, _ClientId} = revault_fsm:id(Client, Remote),
+    %% Set up a second client; because of how config works in the test, it needs
+    Client2 = Client ++ "_2",
+    Priv = ?config(priv_dir, Config),
+    DbDir2 = filename:join([Priv, "db_2"]),
+    ClientPath2 = filename:join([Priv, "data", "client_2"]),
+    filelib:ensure_dir(filename:join([DbDir2, "fakefile"])),
+    filelib:ensure_dir(filename:join([ClientPath2, "fakefile"])),
+    {ok, _} = revault_fsm_sup:start_fsm(DbDir2, Client2, ClientPath2,
+                                        ?config(ignore, Config),  ?config(interval, Config),
+                                        (?config(callback, Config))(Client2)),
+    ok = revault_fsm:client(Client2),
+    ?assertMatch({ok, _}, revault_fsm:id(Client2, Remote)),
+    %% now in initialized mode
+    %% Write files
+    ok = file:write_file(filename:join([ServerPath, "shared"]), "sh1"),
+    ok = file:write_file(filename:join([ClientPath, "shared"]), "sh2"),
+    %% Track em
+    ok = revault_dirmon_event:force_scan(Client, 5000),
+    ok = revault_dirmon_event:force_scan(Server, 5000),
+    %% Delete em
+    ok = file:delete(filename:join([ServerPath, "shared"])),
+    ok = file:delete(filename:join([ClientPath, "shared"])),
+    %% Track the deletion
+    ok = revault_dirmon_event:force_scan(Client, 5000),
+    ok = revault_dirmon_event:force_scan(Server, 5000),
+    %% Sync em
+    ct:pal("SYNC", []),
+    ok = revault_fsm:sync(Client, Remote),
+    %% See the result
+    %% conflicting files are marked, with empty conflict files since nothing exists aside
+    %% from clashing deletions.
+    ?assertEqual({error, enoent}, file:read_file(filename:join([ServerPath, "shared"]))),
+    ?assertEqual({error, enoent}, file:read_file(filename:join([ClientPath, "shared"]))),
+    ?assertEqual({ok, <<"">>}, file:read_file(filename:join([ServerPath, "shared.conflict"])) ),
+    ?assertEqual({ok, <<"">>}, file:read_file(filename:join([ClientPath, "shared.conflict"])) ),
+
+    %% Now when client 2 syncs, it gets the files and conflict files as well
+    ct:pal("SECOND SYNC", []),
+    ok = revault_fsm:sync(Client2, Remote),
+    %% conflicting files are marked, but working files aren't sync'd since they didn't exist here
+    ?assertEqual({error, enoent}, file:read_file(filename:join([ClientPath2, "shared"]))),
+    ?assertEqual({ok, <<"">>}, file:read_file(filename:join([ClientPath2, "shared.conflict"])) ),
+    ok.
+
 prevent_server_clash() ->
     [{doc, "A client from a different server cannot connect to the wrong one "
            "as it is protected by a UUID."},

cli/revault_cli/rebar.config

@@ -1 +1,2 @@
-{deps, [argparse]}.
+{deps, [argparse,
+        {cecho, {git, "https://github.com/ferd/cecho.git", {branch, "master"}}}]}.

cli/revault_cli/src/revault_cli.app.src

@@ -2,10 +2,12 @@
  [{description, "An escript to interact with ReVault nodes"},
   {vsn, "0.1.0"},
   {registered, []},
+  {mod, {revault_cli_app, []}},
   {applications,
    [kernel,
     stdlib,
-    argparse
+    argparse,
+    cecho
    ]},
   {env,[]},
   {modules, []},

cli/revault_cli/src/revault_cli.hrl

@@ -0,0 +1,11 @@
+-define(KEY_BACKSPACE, 127).
+-define(KEY_CTRLA, 1).
+-define(KEY_CTRLE, 5).
+-define(KEY_CTRLD, 4).
+-define(KEY_ENTER, 10).
+-define(KEY_TEXT_RANGE(X), % ignore control codes
+        (not(X < 32) andalso
+         not(X >= 127 andalso X < 160))).
+
+-define(EXEC_LINES, 15).
+-define(MAX_VALIDATION_DELAY, 150). % longest time to validate input, in ms

cli/revault_cli/src/revault_cli_app.erl

@@ -0,0 +1,9 @@
+-module(revault_cli_app).
+-behaviour(application).
+-export([start/2, stop/1]).
+
+start(_Type, _Args) ->
+    revault_curses:start_link(revault_cli_mod).
+
+stop(_State) ->
+    ok.