From be9e9b58f8d8962e160a8a8abb44b569650d5f8c Mon Sep 17 00:00:00 2001 From: Valdimar <=> Date: Wed, 10 Feb 2021 19:47:41 +0100 Subject: [PATCH 1/2] Update the rebar config for newer versions of rebar. Update hackney. Fix the wait command since the response is different from before. Change the way the stream methods work, now they use hackney async. moved the sys.config to a config dir rather than run. --- .gitignore | 1 + Makefile | 2 +- config/sys.config | 31 ++++++++++++++++++++ rebar.config | 32 ++++++++++++++++++++- rebar.lock | 51 +++++++++++++++++++++++++++++++++ run/sys.config | 23 --------------- src/docker_container.erl | 23 +++++++++++++-- src/erldocker.app.src | 9 ++++-- src/erldocker_api.erl | 62 +++++++++++++++++++++++----------------- src/erldocker_app.erl | 4 --- 10 files changed, 177 insertions(+), 61 deletions(-) create mode 100644 config/sys.config create mode 100644 rebar.lock delete mode 100644 run/sys.config diff --git a/.gitignore b/.gitignore index d71e977..095e7f8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ deps/ vmling .applist /log/ +_build/ diff --git a/Makefile b/Makefile index 8b49630..9d19930 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ ERLDOCKER_APPS := asn1,crypto,public_key,ssl,mimetypes,hackney,jsx,erldocker ERL_FLAGS= +sbwt none +swct lazy +swt high +K true run: - ERL_LIBS=deps erl -pa ebin -config run/sys.config -sname erldocker \ + ERL_LIBS=deps erl -pa ebin -config config/sys.config -sname erldocker \ $(ERL_FLAGS) \ -eval '[ok = application:ensure_started(A, permanent) || A <- [$(APPS),$(ACTIVE_APPS),$(ERLDOCKER_APPS)]]' diff --git a/config/sys.config b/config/sys.config new file mode 100644 index 0000000..cd6c71f --- /dev/null +++ b/config/sys.config @@ -0,0 +1,31 @@ +[ + {erldocker, [ +% {unixbridge_port, 32133}, +% {docker_http, <<"http://localhost:32133">>} + %{docker_http, <<"http://localhost:4243">>} + + %% For using unix socket interface to docker + {docker_http, <<"http+unix://%2Fvar%2Frun%2Fdocker.sock">>} + ] + }, + {lager, [ + {handlers, [{lager_console_backend, [{level,info}]}]}, + {crash_log, undefined}, + {colored, true} + ] + }, + {sasl, [ + {sasl_error_logger, false}, + {utc_log, true} + ] + }, + {hackney, [ + {shutdown, 10000}, + {maxr, 10}, + {timeout, 150000}, + {restart, permanent}, + {max_connections, 50}, + {maxt,8} + ] + } +]. diff --git a/rebar.config b/rebar.config index 8df5e07..26addb2 100644 --- a/rebar.config +++ b/rebar.config @@ -1,8 +1,38 @@ {deps, [ - {hackney, "1.16.0"}, + {hackney, "1.17.0"}, {jsx, "2.9.0"}, {erlsh, "0.1.0"}, {fn, "0.3.0", {git, "https://github.com/reiddraper/fn", {tag, "0.3.0"}}}, {lager, "3.8.0"}, {active, "6.1.1"} ]}. +{relx, [{release, {erldocker, "1.0.0"}, + [ + kernel, + compiler, + jsx, + hackney, + lager, + erldocker + ]}, + {sys_config, "./config/sys.config"}, + {extended_start_script, true} + ] +}. + +{profiles, [ + {test, [ + {erl_opts, [ + debug_info, + {parse_transform, lager_transform}, + {sasl, [{utc_log, true}]} + ]}, + {cover_enabled, true}, + {cover_opts, [verbose]} + ] + }, + {prod, [{relx, [{dev_mode, false}, + {include_erts, true}]}] + }] +}. + diff --git a/rebar.lock b/rebar.lock new file mode 100644 index 0000000..f4d5ee9 --- /dev/null +++ b/rebar.lock @@ -0,0 +1,51 @@ +{"1.2.0", +[{<<"active">>,{pkg,<<"active">>,<<"6.1.1">>},0}, + {<<"certifi">>,{pkg,<<"certifi">>,<<"2.5.3">>},1}, + {<<"erlsh">>,{pkg,<<"erlsh">>,<<"0.1.0">>},0}, + {<<"fn">>, + {git,"https://github.com/reiddraper/fn", + {ref,"1e8f3f43e5fa6e5e68eb19184fa60d8da5de99d0"}}, + 0}, + {<<"fs">>,{pkg,<<"fs">>,<<"6.1.1">>},1}, + {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1}, + {<<"hackney">>,{pkg,<<"hackney">>,<<"1.17.0">>},0}, + {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},1}, + {<<"jsx">>,{pkg,<<"jsx">>,<<"2.9.0">>},0}, + {<<"lager">>,{pkg,<<"lager">>,<<"3.8.0">>},0}, + {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},1}, + {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},1}, + {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.1">>},1}, + {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.6">>},1}, + {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},1}]}. +[ +{pkg_hash,[ + {<<"active">>, <<"EB6E403C03B4FCD1C0CE3E62C8655C82F5CEF4F65249EF75FEF1B2B7ED212680">>}, + {<<"certifi">>, <<"70BDD7E7188C804F3A30EE0E7C99655BC35D8AC41C23E12325F36AB449B70651">>}, + {<<"erlsh">>, <<"77FA764E1ACAD2B0E06C0F4BD2AB9D3C3429CC058048938B1399A534D861C88D">>}, + {<<"fs">>, <<"9D147B944D60CFA48A349F12D06C8EE71128F610C90870BDF9A6773206452ED0">>}, + {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>}, + {<<"hackney">>, <<"717EA195FD2F898D9FE9F1CE0AFCC2621A41ECFE137FAE57E7FE6E9484B9AA99">>}, + {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, + {<<"jsx">>, <<"D2F6E5F069C00266CAD52FB15D87C428579EA4D7D73A33669E12679E203329DD">>}, + {<<"lager">>, <<"3402B9A7E473680CA179FC2F1D827CAB88DD37DD1E6113090C6F45EF05228A1C">>}, + {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, + {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>}, + {<<"parse_trans">>, <<"16328AB840CC09919BD10DAB29E431DA3AF9E9E7E7E6F0089DD5A2D2820011D8">>}, + {<<"ssl_verify_fun">>, <<"CF344F5692C82D2CD7554F5EC8FD961548D4FD09E7D22F5B62482E5AEAEBD4B0">>}, + {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]}, +{pkg_hash_ext,[ + {<<"active">>, <<"E34080D6131DC835FAC093EB1232DB26BAA33B279ABD27A986B2D5DE8221966C">>}, + {<<"certifi">>, <<"ED516ACB3929B101208A9D700062D520F3953DA3B6B918D866106FFA980E1C10">>}, + {<<"erlsh">>, <<"94EF1492DD59FEF211F01FFD40C47B6E51C0F59E2A3D0739366E4890961332D9">>}, + {<<"fs">>, <<"EF94E95FFE79916860649FED80AC62B04C322B0BB70F5128144C026B4D171F8B">>}, + {<<"goldrush">>, <<"99CB4128CFFCB3227581E5D4D803D5413FA643F4EB96523F77D9E6937D994CEB">>}, + {<<"hackney">>, <<"64C22225F1EA8855F584720C0E5B3CD14095703AF1C9FBC845BA042811DC671C">>}, + {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, + {<<"jsx">>, <<"8EE1DB1CABAFDD578A2776A6AAAE87C2A8CE54B47B59E9EC7DAB5D7EB71CD8DC">>}, + {<<"lager">>, <<"F6CB541B688EAB60730D8D286EB77256A5A9AD06EAC10D43BEAF55D07E68BBB6">>}, + {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, + {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>}, + {<<"parse_trans">>, <<"07CD9577885F56362D414E8C4C4E6BDF10D43A8767ABB92D24CBE8B24C54888B">>}, + {<<"ssl_verify_fun">>, <<"BDB0D2471F453C88FF3908E7686F86F9BE327D065CC1EC16FA4540197EA04680">>}, + {<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]} +]. diff --git a/run/sys.config b/run/sys.config deleted file mode 100644 index c9d3b30..0000000 --- a/run/sys.config +++ /dev/null @@ -1,23 +0,0 @@ -[ - {erldocker, [ - {unixbridge_port, 32133}, - {docker_http, <<"http://localhost:32133">>} - %{docker_http, <<"http://localhost:4243">>} - - %% For using unix socket interface to docker - %{docker_http, <<"http+unix://%2Fvar%2Frun%2Fdocker.sock">>} - ]}, - - {lager, [ - {handlers, [{lager_console_backend, info}]}, - {crash_log, undefined} - %, {colored, true} - ]}, - - {sasl, [{sasl_error_logger, false}]}, - - {rebar, [ - {log_function, {error_logger, format}}, - {console_log_function, {error_logger, format}} - ]} -]. diff --git a/src/docker_container.erl b/src/docker_container.erl index e3a6c7e..363ceb8 100644 --- a/src/docker_container.erl +++ b/src/docker_container.erl @@ -13,7 +13,7 @@ -export([kill/1]). -export([attach_logs/1, attach_stream/1, attach_stream_with_logs/1]). -export([delete/1, delete/2]). --export([wait/1]). +-export([wait/1, wait/2]). -export([commit/1, commit/2]). -export([copy/2]). % not implemented @@ -105,7 +105,26 @@ delete(CID, Args) -> wait(CID) -> either:bind(erldocker_api:post([containers, CID, wait]), fun wait_proc/1). -wait_proc({[{<<"StatusCode">>, Status}]}) -> {ok, Status}. +wait(CID, Receiver) -> + spawn_link(fun() -> + either:bind(erldocker_api:post([containers, CID, wait]), + fun([{<<"Error">>, Error}, {<<"StatusCode">>, Status}]) -> + case Error of + null -> + Receiver ! {stopped, Status}; + _ -> + Receiver ! {stopped, Error, Status} + end + end) + end). + +wait_proc([{<<"Error">>, Error}, {<<"StatusCode">>, Status}]) -> + case Error of + null -> + {ok, Status}; + _ -> + {error, Error, Status} + end. % @doc Copy files or folders of container. copy(_CID, _Args) -> diff --git a/src/erldocker.app.src b/src/erldocker.app.src index 09f36f9..da7c3b0 100644 --- a/src/erldocker.app.src +++ b/src/erldocker.app.src @@ -1,14 +1,17 @@ {application, erldocker, [ - {description, ""}, + {description, "Wrapper for the Docker API"}, {vsn, "1"}, {registered, []}, + {mod, {erldocker_app, []}}, {applications, [ kernel, stdlib, hackney, jsx ]}, - {mod, { erldocker_app, []}}, - {env, []} + {env, []}, + {modules, []}, + {licenses, []}, + {links, []} ]}. diff --git a/src/erldocker_api.erl b/src/erldocker_api.erl index df793e7..182c033 100644 --- a/src/erldocker_api.erl +++ b/src/erldocker_api.erl @@ -4,7 +4,7 @@ -export([proplist_from_json/1, proplists_from_json/1]). -define(ADDR, application:get_env(erldocker, docker_http, <<"http://localhost:4243">>)). --define(OPTIONS, [{pool, erldocker_pool}]). +-define(OPTIONS, [{recv_timeout, infinity}]). get(URL) -> call(get, <<>>, URL, []). get(URL, Args) -> call(get, <<>>, URL, Args). @@ -20,19 +20,7 @@ post_stream(URL, Args) -> call({post, stream}, <<>>, URL, Args). call({Method, stream}, Body, URL) when is_binary(URL) andalso is_binary(Body) -> error_logger:info_msg("api call: ~p ~s", [{Method, stream}, binary_to_list(URL)]), - case hackney:request(Method, URL, [], Body, ?OPTIONS) of - {ok, StatusCode, _RespHeaders, Client} -> - case StatusCode of - X when X == 200 orelse X == 201 orelse X == 204 -> - Rcv = self(), - Pid = spawn_link(fun() -> read_body(Rcv, Client) end), - {ok, Pid}; - _ -> - {error, StatusCode} - end; - {error, _} = E -> - E - end; + spawn_link(fun() -> async(URL, self()) end); call(Method, Body, URL) when is_binary(URL) andalso is_binary(Body) -> error_logger:info_msg("api call: ~p ~s", [Method, binary_to_list(URL)]), @@ -56,19 +44,39 @@ call(Method, Body, URL) when is_binary(URL) andalso is_binary(Body) -> call(Method, Body, URL, Args) when is_binary(Body) -> call(Method, Body, to_url(URL, Args)). -read_body(Receiver, Client) -> - case hackney:stream_body(Client) of - {ok, Data, Client2} -> - Receiver ! {self(), {data, Data}}, - read_body(Receiver, Client2); - {done, Client2} -> - Receiver ! {self(), {data, eof}}, - {ok, Client2}; - {error, _Reason} = E-> - Receiver ! {self(), E}, - E - end. - +async(Url, Receiver) -> + Options = [{recv_timeout, infinity}, async], + LoopFun = fun(Loop, Ref) -> + receive + {hackney_response, Ref, {status, StatusInt, Reason}} -> + io:format("Status update ~p, ~p~n",[StatusInt, Reason]), + Receiver ! {self(), {status, StatusInt, Reason}}, + Loop(Loop, Ref); + {hackney_response, Ref, {headers, Headers}} -> + Receiver ! {self(), {headers, Headers}}, + io:format("got headers: ~p~n", [Headers]), + Loop(Loop, Ref); + {hackney_response, Ref, done} -> + Receiver ! {self(), {done}}, + ok; + {hackney_response, Ref, Bin} -> + io:format("got chunk: ~p~n", [Bin]), + Receiver ! {self(), {data, Bin}}, + Loop(Loop, Ref); + {Pid, {status, Code, Status}} -> + io:format("Pid ~p returned ~p with code ~p~n", [Pid, Status, Code]), + Receiver ! {self(), {status, Code, Status}}, + Loop(Loop, Ref); + Else -> + % In case this is running in a shell this breaks the loop, + % otherwise the loop will be endless + io:format("ELSE ~p~n", [Else]), + Receiver ! {self(), {Else}} + end + end, + {ok, ClientRef} = hackney:post(Url, [], <<>>, Options), + LoopFun(LoopFun, ClientRef). + argsencode([], Acc) -> hackney_bstr:join(lists:reverse(Acc), <<"&">>); argsencode ([{_K,undefined}|R], Acc) -> diff --git a/src/erldocker_app.erl b/src/erldocker_app.erl index 8a597db..72a1329 100644 --- a/src/erldocker_app.erl +++ b/src/erldocker_app.erl @@ -9,11 +9,7 @@ %% =================================================================== start(_StartType, _StartArgs) -> - ok = hackney_pool:start_pool(erldocker_pool, [{timeout, 150000}, {pool_size, 1}]), - erldocker_sup:start_link(). stop(_State) -> - hackney_pool:stop_pool(erldocker_pool), - ok. From 389f0bb4157d48c58bb78fa66fdee389797808b1 Mon Sep 17 00:00:00 2001 From: Valdimar <=> Date: Mon, 3 May 2021 09:59:48 +0200 Subject: [PATCH 2/2] Fix processing of image pull since it is a mulitiline json which breaks jsx encoding. --- src/erldocker_api.erl | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/erldocker_api.erl b/src/erldocker_api.erl index 182c033..1641514 100644 --- a/src/erldocker_api.erl +++ b/src/erldocker_api.erl @@ -31,8 +31,21 @@ call(Method, Body, URL) when is_binary(URL) andalso is_binary(Body) -> case StatusCode of X when X == 200 orelse X == 201 orelse X == 204 -> case lists:keyfind(<<"Content-Type">>, 1, RespHeaders) of - {_, <<"application/json">>} -> {ok, jsx:decode(RespBody)}; - _ -> {ok, {StatusCode, RespBody}} + {_, <<"application/json">>} -> + case re:split(RespBody, <<"\r\n">>) of + [RespBody] -> + {ok, jsx:decode(RespBody)}; + _ -> + %% The response is a multiline response as a string + Replaced = re:replace( + RespBody, <<"\r\n">>, <<",">>, + [global, {return, list}] + ), + Trimmed = string:trim(Replaced, both, ","), + {ok, jsx:decode(list_to_binary("["++Trimmed++"]"))} + end; + _ -> + {ok, {StatusCode, RespBody}} end; _ -> {error, {StatusCode, RespBody}}