From 7905a9f19cdb0ca2ff70567613784084e3e0a2fd Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 8 May 2015 20:06:43 -0500 Subject: store hex package downloads in shared cache dir --- src/rebar_fetch.erl | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 20bf46b..ec16089 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -40,20 +40,15 @@ download_source(AppDir, Source, State) -> ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), true; {tarball, File} -> - Contents = filename:join(TmpDir, "contents"), ec_file:mkdir_p(AppDir1), - ec_file:mkdir_p(Contents), - ok = erl_tar:extract(File, [{cwd, TmpDir}]), - ok = erl_tar:extract(filename:join(TmpDir, "contents.tar.gz"), - [{cwd, Contents}, compressed]), + {ok, Files} = erl_tar:extract(File, [memory]), + code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), ec_file:remove(filename:absname(AppDir1), [recursive]), - ?DEBUG("Moving contents ~p to ~p", [Contents, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(Contents, filename:absname(AppDir1)), - - ?DEBUG("Removing tmp dir ~p", [TmpDir]), - ec_file:remove(TmpDir, [recursive]), + {"contents.tar.gz", Binary} = lists:keyfind("contents.tar.gz", 1, Files), + ok = erl_tar:extract({binary, Binary}, + [{cwd, filename:absname(AppDir1)}, compressed]), true end catch -- cgit v1.1 From 819d0fb06fbaaec25b1929e971964d5757ba59ff Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 9 May 2015 23:01:17 -0500 Subject: verify checksums of hex packages --- src/rebar_fetch.erl | 48 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index ec16089..16840eb 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -40,16 +40,7 @@ download_source(AppDir, Source, State) -> ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), true; {tarball, File} -> - ec_file:mkdir_p(AppDir1), - {ok, Files} = erl_tar:extract(File, [memory]), - - code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), - - {"contents.tar.gz", Binary} = lists:keyfind("contents.tar.gz", 1, Files), - ok = erl_tar:extract({binary, Binary}, - [{cwd, filename:absname(AppDir1)}, compressed]), - true + verify_and_extract(File, Source, AppDir1, State) end catch C:T -> @@ -69,7 +60,11 @@ needs_update(AppDir, Source, State) -> end. format_error({fetch_fail, Source}) -> - io_lib:format("Failed to fetch and copy dep: ~p", [Source]). + io_lib:format("Failed to fetch and copy dep: ~p", [Source]); +format_error({bad_checksum, File}) -> + io_lib:format("Checksum mismatch against tarball in ~s", [File]); +format_error({bad_registry_checksum, File}) -> + io_lib:format("Checksum mismatch against registry in ~s", [File]). get_resource_type({Type, Location}, Resources) -> find_resource_module(Type, Location, Resources); @@ -93,3 +88,34 @@ find_resource_module(Type, Location, Resources) -> {Type, Module} -> Module end. + +verify_and_extract(File, Source, AppDir, State) -> + ec_file:mkdir_p(AppDir), + {ok, Files} = erl_tar:extract(File, [memory]), + + code:del_path(filename:absname(filename:join(AppDir, "ebin"))), + ec_file:remove(filename:absname(AppDir), [recursive]), + + {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), + {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), + {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), + + Checksum = checksum(Contents, Version, Meta), + RegistryChecksum = rebar_packages:registry_checksum(Source, State), + {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), + + if + Checksum =/= TarChecksum -> + ?PRV_ERROR({bad_checksum, File}); + Checksum =/= RegistryChecksum -> + ?PRV_ERROR({bad_registry_checksum, File}); + true -> + ok = erl_tar:extract({binary, Contents}, + [{cwd, filename:absname(AppDir)}, compressed]), + true + end. + +checksum(Contents, Version, Meta) -> + Blob = <>, + <> = crypto:hash(sha256, Blob), + list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))). -- cgit v1.1 From b636822d73410795cdc46d4722f7225c0f463fd3 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 12 May 2015 09:38:22 -0500 Subject: check md5sum of package against that sent by s3 --- src/rebar_fetch.erl | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 16840eb..75970ed 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -43,6 +43,8 @@ download_source(AppDir, Source, State) -> verify_and_extract(File, Source, AppDir1, State) end catch + _:bad_etag -> + throw(?PRV_ERROR({bad_etag, Source})); C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), throw(?PRV_ERROR({fetch_fail, Source})) @@ -59,6 +61,8 @@ needs_update(AppDir, Source, State) -> true end. +format_error({bad_etag, Source}) -> + io_lib:format("MD5 Checksum comparison failed for: ~p", [Source]); format_error({fetch_fail, Source}) -> io_lib:format("Failed to fetch and copy dep: ~p", [Source]); format_error({bad_checksum, File}) -> -- cgit v1.1 From 56c925b75b86a6304da75635874722348ee21351 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 12 May 2015 16:58:38 +0000 Subject: Ad-hoc attempt at restructuring pkg cache --- src/rebar_fetch.erl | 51 +++++++-------------------------------------------- 1 file changed, 7 insertions(+), 44 deletions(-) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 75970ed..235aa03 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -31,20 +31,14 @@ download_source(AppDir, Source, State) -> Module = get_resource_type(Source, Resources), TmpDir = ec_file:insecure_mkdtemp(), AppDir1 = ec_cnv:to_list(AppDir), - case Module:download(TmpDir, Source, State) of - {ok, _} -> - ec_file:mkdir_p(AppDir1), - code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), - ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true; - {tarball, File} -> - verify_and_extract(File, Source, AppDir1, State) - end + {ok, _} = Module:download(TmpDir, Source, State), + ec_file:mkdir_p(AppDir1), + code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), + ec_file:remove(filename:absname(AppDir1), [recursive]), + ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), + ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), + true catch - _:bad_etag -> - throw(?PRV_ERROR({bad_etag, Source})); C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), throw(?PRV_ERROR({fetch_fail, Source})) @@ -92,34 +86,3 @@ find_resource_module(Type, Location, Resources) -> {Type, Module} -> Module end. - -verify_and_extract(File, Source, AppDir, State) -> - ec_file:mkdir_p(AppDir), - {ok, Files} = erl_tar:extract(File, [memory]), - - code:del_path(filename:absname(filename:join(AppDir, "ebin"))), - ec_file:remove(filename:absname(AppDir), [recursive]), - - {"contents.tar.gz", Contents} = lists:keyfind("contents.tar.gz", 1, Files), - {"VERSION", Version} = lists:keyfind("VERSION", 1, Files), - {"metadata.config", Meta} = lists:keyfind("metadata.config", 1, Files), - - Checksum = checksum(Contents, Version, Meta), - RegistryChecksum = rebar_packages:registry_checksum(Source, State), - {"CHECKSUM", TarChecksum} = lists:keyfind("CHECKSUM", 1, Files), - - if - Checksum =/= TarChecksum -> - ?PRV_ERROR({bad_checksum, File}); - Checksum =/= RegistryChecksum -> - ?PRV_ERROR({bad_registry_checksum, File}); - true -> - ok = erl_tar:extract({binary, Contents}, - [{cwd, filename:absname(AppDir)}, compressed]), - true - end. - -checksum(Contents, Version, Meta) -> - Blob = <>, - <> = crypto:hash(sha256, Blob), - list_to_binary(string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X])))). -- cgit v1.1 From 1fe93136c426a0c742134f392cc7f4d4d7ef7b3b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 12 May 2015 19:23:37 -0500 Subject: throw errors returned by resource:download/3 --- src/rebar_fetch.erl | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 235aa03..96bbff1 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -26,24 +26,34 @@ lock_source(AppDir, Source, State) -> -spec download_source(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> true | {error, any()}. download_source(AppDir, Source, State) -> - try - Resources = rebar_state:resources(State), - Module = get_resource_type(Source, Resources), - TmpDir = ec_file:insecure_mkdtemp(), - AppDir1 = ec_cnv:to_list(AppDir), - {ok, _} = Module:download(TmpDir, Source, State), - ec_file:mkdir_p(AppDir1), - code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), - ec_file:remove(filename:absname(AppDir1), [recursive]), - ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), - ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), - true + try download_source_(AppDir, Source, State) of + true -> + true; + Error -> + throw(Error) catch C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), throw(?PRV_ERROR({fetch_fail, Source})) end. +download_source_(AppDir, Source, State) -> + Resources = rebar_state:resources(State), + Module = get_resource_type(Source, Resources), + TmpDir = ec_file:insecure_mkdtemp(), + AppDir1 = ec_cnv:to_list(AppDir), + case Module:download(TmpDir, Source, State) of + {ok, _} -> + ec_file:mkdir_p(AppDir1), + code:del_path(filename:absname(filename:join(AppDir1, "ebin"))), + ec_file:remove(filename:absname(AppDir1), [recursive]), + ?DEBUG("Moving checkout ~p to ~p", [TmpDir, filename:absname(AppDir1)]), + ok = rebar_file_utils:mv(TmpDir, filename:absname(AppDir1)), + true; + Error -> + Error + end. + -spec needs_update(file:filename_all(), rebar_resource:resource(), rebar_state:t()) -> boolean() | {error, string()}. needs_update(AppDir, Source, State) -> Resources = rebar_state:resources(State), -- cgit v1.1 From 1643f4a1fc8a480c20ce972d9df47669111fff22 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 12 May 2015 19:32:24 -0500 Subject: wrap fetch errors in rebar_fetch PRV_ERROR --- src/rebar_fetch.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 96bbff1..0bb2270 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -30,7 +30,7 @@ download_source(AppDir, Source, State) -> true -> true; Error -> - throw(Error) + throw(?PRV_ERROR(Error)) catch C:T -> ?DEBUG("rebar_fetch exception ~p ~p ~p", [C, T, erlang:get_stacktrace()]), @@ -65,10 +65,12 @@ needs_update(AppDir, Source, State) -> true end. +format_error({failed_extract, CachePath}) -> + io_lib:format("Failed to extract package: ~s", [CachePath]); format_error({bad_etag, Source}) -> - io_lib:format("MD5 Checksum comparison failed for: ~p", [Source]); + io_lib:format("MD5 Checksum comparison failed for: ~s", [Source]); format_error({fetch_fail, Source}) -> - io_lib:format("Failed to fetch and copy dep: ~p", [Source]); + io_lib:format("Failed to fetch and copy dep: ~s", [Source]); format_error({bad_checksum, File}) -> io_lib:format("Checksum mismatch against tarball in ~s", [File]); format_error({bad_registry_checksum, File}) -> -- cgit v1.1 From 86fbaaeb08e7fc35ee87daea62be7febacd14ea9 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 12 May 2015 19:36:10 -0500 Subject: add bad_download error --- src/rebar_fetch.erl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/rebar_fetch.erl') diff --git a/src/rebar_fetch.erl b/src/rebar_fetch.erl index 0bb2270..0aca308 100644 --- a/src/rebar_fetch.erl +++ b/src/rebar_fetch.erl @@ -65,6 +65,8 @@ needs_update(AppDir, Source, State) -> true end. +format_error({bad_download, CachePath}) -> + io_lib:format("Download of package does not match md5sum from server: ~s", [CachePath]); format_error({failed_extract, CachePath}) -> io_lib:format("Failed to extract package: ~s", [CachePath]); format_error({bad_etag, Source}) -> -- cgit v1.1