From 4f32da2ab7e0a7cf2f2e2de911c0c8ea0a7d8848 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 25 Aug 2016 22:21:07 -0400 Subject: Equivalent trim_all in bin split for <17.x The trim_all option used in binary:split/3 is not supported in 17.x. This patch makes an equivalent operation by eliminating empty split fragments. From the docs: trim Removes trailing empty parts of the result (as does trim in re:split/3. trim_all Removes all empty parts of the result. The new expression is therefore equivalent to the old one, but with the added benefit of compatibility. Fixes #1275 --- src/rebar_app_utils.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index d256cac..50c6314 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -182,7 +182,7 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> undefined -> get_package(PkgName, "0", State); <<"~>", Vsn/binary>> -> - [Vsn1] = binary:split(Vsn, [<<" ">>], [trim_all, global]), + [Vsn1] = [X || X <- binary:split(Vsn, [<<" ">>], [global]), X =/= <<>>], get_package(PkgName, Vsn1, State); _ -> {PkgName, PkgVsn} -- cgit v1.1 From 6bd8cda77aaa089f9eaa87dd696cfa63af5f2292 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 27 Aug 2016 15:56:01 -0400 Subject: Fix crash when doing hash check with missing index Specifically, when fetching an application where the expected hash is unknown, the hash is validated from the hex index; when the index is available, the hash is fetched fine and later inserted in the lock file. However, if the index is not available, the call would simply crash. This patch fixes thing so that instead, the index is refreshed before giving up and failing. --- src/rebar_app_utils.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 50c6314..847b0a5 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -190,7 +190,7 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> %% store the expected hash for the dependency Hash1 = case Hash of undefined -> % unknown, define the hash since we know the dep - rebar_packages:registry_checksum({pkg, PkgName1, PkgVsn1, Hash}, State); + fetch_checksum(PkgName1, PkgVsn1, Hash, State); _ -> % keep as is Hash end, @@ -203,6 +203,15 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). +fetch_checksum(PkgName, PkgVsn, Hash, State) -> + try + rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) + catch + _:_ -> + ?INFO("Package ~s-~s not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]), + {ok, _} = rebar_prv_update:do(State), + rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) + end. format_error({missing_package, Package}) -> io_lib:format("Package not found in registry: ~s", [Package]); -- cgit v1.1 From 3da4cc222197e01886d3baaeca7a380e02ff3125 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 25 Nov 2016 21:32:43 -0500 Subject: Type specifications and edocs improvements Includes improvments and function documentation for all modules (in alphabetical order) up to rebar_core, and may have included more in other modules as I saw fit to dig and understand more of the internals. --- src/rebar_app_utils.erl | 79 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 2 deletions(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 847b0a5..7154999 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -44,10 +44,14 @@ %% Public API %% =================================================================== +%% @doc finds the proper app info record for a given app name in a list of +%% such records. -spec find(binary(), [rebar_app_info:t()]) -> {ok, rebar_app_info:t()} | error. find(Name, Apps) -> ec_lists:find(fun(App) -> rebar_app_info:name(App) =:= Name end, Apps). +%% @doc finds the proper app info record for a given app name at a given version +%% in a list of such records. -spec find(binary(), binary(), [rebar_app_info:t()]) -> {ok, rebar_app_info:t()} | error. find(Name, Vsn, Apps) -> ec_lists:find(fun(App) -> @@ -55,11 +59,18 @@ find(Name, Vsn, Apps) -> andalso rebar_app_info:original_vsn(App) =:= Vsn end, Apps). +%% @doc checks if a given file is .app.src file is_app_src(Filename) -> %% If removing the extension .app.src yields a shorter name, %% this is an .app.src file. Filename =/= filename:rootname(Filename, ".app.src"). +%% @doc translates the name of the .app.src[.script] file to where +%% its .app counterpart should be stored. +-spec app_src_to_app(OutDir, SrcFilename) -> OutFilename when + OutDir :: file:filename(), + SrcFilename :: file:filename(), + OutFilename :: file:filename(). app_src_to_app(OutDir, Filename) -> AppFile = case lists:suffix(".app.src", Filename) of @@ -72,10 +83,16 @@ app_src_to_app(OutDir, Filename) -> filelib:ensure_dir(AppFile), AppFile. +%% @doc checks whether the .app file has all the required data to be valid, +%% and cross-references it with compiled modules on disk -spec validate_application_info(rebar_app_info:t()) -> boolean(). validate_application_info(AppInfo) -> validate_application_info(AppInfo, rebar_app_info:app_details(AppInfo)). +%% @doc checks whether the .app file has all the required data to be valid +%% and cross-references it with compiled modules on disk. +%% The app info is passed explicitly as a second argument. +-spec validate_application_info(rebar_app_info:t(), list()) -> boolean(). validate_application_info(AppInfo, AppDetail) -> EbinDir = rebar_app_info:ebin_dir(AppInfo), case rebar_app_info:app_file(AppInfo) of @@ -90,13 +107,37 @@ validate_application_info(AppInfo, AppDetail) -> end end. --spec parse_deps(binary(), list(), rebar_state:t(), list(), integer()) -> [rebar_app_info:t()]. +%% @doc parses all dependencies from the root of the project +-spec parse_deps(Dir, Deps, State, Locks, Level) -> [rebar_app_info:t()] when + Dir :: file:filename(), + Deps :: [tuple() | atom() | binary()], % TODO: meta to source() | lock() + State :: rebar_state:t(), + Locks :: [tuple()], % TODO: meta to [lock()] + Level :: non_neg_integer(). parse_deps(DepsDir, Deps, State, Locks, Level) -> parse_deps(root, DepsDir, Deps, State, Locks, Level). +%% @doc runs `parse_dep/6' for a set of dependencies. +-spec parse_deps(Parent, Dir, Deps, State, Locks, Level) -> [rebar_app_info:t()] when + Parent :: root | binary(), + Dir :: file:filename(), + Deps :: [tuple() | atom() | binary()], % TODO: meta to source() | lock() + State :: rebar_state:t(), + Locks :: [tuple()], % TODO: meta to [lock()] + Level :: non_neg_integer(). parse_deps(Parent, DepsDir, Deps, State, Locks, Level) -> [parse_dep(Dep, Parent, DepsDir, State, Locks, Level) || Dep <- Deps]. +%% @doc for a given dep, return its app info record. The function +%% also has to choose whether to define the dep from its immediate spec +%% (if it is a newer thing) or from the locks specified in the lockfile. +-spec parse_dep(Dep, Parent, Dir, State, Locks, Level) -> rebar_app_info:t() when + Dep :: tuple() | atom() | binary(), % TODO: meta to source() | lock() + Parent :: root | binary(), + Dir :: file:filename(), + State :: rebar_state:t(), + Locks :: [tuple()], % TODO: meta to [lock()] + Level :: non_neg_integer(). parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> Name = case Dep of Dep when is_tuple(Dep) -> @@ -117,6 +158,14 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> end end. +%% @doc converts a dependency definition and a location for it on disk +%% into an app info tuple representing it. +-spec parse_dep(Parent, Dep, Dir, IsLock, State) -> rebar_app_info:t() when + Parent :: root | binary(), + Dep :: tuple() | atom() | binary(), % TODO: meta to source() | lock() + Dir :: file:filename(), + IsLock :: boolean(), + State :: rebar_state:t(). parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> {PkgName1, PkgVsn} = {ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn, undefined}, IsLock, State); @@ -152,6 +201,16 @@ parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(S parse_dep(_, Dep, _, _, _) -> throw(?PRV_ERROR({parse_dep, Dep})). +%% @doc convert a dependency that has just been fetched into +%% an app info record related to it +-spec dep_to_app(Parent, Dir, Name, Vsn, Source, IsLock, State) -> rebar_app_info:t() when + Parent :: root | binary(), + Dir :: file:filename(), + Name :: binary(), + Vsn :: iodata() | undefined, + Source :: tuple(), + IsLock :: boolean(), + State :: rebar_state:t(). dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of @@ -166,7 +225,7 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> not_found -> rebar_app_info:new(Parent, Name, Vsn, Dir, []) end, - update_source(AppInfo0, Source, State) + update_source(AppInfo0, Source, State) end, C = rebar_config:consult(rebar_app_info:dir(AppInfo)), AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), @@ -177,6 +236,11 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). +%% @doc sets the source for a given dependency or app along with metadata +%% around version if required. +-spec update_source(rebar_app_info:t(), Source, rebar_state:t()) -> + rebar_app_info:t() when + Source :: tuple() | atom() | binary(). % TODO: meta to source() update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> {PkgName1, PkgVsn1} = case PkgVsn of undefined -> @@ -203,6 +267,9 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). +%% @doc grab the checksum for a given package +-spec fetch_checksum(atom(), string(), iodata() | undefined, rebar_state:t()) -> + iodata() | no_return(). fetch_checksum(PkgName, PkgVsn, Hash, State) -> try rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) @@ -213,6 +280,8 @@ fetch_checksum(PkgName, PkgVsn, Hash, State) -> rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) end. +%% @doc convert a given exception's payload into an io description. +-spec format_error(any()) -> iolist(). format_error({missing_package, Package}) -> io_lib:format("Package not found in registry: ~s", [Package]); format_error({parse_dep, Dep}) -> @@ -224,6 +293,10 @@ format_error(Error) -> %% Internal functions %% =================================================================== +%% @private find the correct version of a package based on the version +%% and name passed in. +-spec get_package(binary(), binary() | string(), rebar_state:t()) -> + term() | no_return(). get_package(Dep, Vsn, State) -> case rebar_packages:find_highest_matching(Dep, Vsn, ?PACKAGE_TABLE, State) of {ok, HighestDepVsn} -> @@ -232,6 +305,8 @@ get_package(Dep, Vsn, State) -> throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Dep)})) end. +%% @private checks that all the beam files have been properly +%% created. -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> -- cgit v1.1 From 1c96de5e10bd51ba4da9a89c087c191a8c7cd8c5 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 5 Aug 2017 10:35:51 -0400 Subject: Allow top-level apps to take precedence over deps The use case has been described in issue #1478 where a local application can exist while being declared as a dependency as well. This allows, for example, to work on a release where all applications may require to be published independently, or to provide some form of 'vendoring' with a local app. The fix is done by decoupling the dependency source resolution form the dependency parsing. The reason for this being that the discovery phase needs to parse apps for their top-level deps, and dep installation needs to resolve the packages with accuracy. In the current implementation, both code paths call to the same function. This patch splits up the precise discovery and makes it happen *only* when installing dependencies, and only if a top-level app does not already define the application needing resolving. One weakness of this fix is that it necessarily breaks cycle detection in dependencies that involve a root application depending on itself since its own version as a dep will not be expanded. There appears to be no possible way to prevent this, but should be rare enough to be worth the tradeoff for the common case. --- src/rebar_app_utils.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 7154999..4e87adc 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -34,6 +34,7 @@ validate_application_info/2, parse_deps/5, parse_deps/6, + expand_deps_sources/2, dep_to_app/7, format_error/1]). @@ -225,7 +226,7 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> not_found -> rebar_app_info:new(Parent, Name, Vsn, Dir, []) end, - update_source(AppInfo0, Source, State) + rebar_app_info:source(AppInfo0, Source) end, C = rebar_config:consult(rebar_app_info:dir(AppInfo)), AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), @@ -236,6 +237,13 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). +%% @doc Takes a given application app_info record along with the project. +%% If the app is a package, resolve and expand the package definition. +-spec expand_deps_sources(rebar_app_info:t(), rebar_state:t()) -> + rebar_app_info:t(). +expand_deps_sources(Dep, State) -> + update_source(Dep, rebar_app_info:source(Dep), State). + %% @doc sets the source for a given dependency or app along with metadata %% around version if required. -spec update_source(rebar_app_info:t(), Source, rebar_state:t()) -> -- cgit v1.1 From 963c49f5eb9ab5b34e1843fb43305743720917ac Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 6 Aug 2017 07:26:21 -0400 Subject: Unicode support in all the places This is done through 3 main change groups: - replacing `~s` by `~ts` in format strings, so that strings that contain unicode are properly printed rather than crashing - adding the `unicode` argument to all function of the `re` module to ensure transformations on strings containing unicode data are valid instead of crashing (see issue #1302) - replacing `ec_cnv:to_binary/1` and `ec_cnv:to_list/1` with matching functions in `rebar_utils`. The last point has been done, rather than modifying and updating erlware commons, because binary and list conversions can be a contentious subject. For example, if what is being handled is actually bytes from a given binary stream, then forcing a byte-oriented interpretation of the data can corrupt it. As such, it does not appear safe to modify erlware commons' conversion functions since it may not be safe for all its users. Instead, rebar3 reimplements a subset of them (only converting atoms and chardata, ignoring numbers) with the explicit purpose of handling unicode string data. Tests were left as unchanged as possible. This may impact the ability to run rebar3's own suites in a unicode path, but respects a principle of least change for such a large patch. --- src/rebar_app_utils.erl | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 7154999..9422240 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -145,7 +145,7 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> Dep -> Dep end, - case lists:keyfind(ec_cnv:to_binary(Name), 1, Locks) of + case lists:keyfind(rebar_utils:to_binary(Name), 1, Locks) of false -> parse_dep(Parent, Dep, DepsDir, false, State); LockedDep -> @@ -167,18 +167,20 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> IsLock :: boolean(), State :: rebar_state:t(). parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> - {PkgName1, PkgVsn} = {ec_cnv:to_binary(PkgName), ec_cnv:to_binary(Vsn)}, + {PkgName1, PkgVsn} = {rebar_utils:to_binary(PkgName), + rebar_utils:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn, undefined}, IsLock, State); parse_dep(Parent, {Name, {pkg, PkgName}}, DepsDir, IsLock, State) -> %% Package dependency with different package name from app name - dep_to_app(Parent, DepsDir, Name, undefined, {pkg, ec_cnv:to_binary(PkgName), undefined, undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, Name, undefined, {pkg, rebar_utils:to_binary(PkgName), undefined, undefined}, IsLock, State); parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_binary(Vsn) -> %% Versioned Package dependency - {PkgName, PkgVsn} = {ec_cnv:to_binary(Name), ec_cnv:to_binary(Vsn)}, + {PkgName, PkgVsn} = {rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)}, dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn, undefined}, IsLock, State); parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> %% Unversioned package dependency - dep_to_app(Parent, DepsDir, ec_cnv:to_binary(Name), undefined, {pkg, ec_cnv:to_binary(Name), undefined, undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, rebar_utils:to_binary(Name), undefined, {pkg, rebar_utils:to_binary(Name), undefined, undefined}, IsLock, State); parse_dep(Parent, {Name, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); parse_dep(Parent, {Name, _Vsn, Source}, DepsDir, IsLock, State) when is_tuple(Source) -> @@ -212,12 +214,12 @@ parse_dep(_, Dep, _, _, _) -> IsLock :: boolean(), State :: rebar_state:t(). dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> - CheckoutsDir = ec_cnv:to_list(rebar_dir:checkouts_dir(State, Name)), + CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of {ok, App} -> rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); not_found -> - Dir = ec_cnv:to_list(filename:join(DepsDir, Name)), + Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), {ok, AppInfo0} = case rebar_app_info:discover(Dir) of {ok, App} -> @@ -275,7 +277,7 @@ fetch_checksum(PkgName, PkgVsn, Hash, State) -> rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) catch _:_ -> - ?INFO("Package ~s-~s not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]), + ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]), {ok, _} = rebar_prv_update:do(State), rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) end. @@ -283,7 +285,7 @@ fetch_checksum(PkgName, PkgVsn, Hash, State) -> %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({missing_package, Package}) -> - io_lib:format("Package not found in registry: ~s", [Package]); + io_lib:format("Package not found in registry: ~ts", [Package]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error(Error) -> @@ -302,7 +304,7 @@ get_package(Dep, Vsn, State) -> {ok, HighestDepVsn} -> {Dep, HighestDepVsn}; none -> - throw(?PRV_ERROR({missing_package, ec_cnv:to_binary(Dep)})) + throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Dep)})) end. %% @private checks that all the beam files have been properly @@ -310,7 +312,7 @@ get_package(Dep, Vsn, State) -> -spec has_all_beams(file:filename_all(), [module()]) -> true | ?PRV_ERROR({missing_module, module()}). has_all_beams(EbinDir, [Module | ModuleList]) -> - BeamFile = filename:join([EbinDir, ec_cnv:to_list(Module) ++ ".beam"]), + BeamFile = filename:join([EbinDir, rebar_utils:to_list(Module) ++ ".beam"]), case filelib:is_file(BeamFile) of true -> has_all_beams(EbinDir, ModuleList); -- cgit v1.1 From 56b7d88975aa8da6446857cfd92de0825024bf63 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 13 Sep 2018 19:36:00 -0600 Subject: support for hex v2, multiple repository fetching, private organizations (#1884) * update to hex_core for hex-v2 repo support (#1865) * update to hex_core for hex-v2 repo support This patch adds only single repo hex-v2 support through hex_core. Packages no longer filtered out by buildtool metadata and the package index is updated per-package instead of fetched as one large ets dump. * tell travis to also build hex_core branch * support list of repos for hex packages (#1866) * support list of repos for hex packages repos are defined under the hex key in rebar configs. They can be defined at the top level of a project or globally, but not in profiles and the repos configured in dependencies are also ignored. Searching for packages involves first checking for a match in the local repo index cache, in the order repos are defined. If not found each repo is checked through the hex api for any known versions of the package and the first repo with a version that fits the constraint is used. * add {repos, replace, []} for overriding the global & default repos * add hex auth handling for repos (#1874) auth token are kept in a hex.config file that is modified by the rebar3 hex plugin. Repo names that have a : separating a parent and child are considered organizations. The parent repo's auth will be included with the child. So an organization named hexpm:rebar3_test will include any hexpm auth tokens found in the rebar3_test organization's configuration. * move packages to top level of of hexpm cache dir (#1876) * move packages to top level of of hexpm cache dir * append organization name to parent's repo_url when parsing repos * only eval config scripts and apply overrides once per app (#1879) * only eval config scripts and apply overrides once per app * move new resource behaviour to rebar_resource_v2 and keep v1 * cleanup use of rebar_resource module and unused functions * cleanup error messages and unused code * when discovering apps support mix packages as unbuilt apps (#1882) * use hex_core tarball unpacking support in pkg resource (#1883) * use hex_core tarball unpacking support in pkg resource * ignore etag if package doesn't exist and delete if checksum fails * add back tests for bad package checksums * improve bad registry checksum error message --- src/rebar_app_utils.erl | 109 +++++++++++++++++------------------------------- 1 file changed, 39 insertions(+), 70 deletions(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 1d7ef5b..35e908c 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -217,26 +217,23 @@ parse_dep(_, Dep, _, _, _) -> dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> CheckoutsDir = rebar_utils:to_list(rebar_dir:checkouts_dir(State, Name)), AppInfo = case rebar_app_info:discover(CheckoutsDir) of - {ok, App} -> - rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); - not_found -> - Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), - {ok, AppInfo0} = - case rebar_app_info:discover(Dir) of - {ok, App} -> - {ok, rebar_app_info:parent(App, Parent)}; - not_found -> - rebar_app_info:new(Parent, Name, Vsn, Dir, []) - end, - rebar_app_info:source(AppInfo0, Source) - end, - C = rebar_config:consult(rebar_app_info:dir(AppInfo)), - AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), C), - Overrides = rebar_state:get(State, overrides, []), - AppInfo2 = rebar_app_info:set(AppInfo1, overrides, rebar_app_info:get(AppInfo, overrides, [])++Overrides), - AppInfo3 = rebar_app_info:apply_overrides(rebar_app_info:get(AppInfo2, overrides, []), AppInfo2), - AppInfo4 = rebar_app_info:apply_profiles(AppInfo3, [default, prod]), - AppInfo5 = rebar_app_info:profiles(AppInfo4, [default]), + {ok, App} -> + rebar_app_info:source(rebar_app_info:is_checkout(App, true), checkout); + not_found -> + Dir = rebar_utils:to_list(filename:join(DepsDir, Name)), + {ok, AppInfo0} = + case rebar_app_info:discover(Dir) of + {ok, App} -> + {ok, rebar_app_info:is_available(rebar_app_info:parent(App, Parent), + true)}; + not_found -> + rebar_app_info:new(Parent, Name, Vsn, Dir, []) + end, + rebar_app_info:source(AppInfo0, Source) + end, + Overrides = rebar_app_info:get(AppInfo, overrides, []) ++ rebar_state:get(State, overrides, []), + AppInfo2 = rebar_app_info:set(AppInfo, overrides, Overrides), + AppInfo5 = rebar_app_info:profiles(AppInfo2, [default]), rebar_app_info:is_lock(AppInfo5, IsLock). %% @doc Takes a given application app_info record along with the project. @@ -250,52 +247,36 @@ expand_deps_sources(Dep, State) -> %% around version if required. -spec update_source(rebar_app_info:t(), Source, rebar_state:t()) -> rebar_app_info:t() when - Source :: tuple() | atom() | binary(). % TODO: meta to source() + Source :: rebar_resource_v2:source(). update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> - {PkgName1, PkgVsn1} = case PkgVsn of - undefined -> - get_package(PkgName, "0", State); - <<"~>", Vsn/binary>> -> - [Vsn1] = [X || X <- binary:split(Vsn, [<<" ">>], [global]), X =/= <<>>], - get_package(PkgName, Vsn1, State); - _ -> - {PkgName, PkgVsn} - end, - %% store the expected hash for the dependency - Hash1 = case Hash of - undefined -> % unknown, define the hash since we know the dep - fetch_checksum(PkgName1, PkgVsn1, Hash, State); - _ -> % keep as is - Hash - end, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName1, PkgVsn1, Hash1}), - Deps = rebar_packages:deps(PkgName1 - ,PkgVsn1 - ,State), - AppInfo2 = rebar_app_info:resource_type(rebar_app_info:deps(AppInfo1, Deps), pkg), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + case rebar_packages:resolve_version(PkgName, PkgVsn, Hash, + ?PACKAGE_TABLE, State) of + {ok, Package, RepoConfig} -> + #package{key={_, PkgVsn1, _}, + checksum=Hash1, + dependencies=Deps} = Package, + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}), + AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps), + rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + not_found -> + throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); + {error, {invalid_vsn, InvalidVsn}} -> + throw(?PRV_ERROR({invalid_vsn, PkgName, InvalidVsn})) + end; update_source(AppInfo, Source, _State) -> rebar_app_info:source(AppInfo, Source). -%% @doc grab the checksum for a given package --spec fetch_checksum(atom(), string(), iodata() | undefined, rebar_state:t()) -> - iodata() | no_return(). -fetch_checksum(PkgName, PkgVsn, Hash, State) -> - try - rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) - catch - _:_ -> - ?INFO("Package ~ts-~ts not found. Fetching registry updates and trying again...", [PkgName, PkgVsn]), - {ok, _} = rebar_prv_update:do(State), - rebar_packages:registry_checksum({pkg, PkgName, PkgVsn, Hash}, State) - end. - %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). -format_error({missing_package, Package}) -> - io_lib:format("Package not found in registry: ~ts", [Package]); +format_error({missing_package, Name, undefined}) -> + io_lib:format("Package not found in any repo: ~ts.", [rebar_utils:to_binary(Name)]); +format_error({missing_package, Name, Vsn}) -> + io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); +format_error({invalid_vsn, Dep, InvalidVsn}) -> + io_lib:format("Dep ~ts has invalid version ~ts", [Dep, InvalidVsn]); format_error(Error) -> io_lib:format("~p", [Error]). @@ -303,18 +284,6 @@ format_error(Error) -> %% Internal functions %% =================================================================== -%% @private find the correct version of a package based on the version -%% and name passed in. --spec get_package(binary(), binary() | string(), rebar_state:t()) -> - term() | no_return(). -get_package(Dep, Vsn, State) -> - case rebar_packages:find_highest_matching(Dep, Vsn, ?PACKAGE_TABLE, State) of - {ok, HighestDepVsn} -> - {Dep, HighestDepVsn}; - none -> - throw(?PRV_ERROR({missing_package, rebar_utils:to_binary(Dep)})) - end. - %% @private checks that all the beam files have been properly %% created. -spec has_all_beams(file:filename_all(), [module()]) -> -- cgit v1.1 From 8bcb7da1dd294f13bc8d2040ec88f0b41b941e1b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 1 Oct 2018 15:41:34 -0600 Subject: warn if the matched package is retired, skip prerelease (#1897) retired packages are now used the same as any other but a warning will be printed when it is resolved. prerelease versions are skipped unless explicitly given as the version in the constraint or lock file. --- src/rebar_app_utils.erl | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 35e908c..fb08ea9 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -254,10 +254,13 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> {ok, Package, RepoConfig} -> #package{key={_, PkgVsn1, _}, checksum=Hash1, - dependencies=Deps} = Package, - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn1, Hash1, RepoConfig}), + dependencies=Deps, + retired=Retired} = Package, + maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired), + PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))), + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}), AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps), - rebar_app_info:original_vsn(AppInfo2, PkgVsn1); + rebar_app_info:original_vsn(AppInfo2, PkgVsn2); not_found -> throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); {error, {invalid_vsn, InvalidVsn}} -> @@ -269,10 +272,10 @@ update_source(AppInfo, Source, _State) -> %% @doc convert a given exception's payload into an io description. -spec format_error(any()) -> iolist(). format_error({missing_package, Name, undefined}) -> - io_lib:format("Package not found in any repo: ~ts.", [rebar_utils:to_binary(Name)]); + io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]); format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts-~ts.", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); + io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), + rebar_utils:to_binary(Vsn)]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error({invalid_vsn, Dep, InvalidVsn}) -> @@ -284,6 +287,32 @@ format_error(Error) -> %% Internal functions %% =================================================================== +maybe_warn_retired(_, _, _, false) -> + ok; +maybe_warn_retired(_, _, Hash, _) when is_binary(Hash) -> + %% don't warn if this is a lock + ok; +maybe_warn_retired(Name, Vsn, _, R=#{reason := Reason}) -> + Message = maps:get(message, R, ""), + ?WARN("Warning: package ~s-~s is retired: (~s) ~s", + [Name, ec_semver:format(Vsn), retire_reason(Reason), Message]); +maybe_warn_retired(_, _, _, _) -> + ok. + +%% TODO: move to hex_core +retire_reason('RETIRED_OTHER') -> + "other"; +retire_reason('RETIRED_INVALID') -> + "invalid"; +retire_reason('RETIRED_SECURITY') -> + "security"; +retire_reason('RETIRED_DEPRECATED') -> + "deprecated"; +retire_reason('RETIRED_RENAMED') -> + "renamed"; +retire_reason(_Other) -> + "other". + %% @private checks that all the beam files have been properly %% created. -spec has_all_beams(file:filename_all(), [module()]) -> -- cgit v1.1 From dec484643c233fda9c17d38c1854ba7f3f37547b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 08:29:07 -0600 Subject: compiler behaviour (#1893) * add compile type for dynamic project compilation * new rebar_compiler abstraction for running multiple compilers rebar_compiler is a new behaviour that a plugin can implement to be called on any ues of the compile provider to compile source files and keep track of their dependencies. * fix check that modules in .app modules list are from src_dirs * use project_type to find module for building projects * allow plugins to add project builders and compilers --- src/rebar_app_utils.erl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index fb08ea9..605944e 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -273,9 +273,8 @@ update_source(AppInfo, Source, _State) -> -spec format_error(any()) -> iolist(). format_error({missing_package, Name, undefined}) -> io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]); -format_error({missing_package, Name, Vsn}) -> - io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name), - rebar_utils:to_binary(Vsn)]); +format_error({missing_package, Name, Constraint}) -> + io_lib:format("Package not found in any repo: ~ts ~ts", [Name, Constraint]); format_error({parse_dep, Dep}) -> io_lib:format("Failed parsing dep ~p", [Dep]); format_error({invalid_vsn, Dep, InvalidVsn}) -> -- cgit v1.1 From 5944b14f570ee3b42debe71f4628c7ef7824984c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Boroska?= Date: Fri, 16 Nov 2018 16:03:59 +0100 Subject: Fix package upgrade (issue #1945) (#1946) --- src/rebar_app_utils.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 605944e..19ed4dd 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -259,8 +259,7 @@ update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired), PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))), AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}), - AppInfo2 = rebar_app_info:update_opts_deps(AppInfo1, Deps), - rebar_app_info:original_vsn(AppInfo2, PkgVsn2); + rebar_app_info:update_opts_deps(AppInfo1, Deps); not_found -> throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); {error, {invalid_vsn, InvalidVsn}} -> -- cgit v1.1 From 73bf5b4c055ef30089dabb3f99d09410af144b7f Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 30 Nov 2018 09:00:08 -0700 Subject: fix for mix projects that don't have app.src files (#1964) --- src/rebar_app_utils.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/rebar_app_utils.erl') diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 19ed4dd..5fe5ba6 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -224,7 +224,8 @@ dep_to_app(Parent, DepsDir, Name, Vsn, Source, IsLock, State) -> {ok, AppInfo0} = case rebar_app_info:discover(Dir) of {ok, App} -> - {ok, rebar_app_info:is_available(rebar_app_info:parent(App, Parent), + App1 = rebar_app_info:name(App, Name), + {ok, rebar_app_info:is_available(rebar_app_info:parent(App1, Parent), true)}; not_found -> rebar_app_info:new(Parent, Name, Vsn, Dir, []) -- cgit v1.1