From 9d788d893620f868b9c0ee00ddec8ae4d5d8fea7 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 5 Oct 2018 07:57:20 -0400 Subject: Abstracted path management Move path management out of rebar_utils manual code path function handling (which we leave there for backwards compat), and centralize them to allow easier coordination of paths between plugins and deps. On top of path handling, do a check of loaded modules to only purge and reload those that actually need it done in order to prevent all kinds of weird interaction and accidental purge kills. It also allows the possible cohabitation of both at once, with a "in case of conflict pick X" as a policy Changing path handling in providers also highlighted a bunch of bugs in some tests and appears to fix some in other providers, specifically around plugins. --- src/rebar_prv_compile.erl | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) (limited to 'src/rebar_prv_compile.erl') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index a509704..405f73a 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -37,10 +37,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> IsDepsOnly = is_deps_only(State), - DepsPaths = rebar_state:code_paths(State, all_deps), - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - rebar_utils:remove_from_code_path(PluginDepsPaths), - code:add_pathsa(DepsPaths), + rebar_paths:set_paths([deps, plugins], State), Providers = rebar_state:providers(State), Deps = rebar_state:deps_to_build(State), @@ -50,11 +47,10 @@ do(State) -> true -> State; false -> - handle_project_apps(DepsPaths, Providers, State) + handle_project_apps(Providers, State) end, - rebar_utils:cleanup_code_path(rebar_state:code_paths(State1, default) - ++ rebar_state:code_paths(State, all_plugin_deps)), + rebar_paths:set_paths([plugins, deps], State1), {ok, State1}. @@ -62,7 +58,7 @@ is_deps_only(State) -> {Args, _} = rebar_state:command_parsed_args(State), proplists:get_value(deps_only, Args, false). -handle_project_apps(DepsPaths, Providers, State) -> +handle_project_apps(Providers, State) -> Cwd = rebar_state:dir(State), ProjectApps = rebar_state:project_apps(State), {ok, ProjectApps1} = rebar_digraph:compile_order(ProjectApps), @@ -76,7 +72,7 @@ handle_project_apps(DepsPaths, Providers, State) -> %% projects with structures like /apps/foo,/apps/bar,/test build_extra_dirs(State, ProjectApps2), - State3 = update_code_paths(State2, ProjectApps2, DepsPaths), + State3 = update_code_paths(State2, ProjectApps2), rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State2), case rebar_state:has_all_artifacts(State3) of @@ -176,11 +172,10 @@ compile(State, Providers, AppInfo) -> %% The rebar_otp_app compilation step is safe regarding the %% overall path management, so we can just load all plugins back %% in memory. - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - code:add_pathsa(PluginDepsPaths), + rebar_paths:set_paths([plugins, deps], State), AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), %% Clean up after ourselves, leave things as they were. - rebar_utils:remove_from_code_path(PluginDepsPaths), + rebar_paths:set_paths([deps, plugins], State), case AppFileCompileResult of {ok, AppInfo5} -> @@ -206,21 +201,22 @@ build_app(AppInfo, State) -> case lists:keyfind(Type, 1, ProjectBuilders) of {_, Module} -> %% load plugins since thats where project builders would be - PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps), - code:add_pathsa(PluginDepsPaths), - case Module:build(AppInfo) of - ok -> - rebar_utils:remove_from_code_path(PluginDepsPaths); - {error, Reason} -> - throw({error, {Module, Reason}}) + rebar_paths:set_paths([plugins, deps], State), + Res = Module:build(AppInfo), + rebar_paths:set_paths([deps, plugins], State), + case Res of + ok -> ok; + {error, Reason} -> throw({error, {Module, Reason}}) end; _ -> throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) end end. -update_code_paths(State, ProjectApps, DepsPaths) -> + +update_code_paths(State, ProjectApps) -> ProjAppsPaths = paths_for_apps(ProjectApps), ExtrasPaths = paths_for_extras(State, ProjectApps), + DepsPaths = rebar_state:code_paths(State, all_deps), rebar_state:code_paths(State, all_deps, DepsPaths ++ ProjAppsPaths ++ ExtrasPaths). paths_for_apps(Apps) -> paths_for_apps(Apps, []). -- cgit v1.1 From 311ee6b1371c3eea3611dc5d7945b1b5667c75bd Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 6 Oct 2018 11:38:33 -0400 Subject: Fix a bug in compiler path handling Also handle some formatting --- src/rebar_prv_compile.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rebar_prv_compile.erl') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 405f73a..4b36636 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -115,7 +115,7 @@ copy_and_build_project_apps(State, Providers, Apps) -> rebar_app_info:dir(AppInfo), rebar_app_info:out_dir(AppInfo)) || AppInfo <- Apps], - code:add_pathsa([rebar_app_info:out_dir(AppInfo) || AppInfo <- Apps]), + code:add_pathsa([rebar_app_info:ebin_dir(AppInfo) || AppInfo <- Apps]), [compile(State, Providers, AppInfo) || AppInfo <- Apps]. -- cgit v1.1 From dada4e36e6d9a5c4b41bbe1f68389520e7c59ace Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 11 Oct 2018 08:38:37 -0400 Subject: Optimize path handling - Only set paths that need to be put as a priority - Clean up paths before leaving API mode The first point accounted for some performance cost, but the latter one explains the 40% overhead in test runs: since rebar3 calls rebar3 a lot with a bunch of fake apps, and that the new mechanism for path handling by default does not _remove_ paths, it just _orders_ them, we would end up in a situation where as the tests ran, more and more fake paths would get added to the VM. By the time the run was over, all path handling would take longer since more paths needed filtering every time. By resetting paths at the end of an API run, we prevent a given 'project' from polluting another one's runtime and performance once the API successfully returns. --- src/rebar_prv_compile.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/rebar_prv_compile.erl') diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 4b36636..ee96d9f 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -37,7 +37,7 @@ init(State) -> -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> IsDepsOnly = is_deps_only(State), - rebar_paths:set_paths([deps, plugins], State), + rebar_paths:set_paths([deps], State), Providers = rebar_state:providers(State), Deps = rebar_state:deps_to_build(State), @@ -50,7 +50,7 @@ do(State) -> handle_project_apps(Providers, State) end, - rebar_paths:set_paths([plugins, deps], State1), + rebar_paths:set_paths([plugins], State1), {ok, State1}. @@ -172,10 +172,10 @@ compile(State, Providers, AppInfo) -> %% The rebar_otp_app compilation step is safe regarding the %% overall path management, so we can just load all plugins back %% in memory. - rebar_paths:set_paths([plugins, deps], State), + rebar_paths:set_paths([plugins], State), AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), - %% Clean up after ourselves, leave things as they were. - rebar_paths:set_paths([deps, plugins], State), + %% Clean up after ourselves, leave things as they were with deps first + rebar_paths:set_paths([deps], State), case AppFileCompileResult of {ok, AppInfo5} -> @@ -203,7 +203,7 @@ build_app(AppInfo, State) -> %% load plugins since thats where project builders would be rebar_paths:set_paths([plugins, deps], State), Res = Module:build(AppInfo), - rebar_paths:set_paths([deps, plugins], State), + rebar_paths:set_paths([deps], State), case Res of ok -> ok; {error, Reason} -> throw({error, {Module, Reason}}) -- cgit v1.1