Merge tag '3.6.0' into refresh_paths

Bump to 3.6.0

- Introduce support of add and del operations for overrides
- OTP-21 compatibility
- Bump cth_readable for OTP-21 compat
- upgrade relx to 3.25.0
- upgrade bbmustache to 1.5.0
- run compile provider in default namespace from bare compiling (fixes
  hooks for mix builds)
- Resolve string vs list crashing rebar3 in erl_first_files
- Create ERLANG_LIB_*_erl_interface environment variables only if erl_interface exists
- Add hooks to the upgrade command
- Add --start-clean flag to rebar3 shell
- Auto-boot main app in OTP app project templates
- Use maps for child spec examples in templates

11 files changed, 162 insertions, 66 deletions

diff --git a/src/rebar.app.src b/src/rebar.app.src
index 42aecca..5c76f59 100644
--- a/src/rebar.app.src
+++ b/src/rebar.app.src
@@ -3,7 +3,7 @@
 
 {application, rebar,
  [{description, "Rebar: Erlang Build Tool"},
-  {vsn, "3.5.2"},
+  {vsn, "3.6.0"},
   {modules, []},
   {registered, []},
   {applications, [kernel,
diff --git a/src/rebar3.erl b/src/rebar3.erl
index 8e9d4b1..ec8e953 100644
--- a/src/rebar3.erl
+++ b/src/rebar3.erl
@@ -54,7 +54,7 @@
 %% ====================================================================
 
 %% @doc For running with:
-%% erl +sbtu +A0 -noinput -mode minimal -boot start_clean -s rebar3 main -extra "$@"
+%% erl +sbtu +A1 -noinput -mode minimal -boot start_clean -s rebar3 main -extra "$@"
 -spec main() -> no_return().
 main() ->
     List = init:get_plain_arguments(),
@@ -93,6 +93,7 @@ run(BaseState, Commands) ->
 %% arguments passed, if they have any relevance; used to translate
 %% from the escript call-site into a common one with the library
 %% usage.
+-spec run([any(), ...]) -> {ok, rebar_state:t()} | {error, term()}.
 run(RawArgs) ->
     start_and_load_apps(command_line),
 
@@ -239,6 +240,7 @@ parse_args([Task | RawRest]) ->
     {list_to_atom(Task), RawRest}.
 
 %% @private actually not too sure what this does anymore.
+-spec set_options(rebar_state:t(),{[any()],[any()]}) -> {rebar_state:t(),[any()]}.
 set_options(State, {Options, NonOptArgs}) ->
     GlobalDefines = proplists:get_all_values(defines, Options),
 
@@ -387,6 +389,7 @@ state_from_global_config(Config, GlobalConfigFile) ->
     GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])),
     rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins).
 
+-spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}].
 test_state(State) ->
     %% Fetch the test profile's erl_opts only
     Opts = rebar_state:opts(State),
@@ -396,6 +399,7 @@ test_state(State) ->
     TestOpts = safe_define_test_macro(ErlOpts),
     [{extra_src_dirs, ["test"]}, {erl_opts, TestOpts}].
 
+-spec safe_define_test_macro([any()]) -> [any()] | [{'d',atom()} | any()].
 safe_define_test_macro(Opts) ->
     %% defining a compile macro twice results in an exception so
     %% make sure 'TEST' is only defined once
@@ -404,6 +408,7 @@ safe_define_test_macro(Opts) ->
        false -> [{d, 'TEST'}|Opts]
     end.
 
+-spec test_defined([{d, atom()} | {d, atom(), term()} | term()]) -> boolean().
 test_defined([{d, 'TEST'}|_]) -> true;
 test_defined([{d, 'TEST', true}|_]) -> true;
 test_defined([_|Rest]) -> test_defined(Rest);
diff --git a/src/rebar_env.erl b/src/rebar_env.erl
index eea47de..e9adafb 100644
--- a/src/rebar_env.erl
+++ b/src/rebar_env.erl
@@ -37,26 +37,40 @@ create_env(State) ->
 -spec create_env(rebar_state:t(), rebar_dict()) -> proplists:proplist().
 create_env(State, Opts) ->
     BaseDir = rebar_dir:base_dir(State),
-    [
-     {"REBAR_DEPS_DIR",          filename:absname(rebar_dir:deps_dir(State))},
-     {"REBAR_BUILD_DIR",         filename:absname(rebar_dir:base_dir(State))},
-     {"REBAR_ROOT_DIR",          filename:absname(rebar_dir:root_dir(State))},
-     {"REBAR_CHECKOUTS_DIR",     filename:absname(rebar_dir:checkouts_dir(State))},
-     {"REBAR_PLUGINS_DIR",       filename:absname(rebar_dir:plugins_dir(State))},
-     {"REBAR_GLOBAL_CONFIG_DIR", filename:absname(rebar_dir:global_config_dir(State))},
-     {"REBAR_GLOBAL_CACHE_DIR",  filename:absname(rebar_dir:global_cache_dir(Opts))},
-     {"REBAR_TEMPLATE_DIR",      filename:absname(rebar_dir:template_dir(State))},
-     {"REBAR_APP_DIRS",          join_dirs(BaseDir, rebar_dir:lib_dirs(State))},
-     {"REBAR_SRC_DIRS",          join_dirs(BaseDir, rebar_dir:all_src_dirs(Opts))},
-     {"ERLANG_ERTS_VER",         erlang:system_info(version)},
-     {"ERLANG_ROOT_DIR",         code:root_dir()},
-     {"ERLANG_LIB_DIR_erl_interface", code:lib_dir(erl_interface)},
-     {"ERLANG_LIB_VER_erl_interface", re_version(code:lib_dir(erl_interface))},
-     {"ERL",                     filename:join([code:root_dir(), "bin", "erl"])},
-     {"ERLC",                    filename:join([code:root_dir(), "bin", "erlc"])},
-     {"ERLANG_ARCH"  ,           rebar_api:wordsize()},
-     {"ERLANG_TARGET",           rebar_api:get_arch()}
-    ].
+    EnvVars = [
+        {"REBAR_DEPS_DIR",          filename:absname(rebar_dir:deps_dir(State))},
+        {"REBAR_BUILD_DIR",         filename:absname(rebar_dir:base_dir(State))},
+        {"REBAR_ROOT_DIR",          filename:absname(rebar_dir:root_dir(State))},
+        {"REBAR_CHECKOUTS_DIR",     filename:absname(rebar_dir:checkouts_dir(State))},
+        {"REBAR_PLUGINS_DIR",       filename:absname(rebar_dir:plugins_dir(State))},
+        {"REBAR_GLOBAL_CONFIG_DIR", filename:absname(rebar_dir:global_config_dir(State))},
+        {"REBAR_GLOBAL_CACHE_DIR",  filename:absname(rebar_dir:global_cache_dir(Opts))},
+        {"REBAR_TEMPLATE_DIR",      filename:absname(rebar_dir:template_dir(State))},
+        {"REBAR_APP_DIRS",          join_dirs(BaseDir, rebar_dir:lib_dirs(State))},
+        {"REBAR_SRC_DIRS",          join_dirs(BaseDir, rebar_dir:all_src_dirs(Opts))},
+        {"ERLANG_ERTS_VER",         erlang:system_info(version)},
+        {"ERLANG_ROOT_DIR",         code:root_dir()},
+        {"ERL",                     filename:join([code:root_dir(), "bin", "erl"])},
+        {"ERLC",                    filename:join([code:root_dir(), "bin", "erlc"])},
+        {"ERLANG_ARCH"  ,           rebar_api:wordsize()},
+        {"ERLANG_TARGET",           rebar_api:get_arch()}
+    ],
+    EInterfaceVars = create_erl_interface_env(),
+    lists:append([EnvVars, EInterfaceVars]).
+
+-spec create_erl_interface_env() -> list().
+create_erl_interface_env() ->
+    case code:lib_dir(erl_interface) of
+        {error, bad_name} ->
+            ?WARN("erl_interface is missing. ERLANG_LIB_DIR_erl_interface and "
+            "ERLANG_LIB_VER_erl_interface will not be added to the environment.", []),
+            [];
+        Dir ->
+            [
+             {"ERLANG_LIB_DIR_erl_interface", Dir},
+             {"ERLANG_LIB_VER_erl_interface", re_version(Dir)}
+            ]
+    end.
 
 %% ====================================================================
 %% Internal functions
diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl
index c588a25..ebdd9dd 100644
--- a/src/rebar_erlc_compiler.erl
+++ b/src/rebar_erlc_compiler.erl
@@ -285,6 +285,7 @@ gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) ->
 %% files, so that yet to be compiled parse transformations are excluded from it.
 erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) ->
     ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []),
+    valid_erl_first_conf(ErlFirstFilesConf),
     NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)),
     %% NOTE: order of files here is important!
     ErlFirstFiles =
@@ -796,3 +797,10 @@ dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) ->
         undefined -> rebar_dir:recursive(Opts, Dir);
         Recursive -> Recursive
     end.
+
+valid_erl_first_conf(FileList) ->
+    case rebar_utils:is_list_of_strings(FileList) of
+        true -> true;
+        false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_files_first directive",
+                        [FileList])
+    end.
diff --git a/src/rebar_opts.erl b/src/rebar_opts.erl
index 1a927ba..8195a77 100644
--- a/src/rebar_opts.erl
+++ b/src/rebar_opts.erl
@@ -73,36 +73,42 @@ filter_debug_info([H|T]) ->
 apply_overrides(Opts, Name, Overrides) ->
     %% Inefficient. We want the order we get here though.
     Opts1 = lists:foldl(fun({override, O}, OptsAcc) ->
-                                 lists:foldl(fun({deps, Value}, OptsAcc1) ->
-                                                     set(OptsAcc1, {deps,default}, Value);
-                                                ({Key, Value}, OptsAcc1) ->
-                                                     set(OptsAcc1, Key, Value)
-                                             end, OptsAcc, O);
+                                override_opt(O, OptsAcc);
                             (_, OptsAcc) ->
                                  OptsAcc
-                         end, Opts, Overrides),
-
-    Opts2 = lists:foldl(fun({override, N, O}, OptsAcc) when N =:= Name ->
-                                 lists:foldl(fun({deps, Value}, OptsAcc1) ->
-                                                     set(OptsAcc1, {deps,default}, Value);
-                                                ({Key, Value}, OptsAcc1) ->
-                                                     set(OptsAcc1, Key, Value)
-                                             end, OptsAcc, O);
+                        end, Opts, Overrides),
+
+    Opts2 = lists:foldl(fun({add, O}, OptsAcc) ->
+                                add_opt(O, OptsAcc);
+                           (_, OptsAcc) ->
+                                OptsAcc
+                        end, Opts1, Overrides),
+
+    Opts3 = lists:foldl(fun({del, O}, OptsAcc) ->
+                                del_opt(O, OptsAcc);
                             (_, OptsAcc) ->
                                  OptsAcc
-                         end, Opts1, Overrides),
-
-    lists:foldl(fun({add, N, O}, OptsAcc) when N =:= Name ->
-                        lists:foldl(fun({deps, Value}, OptsAcc1) ->
-                                            OldValue = ?MODULE:get(OptsAcc1, {deps,default}, []),
-                                            set(OptsAcc1, {deps,default}, Value++OldValue);
-                                       ({Key, Value}, OptsAcc1) ->
-                                            OldValue = ?MODULE:get(OptsAcc1, Key, []),
-                                            set(OptsAcc1, Key, Value++OldValue)
-                                    end, OptsAcc, O);
-                   (_, OptsAcc) ->
-                        OptsAcc
-                end, Opts2, Overrides).
+                        end, Opts2, Overrides),
+
+    Opts4 = lists:foldl(fun({override, N, O}, OptsAcc) when N =:= Name ->
+                                override_opt(O, OptsAcc);
+                            (_, OptsAcc) ->
+                                 OptsAcc
+                        end, Opts3, Overrides),
+
+    Opts5 = lists:foldl(fun({add, N, O}, OptsAcc) when N =:= Name ->
+                                add_opt(O, OptsAcc);
+                           (_, OptsAcc) ->
+                                OptsAcc
+                        end, Opts4, Overrides),
+
+    Opts6 = lists:foldl(fun({del, N, O}, OptsAcc) when N =:= Name ->
+                                del_opt(O, OptsAcc);
+                            (_, OptsAcc) ->
+                                 OptsAcc
+                        end, Opts5, Overrides),
+
+    Opts6.
 
 add_to_profile(Opts, Profile, KVs) when is_atom(Profile), is_list(KVs) ->
     Profiles = ?MODULE:get(Opts, profiles, []),
@@ -133,6 +139,31 @@ merge_opts(NewOpts, OldOpts) ->
 
 %% Internal functions
 
+add_opt(Opts1, Opts2) ->
+    lists:foldl(fun({deps, Value}, OptsAcc) ->
+                        OldValue = ?MODULE:get(OptsAcc, {deps,default}, []),
+                        set(OptsAcc, {deps,default}, Value++OldValue);
+                    ({Key, Value}, OptsAcc) ->
+                        OldValue = ?MODULE:get(OptsAcc, Key, []),
+                        set(OptsAcc, Key, Value++OldValue)
+                end, Opts2, Opts1).
+
+del_opt(Opts1, Opts2) ->
+    lists:foldl(fun({deps, Value}, OptsAcc) ->
+                        OldValue = ?MODULE:get(OptsAcc, {deps,default}, []),
+                        set(OptsAcc, {deps,default}, OldValue--Value);
+                   ({Key, Value}, OptsAcc) ->
+                        OldValue = ?MODULE:get(OptsAcc, Key, []),
+                        set(OptsAcc, Key, OldValue--Value)
+                end, Opts2, Opts1).
+
+override_opt(Opts1, Opts2) ->
+    lists:foldl(fun({deps, Value}, OptsAcc) ->
+                        set(OptsAcc, {deps,default}, Value);
+                   ({Key, Value}, OptsAcc) ->
+                        set(OptsAcc, Key, Value)
+                end, Opts2, Opts1).
+
 %%
 %% Function for dict:merge/3 (in merge_opts/2) to merge options by priority.
 %%
diff --git a/src/rebar_prv_bare_compile.erl b/src/rebar_prv_bare_compile.erl
index c29a711..5d3e977 100644
--- a/src/rebar_prv_bare_compile.erl
+++ b/src/rebar_prv_bare_compile.erl
@@ -46,7 +46,9 @@ do(State) ->
 
     [AppInfo] = rebar_state:project_apps(State),
     AppInfo1 = rebar_app_info:out_dir(AppInfo, rebar_dir:get_cwd()),
-    rebar_prv_compile:compile(State, AppInfo1),
+
+    %% run compile in the default namespace
+    rebar_prv_compile:compile(rebar_state:namespace(State, default), AppInfo1),
 
     rebar_utils:cleanup_code_path(OrigPath),
 
diff --git a/src/rebar_prv_common_test.erl b/src/rebar_prv_common_test.erl
index f800610..9e71ee7 100644
--- a/src/rebar_prv_common_test.erl
+++ b/src/rebar_prv_common_test.erl
@@ -227,15 +227,20 @@ add_hooks(Opts, State) ->
     case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of
         {false, _} ->
             Opts;
-        {true, false} ->
-            [{ct_hooks, [cth_readable_failonly, cth_readable_shell, cth_retry]} | Opts];
-        {true, {ct_hooks, Hooks}} ->
+        {Other, false} ->
+            [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), cth_retry]} | Opts];
+        {Other, {ct_hooks, Hooks}} ->
             %% Make sure hooks are there once only.
-            ReadableHooks = [cth_readable_failonly, cth_readable_shell, cth_retry],
-            NewHooks =  (Hooks -- ReadableHooks) ++ ReadableHooks,
+            ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), cth_retry],
+            AllReadableHooks = [cth_readable_failonly, cth_retry,
+                                cth_readable_shell, cth_readable_compact_shell],
+            NewHooks =  (Hooks -- AllReadableHooks) ++ ReadableHooks,
             lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks})
     end.
 
+readable_shell_type(true) -> cth_readable_shell;
+readable_shell_type(compact) -> cth_readable_compact_shell.
+
 select_tests(_, _, _, {error, _} = Error) -> Error;
 select_tests(State, ProjectApps, CmdOpts, CfgOpts) ->
     %% set application env if sys_config argument is provided
@@ -425,20 +430,21 @@ append(A, B) -> A ++ B.
 
 add_transforms(CTOpts, State) when is_list(CTOpts) ->
     case readable(State) of
-        true ->
-            ReadableTransform = [{parse_transform, cth_readable_transform}],
-            (CTOpts -- ReadableTransform) ++ ReadableTransform;
         false ->
-            CTOpts
+            CTOpts;
+        Other when Other == true; Other == compact ->
+            ReadableTransform = [{parse_transform, cth_readable_transform}],
+            (CTOpts -- ReadableTransform) ++ ReadableTransform
     end;
 add_transforms({error, _} = Error, _State) -> Error.
 
 readable(State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
     case proplists:get_value(readable, RawOpts) of
-        true  -> true;
-        false -> false;
-        undefined -> rebar_state:get(State, ct_readable, true)
+        "true"  -> true;
+        "false" -> false;
+        "compact" -> compact;
+        undefined -> rebar_state:get(State, ct_readable, compact)
     end.
 
 test_dirs(State, Apps, Opts) ->
@@ -762,7 +768,7 @@ ct_opts(_State) ->
      {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)},
      {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)},
      {include, undefined, "include", string, help(include)},
-     {readable, undefined, "readable", boolean, help(readable)},
+     {readable, undefined, "readable", string, help(readable)},
      {verbose, $v, "verbose", boolean, help(verbose)},
      {name, undefined, "name", atom, help(name)},
      {sname, undefined, "sname", atom, help(sname)},
@@ -831,7 +837,7 @@ help(create_priv_dir) ->
 help(include) ->
     "Directories containing additional include files";
 help(readable) ->
-    "Shows test case names and only displays logs to shell on failures";
+    "Shows test case names and only displays logs to shell on failures (true | compact | false)";
 help(verbose) ->
     "Verbose output";
 help(name) ->
diff --git a/src/rebar_prv_local_install.erl b/src/rebar_prv_local_install.erl
index bb019c4..c41812f 100644
--- a/src/rebar_prv_local_install.erl
+++ b/src/rebar_prv_local_install.erl
@@ -64,7 +64,7 @@ format_error(Reason) ->
 bin_contents(OutputDir) ->
     <<"#!/usr/bin/env sh
 
-erl -pz ", (rebar_utils:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A0 -noshell -boot start_clean -s rebar3 main $REBAR3_ERL_ARGS -extra \"$@\"
+erl -pz ", (rebar_utils:to_binary(OutputDir))/binary,"/*/ebin +sbtu +A1 -noshell -boot start_clean -s rebar3 main $REBAR3_ERL_ARGS -extra \"$@\"
 ">>.
 
 extract_escript(State, ScriptPath) ->
diff --git a/src/rebar_prv_shell.erl b/src/rebar_prv_shell.erl
index 9a320ad..af8d99f 100644
--- a/src/rebar_prv_shell.erl
+++ b/src/rebar_prv_shell.erl
@@ -76,6 +76,9 @@ init(State) ->
                          "shell. (E.g. --apps app1,app2,app3) Defaults "
                          "to rebar.config {shell, [{apps, Apps}]} or "
                          "relx apps if not specified."},
+                        {start_clean, undefined, "start-clean", boolean,
+                         "Cancel any applications in the 'apps' list "
+                         "or release."},
                         {user_drv_args, undefined, "user_drv_args", string,
                          "Arguments passed to user_drv start function for "
                          "creating custom shells."}]}
@@ -311,7 +314,12 @@ find_apps_option(State) ->
     {Opts, _} = rebar_state:command_parsed_args(State),
     case debug_get_value(apps, Opts, no_value,
                          "Found shell apps from command line option.") of
-        no_value -> no_value;
+        no_value ->
+            case debug_get_value(start_clean, Opts, false,
+                                "Found start-clean argument to disable apps") of
+                false -> no_value;
+                true -> []
+            end;
         AppsStr ->
             [ list_to_atom(AppStr)
               || AppStr <- rebar_string:lexemes(AppsStr, " ,:") ]
diff --git a/src/rebar_prv_upgrade.erl b/src/rebar_prv_upgrade.erl
index cd75672..e4469cf 100644
--- a/src/rebar_prv_upgrade.erl
+++ b/src/rebar_prv_upgrade.erl
@@ -43,6 +43,19 @@ init(State) ->
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
+    Cwd = rebar_state:dir(State),
+    Providers = rebar_state:providers(State),
+    rebar_hooks:run_project_and_app_hooks(Cwd, pre, ?PROVIDER, Providers, State),
+    case do_(State) of
+        {ok, NewState} ->
+            rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, NewState),
+            {ok, NewState};
+        Other ->
+            rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State),
+            Other
+    end.
+
+do_(State) ->
     {Args, _} = rebar_state:command_parsed_args(State),
     Locks = rebar_state:get(State, {locks, default}, []),
     %% We have 3 sources of dependencies to upgrade from:
diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl
index 604abb8..2ded481 100644
--- a/src/rebar_utils.erl
+++ b/src/rebar_utils.erl
@@ -73,7 +73,8 @@
          list_dir/1,
          user_agent/0,
          reread_config/1,
-         get_proxy_auth/0]).
+         get_proxy_auth/0,
+         is_list_of_strings/1]).
 
 
 %% for internal use only
@@ -919,3 +920,11 @@ get_proxy_auth() ->
         undefined -> [];
         {ok, ProxyAuth} -> ProxyAuth
     end.
+
+-spec rebar_utils:is_list_of_strings(term()) -> boolean().
+is_list_of_strings(List) when not is_list(hd(List)) ->
+    false;
+is_list_of_strings(List) when is_list(hd(List)) ->
+    true;
+is_list_of_strings(List) when is_list(List) ->
+    true.