add `--app=...` and `--suite=...` options for the eunit provider to

run subsets of test suites

2 files changed, 341 insertions, 76 deletions

diff --git a/src/rebar_prv_eunit.erl b/src/rebar_prv_eunit.erl
index 0e0e937..8ac0187 100644
--- a/src/rebar_prv_eunit.erl
+++ b/src/rebar_prv_eunit.erl
@@ -37,13 +37,14 @@ init(State) ->
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
     ?INFO("Performing EUnit tests...", []),
-    {Opts, _} = rebar_state:command_parsed_args(State),
-    EUnitOpts = resolve_eunit_opts(State, Opts),
-    TestApps = filter_checkouts(rebar_state:project_apps(State)),
-    ok = compile_tests(State, TestApps),
-    ok = maybe_cover_compile(State, Opts),
-    AppsToTest = test_dirs(State, TestApps),
-    Result = eunit:test(AppsToTest, EUnitOpts),
+    case prepare_tests(State) of
+        {ok, Tests} -> do_tests(State, Tests);
+        Error       -> Error
+    end.
+
+do_tests(State, Tests) ->
+    EUnitOpts = resolve_eunit_opts(State),
+    Result = eunit:test(Tests, EUnitOpts),
     ok = rebar_prv_cover:maybe_write_coverdata(State, ?PROVIDER),
     case handle_results(Result) of
         {error, Reason} ->
@@ -58,45 +59,9 @@ format_error(unknown_error) ->
 format_error({error_running_tests, Reason}) ->
     io_lib:format("Error running tests: ~p", [Reason]).
 
-eunit_opts(_State) ->
-    [{cover, $c, "cover", boolean, help(cover)},
-     {verbose, $v, "verbose", boolean, help(verbose)}].
-
-help(cover) -> "Generate cover data";
-help(verbose) -> "Verbose output".
-
-filter_checkouts(Apps) -> filter_checkouts(Apps, []).
-
-filter_checkouts([], Acc) -> lists:reverse(Acc);
-filter_checkouts([App|Rest], Acc) ->
-    AppDir = filename:absname(rebar_app_info:dir(App)),
-    CheckoutsDir = filename:absname("_checkouts"),
-    case lists:prefix(CheckoutsDir, AppDir) of
-        true -> filter_checkouts(Rest, Acc);
-        false -> filter_checkouts(Rest, [App|Acc])
-    end.
-
-resolve_eunit_opts(State, Opts) ->
-    EUnitOpts = rebar_state:get(State, eunit_opts, []),
-    case proplists:get_value(verbose, Opts, false) of
-        true -> set_verbose(EUnitOpts);
-        false -> EUnitOpts
-    end.
-
-test_dirs(State, TestApps) ->
-    %% we need to add "./ebin" if it exists but only if it's not already
-    %%  due to be added
-    F = fun(App) -> rebar_app_info:dir(App) =/= rebar_dir:get_cwd() end,
-    BareEbin = filename:join([rebar_dir:base_dir(State), "ebin"]),
-    case lists:any(F, TestApps) andalso filelib:is_dir(BareEbin) of
-        false -> application_dirs(TestApps, []);
-        true  -> [{dir, BareEbin}|application_dirs(TestApps, [])]
-    end.
-
-application_dirs([], Acc) -> lists:reverse(Acc);
-application_dirs([App|Rest], Acc) ->
-    AppName = list_to_atom(binary_to_list(rebar_app_info:name(App))),
-    application_dirs(Rest, [{application, AppName}|Acc]).
+%% ===================================================================
+%% Internal functions
+%% ===================================================================
 
 test_state(State) ->
     ErlOpts = rebar_state:get(State, eunit_compile_opts, []),
@@ -120,14 +85,33 @@ first_files(State) ->
     EUnitFirst = rebar_state:get(State, eunit_first_files, []),
     [{erl_first_files, EUnitFirst}].
 
-set_verbose(Opts) ->
-    %% if `verbose` is already set don't set it again
-    case lists:member(verbose, Opts) of
-        true -> Opts;
-        false -> [verbose] ++ Opts
+prepare_tests(State) ->
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    resolve_apps(State, RawOpts).
+
+resolve_apps(State, RawOpts) ->
+    case proplists:get_value(app, RawOpts) of
+        undefined -> resolve_suites(State, project_apps(State), RawOpts);
+        %% convert app name strings to `rebar_app_info` objects
+        Apps      -> AppNames = string:tokens(Apps, [$,]),
+                     ProjectApps = project_apps(State),
+                     case filter_apps_by_name(AppNames, ProjectApps) of
+                         {ok, TestApps} -> resolve_suites(State, TestApps, RawOpts);
+                         Error          -> Error
+                     end
     end.
 
-compile_tests(State, TestApps) ->
+resolve_suites(State, Apps, RawOpts) ->
+    case proplists:get_value(suite, RawOpts) of
+        undefined  -> compile_tests(State, Apps, all, RawOpts);
+        Suites     -> SuiteNames = string:tokens(Suites, [$,]),
+                      case filter_suites_by_apps(SuiteNames, Apps) of
+                          {ok, S} -> compile_tests(State, Apps, S, RawOpts);
+                          Error   -> Error
+                      end
+    end.
+
+compile_tests(State, TestApps, Suites, RawOpts) ->
     F = fun(AppInfo) ->
         AppDir = rebar_app_info:dir(AppInfo),
         S = case rebar_app_info:state(AppInfo) of
@@ -141,24 +125,82 @@ compile_tests(State, TestApps) ->
                                          ec_cnv:to_list(rebar_app_info:out_dir(AppInfo)))
     end,
     lists:foreach(F, TestApps),
-    case filelib:is_dir(filename:join([rebar_dir:get_cwd(), "test"])) of
-        true  -> compile_bare_tests(State, TestApps);
-        false -> ok
+    ok = maybe_cover_compile(State, RawOpts),
+    {ok, test_set(TestApps, Suites)}.
+
+maybe_cover_compile(State, Opts) ->
+    State1 = case proplists:get_value(cover, Opts, false) of
+        true  -> rebar_state:set(State, cover_enabled, true);
+        false -> State
+    end,
+    rebar_prv_cover:maybe_cover_compile(State1).
+
+project_apps(State) ->
+    filter_checkouts(rebar_state:project_apps(State)).
+
+filter_checkouts(Apps) -> filter_checkouts(Apps, []).
+
+filter_checkouts([], Acc) -> lists:reverse(Acc);
+filter_checkouts([App|Rest], Acc) ->
+    AppDir = filename:absname(rebar_app_info:dir(App)),
+    CheckoutsDir = filename:absname("_checkouts"),
+    case lists:prefix(CheckoutsDir, AppDir) of
+        true -> filter_checkouts(Rest, Acc);
+        false -> filter_checkouts(Rest, [App|Acc])
+    end.
+
+%% make sure applications specified actually exist
+filter_apps_by_name(AppNames, ProjectApps) ->
+    filter_apps_by_name(AppNames, ProjectApps, []).
+
+filter_apps_by_name([], _ProjectApps, Acc) -> {ok, lists:reverse(Acc)};
+filter_apps_by_name([Name|Rest], ProjectApps, Acc) ->
+    case find_app_by_name(Name, ProjectApps) of
+        {error, app_not_found} ->
+            ?PRV_ERROR({error_running_tests,
+                        "Application `" ++ Name ++ "' not found in project."});
+        App ->
+            filter_apps_by_name(Rest, ProjectApps, [App|Acc])
+    end.
+
+find_app_by_name(_, []) -> {error, app_not_found};
+find_app_by_name(Name, [App|Rest]) ->
+    case Name == binary_to_list(rebar_app_info:name(App)) of
+        true  -> App;
+        false -> find_app_by_name(Name, Rest)
     end.
 
-compile_bare_tests(State, TestApps) ->
-    F = fun(App) -> rebar_app_info:dir(App) == rebar_dir:get_cwd() end,
-    case lists:filter(F, TestApps) of
-        %% compile and link just the `test` directory of the base dir
-        [] ->
-            Source = filename:join([rebar_dir:get_cwd(), "test"]),
-            Target = filename:join([rebar_dir:base_dir(State), "test"]),
-            ok = rebar_file_utils:symlink_or_copy(Source, Target),
-            rebar_erlc_compiler:compile(replace_src_dirs(State),
-                                        rebar_dir:base_dir(State),
-                                        filename:join([rebar_dir:base_dir(State), "ebin"]));
-        %% already compiled `./test` so do nothing
-        _  -> ok
+%% ensure specified suites are in the applications included
+filter_suites_by_apps(Suites, ProjectApps) ->
+    filter_suites_by_apps(Suites, ProjectApps, []).
+
+filter_suites_by_apps([], _ProjectApps, Acc) -> {ok, lists:reverse(Acc)};
+filter_suites_by_apps([Suite|Rest], Apps, Acc) ->
+    Modules = app_modules([binary_to_atom(rebar_app_info:name(A), unicode) || A <- Apps], []),
+    case lists:member(list_to_atom(Suite), Modules) of
+        false ->
+            ?PRV_ERROR({error_running_tests,
+                        "Module `" ++ Suite ++ "' not found in applications."});
+        true  ->
+            filter_suites_by_apps(Rest, Apps, [Suite|Acc])
+    end.
+
+app_modules([], Acc) -> Acc;
+app_modules([App|Rest], Acc) ->
+    Unload = case application:load(App) of
+        ok                           -> true;
+        {error, {already_loaded, _}} -> false
+    end,
+    NewAcc = case application:get_key(App, modules) of
+        {ok, Modules} -> Modules ++ Acc;
+        undefined     -> Acc
+    end,
+    case Unload of
+        true  ->
+            application:unload(App),
+            app_modules(Rest, NewAcc);
+        false ->
+            app_modules(Rest, NewAcc)
     end.
 
 replace_src_dirs(State) ->
@@ -167,15 +209,44 @@ replace_src_dirs(State) ->
     StrippedOpts = lists:keydelete(src_dirs, 1, ErlOpts),
     rebar_state:set(State, erl_opts, [{src_dirs, ["test"]}|StrippedOpts]).
 
-maybe_cover_compile(State, Opts) ->
-    State1 = case proplists:get_value(cover, Opts, false) of
-        true  -> rebar_state:set(State, cover_enabled, true);
-        false -> State
-    end,
-    rebar_prv_cover:maybe_cover_compile(State1).
+test_set(Apps, Suites) -> test_set(Apps, Suites, []).
+
+test_set([], all, Acc) -> lists:reverse(Acc);
+test_set(_, [], Acc) -> lists:reverse(Acc);
+test_set([App|Rest], all, Acc) ->
+    AppName = list_to_atom(binary_to_list(rebar_app_info:name(App))),
+    test_set(Rest, all, [{application, AppName}|Acc]);
+test_set(Apps, [Suite|Rest], Acc) ->
+    test_set(Apps, Rest, [{module, list_to_atom(Suite)}|Acc]).
+
+resolve_eunit_opts(State) ->
+    {Opts, _} = rebar_state:command_parsed_args(State),
+    EUnitOpts = rebar_state:get(State, eunit_opts, []),
+    case proplists:get_value(verbose, Opts, false) of
+        true  -> set_verbose(EUnitOpts);
+        false -> EUnitOpts
+    end.
+
+set_verbose(Opts) ->
+    %% if `verbose` is already set don't set it again
+    case lists:member(verbose, Opts) of
+        true  -> Opts;
+        false -> [verbose] ++ Opts
+    end.
 
 handle_results(ok) -> ok;
 handle_results(error) ->
     {error, unknown_error};
 handle_results({error, Reason}) ->
     {error, {error_running_tests, Reason}}.
+
+eunit_opts(_State) ->
+    [{app, undefined, "app", string, help(app)},
+     {cover, $c, "cover", boolean, help(cover)},
+     {suite, undefined, "suite", string, help(suite)},
+     {verbose, $v, "verbose", boolean, help(verbose)}].
+
+help(app)    -> "List of application test suites to run";
+help(cover)   -> "Generate cover data";
+help(suite)   -> "List of test suites to run";
+help(verbose) -> "Verbose output".
diff --git a/test/rebar_eunit_SUITE.erl b/test/rebar_eunit_SUITE.erl
index 39dc30c..33df773 100644
--- a/test/rebar_eunit_SUITE.erl
+++ b/test/rebar_eunit_SUITE.erl
@@ -11,7 +11,14 @@
          test_basic_exports/1,
          test_multi_exports/1,
          test_basic_defines/1,
-         test_multi_defines/1]).
+         test_multi_defines/1,
+         test_single_app_flag/1,
+         test_multiple_app_flag/1,
+         test_nonexistent_app_flag/1,
+         test_single_suite_flag/1,
+         test_suite_in_app_flag/1,
+         test_suite_in_wrong_app_flag/1,
+         test_nonexistent_suite_flag/1]).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
@@ -32,7 +39,10 @@ init_per_testcase(_, Config) ->
 all() ->
     [test_basic_app, test_multi_app, test_profile,
      test_basic_exports, test_multi_exports,
-     test_basic_defines, test_multi_defines].
+     test_basic_defines, test_multi_defines,
+     test_single_app_flag, test_multiple_app_flag, test_nonexistent_app_flag,
+     test_single_suite_flag, test_suite_in_app_flag,
+     test_suite_in_wrong_app_flag, test_nonexistent_suite_flag].
 
 test_basic_app(Config) ->
     AppDir = ?config(apps, Config),
@@ -192,3 +202,187 @@ test_multi_defines(Config) ->
     lists:foreach(fun(Expect) -> true = lists:member(Expect, SuiteOpts1) end, Expect),
     lists:foreach(fun(Expect) -> true = lists:member(Expect, AppOpts2) end, Expect),
     lists:foreach(fun(Expect) -> true = lists:member(Expect, SuiteOpts2) end, Expect).
+
+test_single_app_flag(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
+                                      Name1,
+                                      Vsn1,
+                                      [kernel, stdlib]),
+    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name2]),
+                                      Name2,
+                                      Vsn2,
+                                      [kernel, stdlib]),
+
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    rebar_test_utils:run_and_check(Config,
+                                   RebarConfig,
+                                   ["eunit", "--app=" ++ Name1],
+                                   {ok, [{app, Name1}, {app, Name2}]}),
+
+    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
+    {module, Suite1} = code:ensure_loaded(Suite1),
+    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
+    {error, nofile} = code:ensure_loaded(Suite2).
+
+test_multiple_app_flag(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
+                                      Name1,
+                                      Vsn1,
+                                      [kernel, stdlib]),
+    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name2]),
+                                      Name2,
+                                      Vsn2,
+                                      [kernel, stdlib]),
+
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    rebar_test_utils:run_and_check(Config,
+                                   RebarConfig,
+                                   ["eunit", "--app=" ++ Name1 ++ "," ++ Name2],
+                                   {ok, [{app, Name1}, {app, Name2}]}),
+
+    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
+    {module, Suite1} = code:ensure_loaded(Suite1),
+    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
+    {module, Suite2} = code:ensure_loaded(Suite2).
+
+test_nonexistent_app_flag(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
+                                      Name1,
+                                      Vsn1,
+                                      [kernel, stdlib]),
+    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name2]),
+                                      Name2,
+                                      Vsn2,
+                                      [kernel, stdlib]),
+
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    {error, {_, Error}} = rebar_test_utils:run_and_check(Config,
+                                                         RebarConfig,
+                                                         ["eunit", "--app=not_a_real_app"],
+                                                         return),
+
+    Error = {error_running_tests, "Application `not_a_real_app' not found in project."}.
+
+test_single_suite_flag(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
+                                      Name1,
+                                      Vsn1,
+                                      [kernel, stdlib]),
+    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name2]),
+                                      Name2,
+                                      Vsn2,
+                                      [kernel, stdlib]),
+
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    rebar_test_utils:run_and_check(Config,
+                                   RebarConfig,
+                                   ["eunit", "--suite=not_a_real_src_" ++ Name1],
+                                   {ok, [{app, Name1}, {app, Name2}]}),
+
+    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
+    {module, Suite1} = code:ensure_loaded(Suite1).
+
+test_suite_in_app_flag(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
+                                      Name1,
+                                      Vsn1,
+                                      [kernel, stdlib]),
+    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name2]),
+                                      Name2,
+                                      Vsn2,
+                                      [kernel, stdlib]),
+
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    rebar_test_utils:run_and_check(Config,
+                                   RebarConfig,
+                                   ["eunit",
+                                    "--app=" ++ Name1,
+                                    "--suite=not_a_real_src_" ++ Name1],
+                                   {ok, [{app, Name1}, {app, Name2}]}),
+
+    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
+    {module, Suite1} = code:ensure_loaded(Suite1),
+    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
+    {error, nofile} = code:ensure_loaded(Suite2).
+
+test_suite_in_wrong_app_flag(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
+                                      Name1,
+                                      Vsn1,
+                                      [kernel, stdlib]),
+    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name2]),
+                                      Name2,
+                                      Vsn2,
+                                      [kernel, stdlib]),
+
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    {error, {_, Error}} = rebar_test_utils:run_and_check(Config,
+                                                         RebarConfig,
+                                                         ["eunit",
+                                                         "--app=" ++ Name1,
+                                                         "--suite=not_a_real_src_" ++ Name2],
+                                                         return),
+
+    Error = {error_running_tests, "Module `not_a_real_src_" ++
+                                  Name2 ++
+                                  "' not found in applications."}.
+
+test_nonexistent_suite_flag(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
+                                      Name1,
+                                      Vsn1,
+                                      [kernel, stdlib]),
+    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name2]),
+                                      Name2,
+                                      Vsn2,
+                                      [kernel, stdlib]),
+
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    {error, {_, Error}} = rebar_test_utils:run_and_check(Config,
+                                                         RebarConfig,
+                                                         ["eunit", "--suite=not_a_real_module"],
+                                                         return),
+
+    Error = {error_running_tests, "Module `not_a_real_module' not found in applications."}.