Fix Parallel Compilation

This patch does two things:

1. it broadens the interface for the compiler module so that
   non-first-file modules can possibly be parallelized. This is done by
   dynamically switching on `[ListOfFiles]`, which remains sequential as
   before, or `{[SeqPriority], [Parallel]}`, which divides regular files
   between higher priority ones and those that can be in parallel
2. implements this mechanism in the rebar compiler, based on the erl
   file digraph. If a file has an in-neighbour, it is depended on by
   another file. The mechanism therefore makes it so all files that have
   dependants get compiled in their strict relative sequential order
   first, and then the undepended-on files get compiled together in
   parallel.

By running:

  ./rebar3 ct --suite test/rebar_compile_SUITE.erl --case \
  recompile_when_parse_transform_inline_changes --repeat 50

the previous iteration of this would rapidly fail, and this one succeeds
every time.

1 files changed, 11 insertions, 2 deletions

diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl
index b04c2c4..55666ba 100644
--- a/src/rebar_compiler.erl
+++ b/src/rebar_compiler.erl
@@ -21,7 +21,10 @@
                                            out_mappings => out_mappings()}.
 -callback needed_files(digraph:graph(), [file:filename()], out_mappings(),
                        rebar_app_info:t()) ->
-    {{[file:filename()], term()}, {[file:filename()], term()}}.
+    {{[file:filename()], term()}, % ErlFirstFiles (erl_opts global priority)
+     {[file:filename()] | % [Sequential]
+      {[file:filename()], [file:filename()]}, % {Sequential, Parallel}
+      term()}}.
 -callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()].
 -callback compile(file:filename(), out_mappings(), rebar_dict(), list()) ->
     ok | {ok, [string()]} | {ok, [string()], [string()]}.
@@ -77,7 +80,13 @@ run(CompilerMod, AppInfo, Label) ->
     true = digraph:delete(G),
 
     compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod),
-    compile_parallel(RestFiles, Opts, BaseOpts, Mappings, CompilerMod).
+    case RestFiles of
+        {Sequential, Parallel} -> % new parallelizable form
+            compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod),
+            compile_parallel(Parallel, Opts, BaseOpts, Mappings, CompilerMod);
+        _ when is_list(RestFiles) -> % traditional sequential build
+            compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod)
+    end.
 
 compile_each([], _Opts, _Config, _Outs, _CompilerMod) ->
     ok;