Merge pull request #467 from kejv/support-parse-transforms2

Refactor logic and optimizations in rebar_erlc_compiler:doterl_compile/4

Makefile

@@ -6,6 +6,7 @@ OTPVSNCMD='io:fwrite("~s",[rebar_utils:otp_release()]), halt().'
 OTPVSN=$(shell erl -pa ebin/ -noshell -eval $(OTPVSNCMD))
 PLT_FILENAME=~/.dialyzer_rebar_$(OTPVSN)_plt
 LOG_LEVEL?=debug
+RT_TARGETS?=inttest
 
 all:
 	./bootstrap
@@ -73,6 +74,6 @@ test_eunit: all
 	@$(REBAR) eunit
 
 test_inttest: all deps
-	@$(RETEST) -l $(LOG_LEVEL) inttest
+	@$(RETEST) -l $(LOG_LEVEL) $(RT_TARGETS)
 
 travis: clean debug xref clean all deps test

THANKS

@@ -135,3 +135,4 @@ Pavel Baturko
 Igor Savchuk
 Mark Anderson
 Brian H. Ward
+David Kubecka

inttest/erlc_dep_graph/erlc_dep_graph_rt.erl

@@ -0,0 +1,111 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+%% -------------------------------------------------------------------
+%%
+%% rebar: Erlang Build Tools
+%%
+%% Copyright (c) 2015 David Kubecka
+%%
+%% Permission is hereby granted, free of charge, to any person obtaining a copy
+%% of this software and associated documentation files (the "Software"), to deal
+%% in the Software without restriction, including without limitation the rights
+%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the Software is
+%% furnished to do so, subject to the following conditions:
+%%
+%% The above copyright notice and this permission notice shall be included in
+%% all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+%% THE SOFTWARE.
+%% -------------------------------------------------------------------
+-module(erlc_dep_graph_rt).
+-export([files/0,
+         run/1]).
+
+-include_lib("eunit/include/eunit.hrl").
+
+files() ->
+    [{copy, "../../rebar", "rebar"},
+     {copy, "rebar.config", "rebar.config"},
+     {copy, "src", "src"},
+     {copy, "include", "include"},
+     {copy, "extra_include", "extra_include"}].
+
+run(_Dir) ->
+    compile_all(ok, ""),
+    check_beams_ok(),
+    check_beams_untouched(filelib:wildcard("ebin/*.beam")),
+    modify_and_recompile_ok("src/lisp.erl", "ebin/lisp.beam"),
+
+    clean_all_ok(),
+    compile_all(error, "-C rebar.config.non-existing"),
+    compile_all(ok, ""),
+    modify_and_recompile_ok("extra_include/extra.hrl", "ebin/java.beam"),
+
+    Java = "src/java.erl",
+    {ok, OrigContent} = file:read_file(Java),
+    %% Remove header file inclusion
+    {ok, _} = file:copy("src/java.erl.no_extra", Java),
+    %% Ensure recompilation
+    touch([Java]),
+    compile_all(ok, ""),
+    %% Modify that header file
+    touch(["extra_include/extra.hrl"]),
+    %% Ensure we don't have to recompile anything
+    check_beams_untouched(["ebin/java.beam"]),
+    %% Clean up
+    ok = file:write_file(Java, OrigContent),
+
+    %% Check that changes propagate deeply through the dependency tree
+    modify_and_recompile_ok("include/lambda.hrl", "ebin/perl.beam"),
+
+    ok.
+
+check_beams_ok() ->
+    F = fun(BeamFile) -> ?assert(filelib:is_regular(BeamFile)) end,
+    with_erl_beams(F).
+
+check_beams_untouched(Beams) ->
+    compile_all_and_assert_mtimes(Beams, fun erlang:'=:='/2).
+
+modify_and_recompile_ok(TouchFile, CheckFile) ->
+    touch([TouchFile]),
+    compile_all_and_assert_mtimes([CheckFile], fun erlang:'<'/2).
+
+compile_all_and_assert_mtimes(Beams, Cmp) ->
+    BeamsModifiedBefore = mtime_ns(Beams),
+    compile_all(ok, ""),
+    BeamsModifiedAfter = mtime_ns(Beams),
+    lists:zipwith(fun(Before, After) -> ?assert(Cmp(Before, After)) end,
+                  BeamsModifiedBefore, BeamsModifiedAfter).
+
+with_erl_beams(F) ->
+    lists:map(
+        fun(ErlFile) ->
+            ErlRoot = filename:rootname(filename:basename(ErlFile)),
+            BeamFile = filename:join("ebin", ErlRoot ++ ".beam"),
+            F(BeamFile)
+        end,
+        filelib:wildcard("src/*.erl")).
+
+mtime_ns(Files) ->
+    [os:cmd("stat -c%y " ++ File) || File <- Files].
+
+touch(Files) ->
+    %% Sleep one second so that filelib:last_modified/1 is guaranteed to notice
+    %% that files have changed.
+    ok = timer:sleep(1000),
+    [os:cmd("touch " ++ File) || File <- Files].
+
+compile_all(Result, Opts) ->
+    ?assertMatch({Result, _},
+        retest_sh:run("./rebar " ++ Opts ++ " compile", [])).
+
+clean_all_ok() ->
+    ?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])).

inttest/erlc_dep_graph/extra_include/extra.hrl

@@ -0,0 +1,3 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+-define(CONCISE, impossible).

inttest/erlc_dep_graph/include/lambda.hrl

@@ -0,0 +1,3 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+-define(FUN, fake).

inttest/erlc_dep_graph/rebar.config

@@ -0,0 +1,6 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+{erl_opts,
+ [
+  {i, "extra_include"}
+ ]}.

inttest/erlc_dep_graph/src/foo.app.src

@@ -0,0 +1,7 @@
+{application,foo,
+             [{description,[]},
+              {vsn,"1.0.0"},
+              {registered,[]},
+              {applications,[kernel,stdlib]},
+              {env,[]}
+              ]}.

inttest/erlc_dep_graph/src/java.erl

@@ -0,0 +1,11 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+-module(java).
+
+-export([factory/0]).
+
+-include("lambda.hrl").
+-include("extra.hrl").
+
+factory() ->
+	?FUN.

inttest/erlc_dep_graph/src/java.erl.no_extra

@@ -0,0 +1,10 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+-module(java).
+
+-export([factory/0]).
+
+-include("lambda.hrl").
+
+factory() ->
+	?FUN.

inttest/erlc_dep_graph/src/lisp.erl

@@ -0,0 +1,13 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+-module(lisp).
+
+-export([parse_transform/2]).
+
+-include("lambda.hrl").
+-ifdef(NOT_DEFINED).
+-include_lib("include/non/existent.hrl").
+-endif.
+
+parse_transform(Forms, _Options) ->
+    Forms.

inttest/erlc_dep_graph/src/perl.erl

@@ -0,0 +1,10 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+-module(perl).
+
+-export(['$_'/0]).
+
+-compile({parse_transform, lisp}).
+
+'$_'() ->
+	anything.

src/rebar_base_compiler.erl

@@ -129,21 +129,10 @@ remove_common_path1([Part | RestFilename], [Part | RestPath]) ->
 remove_common_path1(FilenameParts, _) ->
     filename:join(FilenameParts).
 
-
-compile(Unit, Config, CompileFn) ->
-    case CompileFn(Unit, Config) of
-        ok ->
-            ok;
-        skipped ->
-            skipped;
-        Error ->
-            Error
-    end.
-
 compile_each([], _Config, _CompileFn) ->
     ok;
 compile_each([Unit | Rest], Config, CompileFn) ->
-    case compile(Unit, Config, CompileFn) of
+    case CompileFn(Unit, Config) of
         ok ->
             ?CONSOLE("Compiled ~s\n", [unit_source(Unit)]);
         {ok, Warnings} ->
@@ -224,7 +213,7 @@ compile_worker(QueuePid, Config, CompileFn) ->
     QueuePid ! {next, self()},
     receive
         {compile, Source} ->
-            case catch(compile(Source, Config, CompileFn)) of
+            case catch(CompileFn(Source, Config)) of
                 {ok, Ws} ->
                     QueuePid ! {compiled, Source, Ws},
                     compile_worker(QueuePid, Config, CompileFn);

src/rebar_erlc_compiler.erl

@@ -36,25 +36,17 @@
 -include("rebar.hrl").
 -include_lib("stdlib/include/erl_compile.hrl").
 
--define(ERLCINFO_VSN, 1).
+-define(ERLCINFO_VSN, 2).
 -define(ERLCINFO_FILE, "erlcinfo").
 -type erlc_info_v() :: {digraph:vertex(), term()} | 'false'.
 -type erlc_info_e() :: {digraph:vertex(), digraph:vertex()}.
--type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e())}.
+-type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e()), list(string())}.
 -record(erlcinfo,
         {
           vsn = ?ERLCINFO_VSN :: pos_integer(),
-          info = {[], []} :: erlc_info()
+          info = {[], [], []} :: erlc_info()
         }).
 
--ifdef(namespaced_types).
-%% digraph:graph() exists starting from Erlang 17.
--type rebar_digraph() :: digraph:graph().
--else.
-%% digraph() has been obsoleted in Erlang 17 and deprecated in 18.
--type rebar_digraph() :: digraph().
--endif.
-
 %% ===================================================================
 %% Public API
 %% ===================================================================
@@ -109,7 +101,7 @@ compile(Config, _AppFile) ->
     doterl_compile(Config, "ebin").
 
 -spec clean(rebar_config:config(), file:filename()) -> 'ok'.
-clean(Config, _AppFile) ->
+clean(_Config, _AppFile) ->
     MibFiles = rebar_utils:find_files_by_ext("mibs", ".mib"),
     MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles],
     rebar_file_utils:delete_each(
@@ -123,7 +115,7 @@ clean(Config, _AppFile) ->
         || F <- YrlFiles ]),
 
     %% Delete the build graph, if any
-    rebar_file_utils:rm_rf(erlcinfo_file(Config)),
+    rebar_file_utils:rm_rf(erlcinfo_file()),
 
     %% Erlang compilation is recursive, so it's possible that we have a nested
     %% directory structure in ebin with .beam files within. As such, we want
@@ -292,144 +284,85 @@ doterl_compile(Config, OutDir) ->
     doterl_compile(Config, OutDir, [], ErlOpts).
 
 doterl_compile(Config, OutDir, MoreSources, ErlOpts) ->
-    ErlFirstFilesConf = rebar_config:get_list(Config, erl_first_files, []),
     ?DEBUG("erl_opts ~p~n", [ErlOpts]),
     %% Support the src_dirs option allowing multiple directories to
     %% contain erlang source. This might be used, for example, should
     %% eunit tests be separated from the core application source.
     SrcDirs = rebar_utils:src_dirs(proplists:append_values(src_dirs, ErlOpts)),
     AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources,
-    %% NOTE: If and when erl_first_files is not inherited anymore
-    %% (rebar_config:get_local instead of rebar_config:get_list), consider
-    %% logging a warning message for any file listed in erl_first_files which
-    %% wasn't found via gather_src.
-    RestErls = [File || File <- AllErlFiles,
-                        not lists:member(File, ErlFirstFilesConf)],
-    %% NOTE: order of files in ErlFirstFiles is important!
-    ErlFirstFiles = [File || File <- ErlFirstFilesConf,
-                             lists:member(File, AllErlFiles)],
+
     %% Make sure that ebin/ exists and is on the path
     ok = filelib:ensure_dir(filename:join("ebin", "dummy.beam")),
     CurrPath = code:get_path(),
     true = code:add_path(filename:absname("ebin")),
     OutDir1 = proplists:get_value(outdir, ErlOpts, OutDir),
-    G = init_erlcinfo(Config, AllErlFiles),
-    %% Split RestErls so that files which are depended on are treated
-    %% like erl_first_files.
-    {OtherFirstErls, OtherErls} =
-        lists:partition(
-          fun(F) ->
-                  Children = get_children(G, F),
-                  log_files(?FMT("Files dependent on ~s", [F]), Children),
 
-                  case erls(Children) of
-                      [] ->
-                          %% There are no files dependent on this file.
-                          false;
-                      _ ->
-                          %% There are some files dependent on the file.
-                          %% Thus the file has higher priority
-                          %% and should be compiled in the first place.
-                          true
-                  end
-          end, RestErls),
-    %% Dependencies of OtherFirstErls that must be compiled first.
-    OtherFirstErlsDeps = lists:flatmap(
-                           fun(Erl) -> erls(get_parents(G, Erl)) end,
-                           OtherFirstErls),
-    %% NOTE: In case the way we retrieve OtherFirstErlsDeps or merge
-    %% it with OtherFirstErls does not result in the correct compile
-    %% priorities, or the method in use proves to be too slow for
-    %% certain projects, consider using a more elaborate method (maybe
-    %% digraph_utils) or alternatively getting and compiling the .erl
-    %% parents of an individual Source in internal_erl_compile. By not
-    %% handling this in internal_erl_compile, we also avoid extra
-    %% needs_compile/2 calls.
-    FirstErls = ErlFirstFiles ++ uo_merge(OtherFirstErlsDeps, OtherFirstErls),
+    G = init_erlcinfo(proplists:get_all_values(i, ErlOpts), AllErlFiles),
+    NeededErlFiles = needed_files(G, OutDir1, AllErlFiles),
+    ErlFirstFiles = erl_first_files(Config, NeededErlFiles),
+    {DepErls, OtherErls} = lists:partition(
+        fun(Source) -> digraph:in_degree(G, Source) > 0 end,
+        [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]),
+    DepErlsOrdered = digraph_utils:topsort(digraph_utils:subgraph(G, DepErls)),
+    FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered),
     ?DEBUG("Files to compile first: ~p~n", [FirstErls]),
+
     rebar_base_compiler:run(
       Config, FirstErls, OtherErls,
-      fun(S, C) ->
-              internal_erl_compile(C, S, OutDir1, ErlOpts, G)
-      end),
+      fun(S, C) -> internal_erl_compile(C, S, OutDir1, ErlOpts) end),
     true = rebar_utils:cleanup_code_path(CurrPath),
     ok.
 
-%%
-%% Return all .erl files from a list of files
-%%
-erls(Files) ->
-    [Erl || Erl <- Files, filename:extension(Erl) =:= ".erl"].
+erl_first_files(Config, NeededErlFiles) ->
+    %% NOTE: rebar_config:get_local perhaps?
+    ErlFirstFilesConf = rebar_config:get_list(Config, erl_first_files, []),
+    %% NOTE: order of files in ErlFirstFiles is important!
+    [File || File <- ErlFirstFilesConf, lists:member(File, NeededErlFiles)].
 
-%%
-%% Return a list without duplicates while preserving order
-%%
-ulist(L) ->
-    ulist(L, []).
+%% Get subset of SourceFiles which need to be recompiled, respecting
+%% dependencies induced by given graph G.
+needed_files(G, OutDir, SourceFiles) ->
+    lists:filter(fun(Source) ->
+        Target = target_base(OutDir, Source) ++ ".beam",
+        digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)}
+    end, SourceFiles).
 
-ulist([H|T], Acc) ->
-    case lists:member(H, T) of
-        true ->
-            ulist(T, Acc);
-        false ->
-            ulist(T, [H|Acc])
-    end;
-ulist([], Acc) ->
-    lists:reverse(Acc).
+target_base(OutDir, Source) ->
+    filename:join(OutDir, filename:basename(Source, ".erl")).
 
-%%
-%% Merge two lists without duplicates while preserving order
-%%
-uo_merge(L1, L2) ->
-    lists:foldl(fun(E, Acc) -> u_add_element(E, Acc) end, ulist(L1), L2).
-
-u_add_element(Elem, [Elem|_]=Set) -> Set;
-u_add_element(Elem, [E1|Set])     -> [E1|u_add_element(Elem, Set)];
-u_add_element(Elem, [])           -> [Elem].
-
--spec include_path(file:filename(),
-                   rebar_config:config()) -> [file:filename(), ...].
-include_path(Source, Config) ->
-    ErlOpts = rebar_config:get(Config, erl_opts, []),
-    lists:usort(["include", filename:dirname(Source)]
-                ++ proplists:get_all_values(i, ErlOpts)).
-
--spec needs_compile(file:filename(), file:filename(),
-                    [string()]) -> boolean().
-needs_compile(Source, Target, Parents) ->
-    TargetLastMod = filelib:last_modified(Target),
-    lists:any(fun(I) -> TargetLastMod < filelib:last_modified(I) end,
-              [Source] ++ Parents).
-
-check_erlcinfo(_Config, #erlcinfo{vsn=?ERLCINFO_VSN}) ->
-    ok;
-check_erlcinfo(Config, #erlcinfo{vsn=Vsn}) ->
-    ?ABORT("~s file version is incompatible. expected: ~b got: ~b~n",
-           [erlcinfo_file(Config), ?ERLCINFO_VSN, Vsn]);
-check_erlcinfo(Config, _) ->
-    ?ABORT("~s file is invalid. Please delete before next run.~n",
-           [erlcinfo_file(Config)]).
-
-erlcinfo_file(_Config) ->
+erlcinfo_file() ->
     filename:join([rebar_utils:get_cwd(), ".rebar", ?ERLCINFO_FILE]).
 
-init_erlcinfo(Config, Erls) ->
-    G = restore_erlcinfo(Config),
-    %% Get a unique list of dirs based on the source files' locations.
-    %% This is used for finding files in sub dirs of the configured
-    %% src_dirs. For example, src/sub_dir/foo.erl.
-    Dirs = sets:to_list(lists:foldl(
-                          fun(Erl, Acc) ->
-                                  Dir = filename:dirname(Erl),
-                                  sets:add_element(Dir, Acc)
-                          end, sets:new(), Erls)),
-    Updates = [update_erlcinfo(G, Erl, include_path(Erl, Config) ++ Dirs)
-               || Erl <- Erls],
-    Modified = lists:member(modified, Updates),
-    ok = store_erlcinfo(G, Config, Modified),
+%% Get dependency graph of given Erls files and their dependencies (header files,
+%% parse transforms, behaviours etc.) located in their directories or given
+%% InclDirs.  Note that last modification times stored in vertices already respect
+%% dependencies induced by given graph G.
+init_erlcinfo(InclDirs, Erls) ->
+    G = digraph:new([acyclic]),
+    try restore_erlcinfo(G, InclDirs)
+    catch
+        _:_ ->
+            ?WARN("Failed to restore ~s file. Discarding it.~n", [erlcinfo_file()]),
+            ok = file:delete(erlcinfo_file())
+    end,
+    Dirs = source_and_include_dirs(InclDirs, Erls),
+    Modified = lists:foldl(update_erlcinfo_fun(G, Dirs), false, Erls),
+    if Modified -> store_erlcinfo(G, InclDirs); not Modified -> ok end,
     G.
 
-update_erlcinfo(G, Source, Dirs) ->
+source_and_include_dirs(InclDirs, Erls) ->
+    SourceDirs = lists:map(fun filename:dirname/1, Erls),
+    lists:usort(["include" | InclDirs ++ SourceDirs]).
+
+update_erlcinfo_fun(G, Dirs) ->
+    fun(Erl, Modified) ->
+        case update_erlcinfo(G, Dirs, Erl) of
+            modified -> true;
+            unmodified -> Modified
+        end
+    end.
+
+update_erlcinfo(G, Dirs, Source) ->
     case digraph:vertex(G, Source) of
         {_, LastUpdated} ->
             case filelib:last_modified(Source) of
@@ -440,78 +373,68 @@ update_erlcinfo(G, Source, Dirs) ->
                     digraph:del_vertex(G, Source),
                     modified;
                 LastModified when LastUpdated < LastModified ->
-                    modify_erlcinfo(G, Source, Dirs),
-                    modified;
+                    modify_erlcinfo(G, Source, LastModified, Dirs);
                 _ ->
-                    unmodified
+                    Modified = lists:foldl(
+                        update_erlcinfo_fun(G, Dirs),
+                        false, digraph:out_neighbours(G, Source)),
+                    MaxModified = update_max_modified_deps(G, Source),
+                    case Modified orelse MaxModified > LastUpdated of
+                        true -> modified;
+                        false -> unmodified
+                    end
             end;
         false ->
-            modify_erlcinfo(G, Source, Dirs),
-            modified
+            modify_erlcinfo(G, Source, filelib:last_modified(Source), Dirs)
     end.
 
-modify_erlcinfo(G, Source, Dirs) ->
+update_max_modified_deps(G, Source) ->
+    MaxModified = lists:max(lists:map(
+        fun(File) -> {_, MaxModified} = digraph:vertex(G, File), MaxModified end,
+        [Source|digraph:out_neighbours(G, Source)])),
+    digraph:add_vertex(G, Source, MaxModified),
+    MaxModified.
+
+modify_erlcinfo(G, Source, LastModified, Dirs) ->
     {ok, Fd} = file:open(Source, [read]),
     Incls = parse_attrs(Fd, []),
     AbsIncls = expand_file_names(Incls, Dirs),
     ok = file:close(Fd),
-    LastUpdated = {date(), time()},
-    digraph:add_vertex(G, Source, LastUpdated),
+    digraph:add_vertex(G, Source, LastModified),
+    digraph:del_edges(G, digraph:out_edges(G, Source)),
     lists:foreach(
       fun(Incl) ->
-              update_erlcinfo(G, Incl, Dirs),
+              update_erlcinfo(G, Dirs, Incl),
               digraph:add_edge(G, Source, Incl)
-      end, AbsIncls).
+      end, AbsIncls),
+    modified.
 
-restore_erlcinfo(Config) ->
-    File = erlcinfo_file(Config),
-    G = digraph:new(),
-    case file:read_file(File) of
+restore_erlcinfo(G, InclDirs) ->
+    case file:read_file(erlcinfo_file()) of
         {ok, Data} ->
-            try binary_to_term(Data) of
-                Erlcinfo ->
-                    ok = check_erlcinfo(Config, Erlcinfo),
-                    #erlcinfo{info=ErlcInfo} = Erlcinfo,
-                    {Vs, Es} = ErlcInfo,
-                    lists:foreach(
-                      fun({V, LastUpdated}) ->
-                              digraph:add_vertex(G, V, LastUpdated)
-                      end, Vs),
-                    lists:foreach(
-                      fun({V1, V2}) ->
-                              digraph:add_edge(G, V1, V2)
-                      end, Es)
-            catch
-                error:badarg ->
-                    ?ERROR(
-                       "Failed (binary_to_term) to restore rebar info file."
-                       " Discard file.~n", []),
-                    ok
-            end;
-        _Err ->
+            %% Since externally passed InclDirs can influence erlcinfo graph (see
+            %% modify_erlcinfo), we have to check here that they didn't change.
+            #erlcinfo{vsn=?ERLCINFO_VSN, info={Vs, Es, InclDirs}} =
+                binary_to_term(Data),
+            lists:foreach(
+              fun({V, LastUpdated}) ->
+                      digraph:add_vertex(G, V, LastUpdated)
+              end, Vs),
+            lists:foreach(
+              fun({_, V1, V2, _}) ->
+                      digraph:add_edge(G, V1, V2)
+              end, Es);
+        {error, _} ->
             ok
-    end,
-    G.
+    end.
 
-store_erlcinfo(_G, _Config, _Modified = false) ->
-    ok;
-store_erlcinfo(G, Config, _Modified) ->
-    Vs = lists:map(
-           fun(V) ->
-                   digraph:vertex(G, V)
-           end, digraph:vertices(G)),
-    Es = lists:flatmap(
-           fun({V, _}) ->
-                   lists:map(
-                     fun(E) ->
-                             {_, V1, V2, _} = digraph:edge(G, E),
-                             {V1, V2}
-                     end, digraph:out_edges(G, V))
-           end, Vs),
-    File = erlcinfo_file(Config),
+store_erlcinfo(G, InclDirs) ->
+    Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
+    Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
+    File = erlcinfo_file(),
     ok = filelib:ensure_dir(File),
-    Data = term_to_binary(#erlcinfo{info={Vs, Es}}, [{compressed, 9}]),
-    file:write_file(File, Data).
+    Data = term_to_binary(#erlcinfo{info={Vs, Es, InclDirs}}, [{compressed, 2}]),
+    ok = file:write_file(File, Data).
 
 %% NOTE: If, for example, one of the entries in Files refers to
 %% gen_server.erl, that entry will be dropped. It is dropped because
@@ -545,46 +468,18 @@ expand_file_names(Files, Dirs) ->
               end
       end, Files).
 
--spec get_parents(rebar_digraph(), file:filename()) -> [file:filename()].
-get_parents(G, Source) ->
-    %% Return all files which the Source depends upon.
-    digraph_utils:reachable_neighbours([Source], G).
-
--spec get_children(rebar_digraph(), file:filename()) -> [file:filename()].
-get_children(G, Source) ->
-    %% Return all files dependent on the Source.
-    digraph_utils:reaching_neighbours([Source], G).
-
 -spec internal_erl_compile(rebar_config:config(), file:filename(),
-                           file:filename(), list(),
-                           rebar_digraph()) -> 'ok' | 'skipped'.
-internal_erl_compile(Config, Source, OutDir, ErlOpts, G) ->
-    %% Determine the target name and includes list by inspecting the source file
-    Module = filename:basename(Source, ".erl"),
-    Parents = get_parents(G, Source),
-    log_files(?FMT("Dependencies of ~s", [Source]), Parents),
-
-    %% Construct the target filename
-    Target = filename:join([OutDir | string:tokens(Module, ".")]) ++ ".beam",
-    ok = filelib:ensure_dir(Target),
-
-    %% If the file needs compilation, based on last mod date of includes or
-    %% the target
-    case needs_compile(Source, Target, Parents) of
-        true ->
-            Opts = [{outdir, filename:dirname(Target)}] ++
-                ErlOpts ++ [{i, "include"}, return],
-            case compile:file(Source, Opts) of
-                {ok, _Mod} ->
-                    ok;
-                {ok, _Mod, Ws} ->
-                    rebar_base_compiler:ok_tuple(Config, Source, Ws);
-                {error, Es, Ws} ->
-                    rebar_base_compiler:error_tuple(Config, Source,
-                                                    Es, Ws, Opts)
-            end;
-        false ->
-            skipped
+    file:filename(), list()) -> ok | {ok, any()} | {error, any(), any()}.
+internal_erl_compile(Config, Source, OutDir, ErlOpts) ->
+    ok = filelib:ensure_dir(OutDir),
+    Opts = [{outdir, OutDir}] ++ ErlOpts ++ [{i, "include"}, return],
+    case compile:file(Source, Opts) of
+        {ok, _Mod} ->
+            ok;
+        {ok, _Mod, Ws} ->
+            rebar_base_compiler:ok_tuple(Config, Source, Ws);
+        {error, Es, Ws} ->
+            rebar_base_compiler:error_tuple(Config, Source, Es, Ws, Opts)
     end.
 
 -spec compile_mib(file:filename(), file:filename(),
@@ -627,7 +522,7 @@ compile_yrl(Source, Target, Config) ->
 -spec compile_xrl_yrl(rebar_config:config(), file:filename(),
                       file:filename(), list(), module()) -> 'ok'.
 compile_xrl_yrl(Config, Source, Target, Opts, Mod) ->
-    case needs_compile(Source, Target, []) of
+    case needs_compile(Source, Target) of
         true ->
             case Mod:file(Source, Opts ++ [{return, true}]) of
                 {ok, _} ->
@@ -642,6 +537,9 @@ compile_xrl_yrl(Config, Source, Target, Opts, Mod) ->
             skipped
     end.
 
+needs_compile(Source, Target) ->
+    filelib:last_modified(Source) > filelib:last_modified(Target).
+
 gather_src([], Srcs) ->
     Srcs;
 gather_src([Dir|Rest], Srcs) ->
@@ -676,21 +574,15 @@ parse_attrs(Fd, Includes) ->
     end.
 
 process_attr(Form, Includes) ->
-    try
-        AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)),
-        process_attr(AttrName, Form, Includes)
-    catch _:_ ->
-            %% TODO: We should probably try to be more specific here
-            %% and not suppress all errors.
-            Includes
-    end.
+    AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)),
+    process_attr(AttrName, Form, Includes).
 
 process_attr(import, Form, Includes) ->
     case erl_syntax_lib:analyze_import_attribute(Form) of
         {Mod, _Funs} ->
-            [atom_to_list(Mod) ++ ".erl"|Includes];
+            [module_to_erl(Mod)|Includes];
         Mod ->
-            [atom_to_list(Mod) ++ ".erl"|Includes]
+            [module_to_erl(Mod)|Includes]
     end;
 process_attr(file, Form, Includes) ->
     {File, _} = erl_syntax_lib:analyze_file_attribute(Form),
@@ -702,29 +594,35 @@ process_attr(include, Form, Includes) ->
 process_attr(include_lib, Form, Includes) ->
     [FileNode] = erl_syntax:attribute_arguments(Form),
     RawFile = erl_syntax:string_value(FileNode),
-    File = maybe_expand_include_lib_path(RawFile),
-    [File|Includes];
+    maybe_expand_include_lib_path(RawFile) ++ Includes;
 process_attr(behaviour, Form, Includes) ->
     [FileNode] = erl_syntax:attribute_arguments(Form),
-    File = erl_syntax:atom_name(FileNode) ++ ".erl",
+    File = module_to_erl(erl_syntax:atom_value(FileNode)),
     [File|Includes];
 process_attr(compile, Form, Includes) ->
     [Arg] = erl_syntax:attribute_arguments(Form),
     case erl_syntax:concrete(Arg) of
         {parse_transform, Mod} ->
-            [atom_to_list(Mod) ++ ".erl"|Includes];
+            [module_to_erl(Mod)|Includes];
         {core_transform, Mod} ->
-            [atom_to_list(Mod) ++ ".erl"|Includes];
+            [module_to_erl(Mod)|Includes];
         L when is_list(L) ->
             lists:foldl(
-              fun({parse_transform, M}, Acc) ->
-                      [atom_to_list(M) ++ ".erl"|Acc];
-                 ({core_transform, M}, Acc) ->
-                      [atom_to_list(M) ++ ".erl"|Acc];
+              fun({parse_transform, Mod}, Acc) ->
+                      [module_to_erl(Mod)|Acc];
+                 ({core_transform, Mod}, Acc) ->
+                      [module_to_erl(Mod)|Acc];
                  (_, Acc) ->
                       Acc
-              end, Includes, L)
-    end.
+              end, Includes, L);
+        _ ->
+            Includes
+    end;
+process_attr(_, _Form, Includes) ->
+    Includes.
+
+module_to_erl(Mod) ->
+    atom_to_list(Mod) ++ ".erl".
 
 %% Given the filename from an include_lib attribute, if the path
 %% exists, return unmodified, or else get the absolute ERL_LIBS
@@ -732,7 +630,7 @@ process_attr(compile, Form, Includes) ->
 maybe_expand_include_lib_path(File) ->
     case filelib:is_regular(File) of
         true ->
-            File;
+            [File];
         false ->
             expand_include_lib_path(File)
     end.
@@ -747,8 +645,10 @@ expand_include_lib_path(File) ->
     Split = filename:split(filename:dirname(File)),
     Lib = hd(Split),
     SubDir = filename:join(tl(Split)),
-    Dir = code:lib_dir(list_to_atom(Lib), list_to_atom(SubDir)),
-    filename:join(Dir, File1).
+    case code:lib_dir(list_to_atom(Lib), list_to_atom(SubDir)) of
+        {error, bad_name} -> [];
+        Dir -> [filename:join(Dir, File1)]
+    end.
 
 %%
 %% Ensure all files in a list are present and abort if one is missing
@@ -762,13 +662,3 @@ check_file(File) ->
         false -> ?ABORT("File ~p is missing, aborting\n", [File]);
         true -> File
     end.
-
-%% Print prefix followed by list of files. If the list is empty, print
-%% on the same line, otherwise use a separate line.
-log_files(Prefix, Files) ->
-    case Files of
-        [] ->
-            ?DEBUG("~s: ~p~n", [Prefix, Files]);
-        _ ->
-            ?DEBUG("~s:~n~p~n", [Prefix, Files])
-    end.