Merge pull request #467 from kejv/support-parse-transforms2

Refactor logic and optimizations in rebar_erlc_compiler:doterl_compile/4
This commit is contained in:
Fred Hebert 2015-04-07 16:33:05 -04:00
commit b557e5fedc
13 changed files with 324 additions and 269 deletions

View file

@ -6,6 +6,7 @@ OTPVSNCMD='io:fwrite("~s",[rebar_utils:otp_release()]), halt().'
OTPVSN=$(shell erl -pa ebin/ -noshell -eval $(OTPVSNCMD)) OTPVSN=$(shell erl -pa ebin/ -noshell -eval $(OTPVSNCMD))
PLT_FILENAME=~/.dialyzer_rebar_$(OTPVSN)_plt PLT_FILENAME=~/.dialyzer_rebar_$(OTPVSN)_plt
LOG_LEVEL?=debug LOG_LEVEL?=debug
RT_TARGETS?=inttest
all: all:
./bootstrap ./bootstrap
@ -73,6 +74,6 @@ test_eunit: all
@$(REBAR) eunit @$(REBAR) eunit
test_inttest: all deps test_inttest: all deps
@$(RETEST) -l $(LOG_LEVEL) inttest @$(RETEST) -l $(LOG_LEVEL) $(RT_TARGETS)
travis: clean debug xref clean all deps test travis: clean debug xref clean all deps test

1
THANKS
View file

@ -135,3 +135,4 @@ Pavel Baturko
Igor Savchuk Igor Savchuk
Mark Anderson Mark Anderson
Brian H. Ward Brian H. Ward
David Kubecka

View file

@ -0,0 +1,111 @@
%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 et
%% -------------------------------------------------------------------
%%
%% rebar: Erlang Build Tools
%%
%% Copyright (c) 2015 David Kubecka
%%
%% Permission is hereby granted, free of charge, to any person obtaining a copy
%% of this software and associated documentation files (the "Software"), to deal
%% in the Software without restriction, including without limitation the rights
%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
%% copies of the Software, and to permit persons to whom the Software is
%% furnished to do so, subject to the following conditions:
%%
%% The above copyright notice and this permission notice shall be included in
%% all copies or substantial portions of the Software.
%%
%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
%% THE SOFTWARE.
%% -------------------------------------------------------------------
-module(erlc_dep_graph_rt).
-export([files/0,
run/1]).
-include_lib("eunit/include/eunit.hrl").
files() ->
[{copy, "../../rebar", "rebar"},
{copy, "rebar.config", "rebar.config"},
{copy, "src", "src"},
{copy, "include", "include"},
{copy, "extra_include", "extra_include"}].
run(_Dir) ->
compile_all(ok, ""),
check_beams_ok(),
check_beams_untouched(filelib:wildcard("ebin/*.beam")),
modify_and_recompile_ok("src/lisp.erl", "ebin/lisp.beam"),
clean_all_ok(),
compile_all(error, "-C rebar.config.non-existing"),
compile_all(ok, ""),
modify_and_recompile_ok("extra_include/extra.hrl", "ebin/java.beam"),
Java = "src/java.erl",
{ok, OrigContent} = file:read_file(Java),
%% Remove header file inclusion
{ok, _} = file:copy("src/java.erl.no_extra", Java),
%% Ensure recompilation
touch([Java]),
compile_all(ok, ""),
%% Modify that header file
touch(["extra_include/extra.hrl"]),
%% Ensure we don't have to recompile anything
check_beams_untouched(["ebin/java.beam"]),
%% Clean up
ok = file:write_file(Java, OrigContent),
%% Check that changes propagate deeply through the dependency tree
modify_and_recompile_ok("include/lambda.hrl", "ebin/perl.beam"),
ok.
check_beams_ok() ->
F = fun(BeamFile) -> ?assert(filelib:is_regular(BeamFile)) end,
with_erl_beams(F).
check_beams_untouched(Beams) ->
compile_all_and_assert_mtimes(Beams, fun erlang:'=:='/2).
modify_and_recompile_ok(TouchFile, CheckFile) ->
touch([TouchFile]),
compile_all_and_assert_mtimes([CheckFile], fun erlang:'<'/2).
compile_all_and_assert_mtimes(Beams, Cmp) ->
BeamsModifiedBefore = mtime_ns(Beams),
compile_all(ok, ""),
BeamsModifiedAfter = mtime_ns(Beams),
lists:zipwith(fun(Before, After) -> ?assert(Cmp(Before, After)) end,
BeamsModifiedBefore, BeamsModifiedAfter).
with_erl_beams(F) ->
lists:map(
fun(ErlFile) ->
ErlRoot = filename:rootname(filename:basename(ErlFile)),
BeamFile = filename:join("ebin", ErlRoot ++ ".beam"),
F(BeamFile)
end,
filelib:wildcard("src/*.erl")).
mtime_ns(Files) ->
[os:cmd("stat -c%y " ++ File) || File <- Files].
touch(Files) ->
%% Sleep one second so that filelib:last_modified/1 is guaranteed to notice
%% that files have changed.
ok = timer:sleep(1000),
[os:cmd("touch " ++ File) || File <- Files].
compile_all(Result, Opts) ->
?assertMatch({Result, _},
retest_sh:run("./rebar " ++ Opts ++ " compile", [])).
clean_all_ok() ->
?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])).

View file

@ -0,0 +1,3 @@
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 ft=erlang et
-define(CONCISE, impossible).

View file

@ -0,0 +1,3 @@
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 ft=erlang et
-define(FUN, fake).

View file

@ -0,0 +1,6 @@
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 ft=erlang et
{erl_opts,
[
{i, "extra_include"}
]}.

View file

@ -0,0 +1,7 @@
{application,foo,
[{description,[]},
{vsn,"1.0.0"},
{registered,[]},
{applications,[kernel,stdlib]},
{env,[]}
]}.

View file

@ -0,0 +1,11 @@
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 ft=erlang et
-module(java).
-export([factory/0]).
-include("lambda.hrl").
-include("extra.hrl").
factory() ->
?FUN.

View file

@ -0,0 +1,10 @@
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 ft=erlang et
-module(java).
-export([factory/0]).
-include("lambda.hrl").
factory() ->
?FUN.

View file

@ -0,0 +1,13 @@
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 ft=erlang et
-module(lisp).
-export([parse_transform/2]).
-include("lambda.hrl").
-ifdef(NOT_DEFINED).
-include_lib("include/non/existent.hrl").
-endif.
parse_transform(Forms, _Options) ->
Forms.

View file

@ -0,0 +1,10 @@
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 ft=erlang et
-module(perl).
-export(['$_'/0]).
-compile({parse_transform, lisp}).
'$_'() ->
anything.

View file

@ -129,21 +129,10 @@ remove_common_path1([Part | RestFilename], [Part | RestPath]) ->
remove_common_path1(FilenameParts, _) -> remove_common_path1(FilenameParts, _) ->
filename:join(FilenameParts). filename:join(FilenameParts).
compile(Unit, Config, CompileFn) ->
case CompileFn(Unit, Config) of
ok ->
ok;
skipped ->
skipped;
Error ->
Error
end.
compile_each([], _Config, _CompileFn) -> compile_each([], _Config, _CompileFn) ->
ok; ok;
compile_each([Unit | Rest], Config, CompileFn) -> compile_each([Unit | Rest], Config, CompileFn) ->
case compile(Unit, Config, CompileFn) of case CompileFn(Unit, Config) of
ok -> ok ->
?CONSOLE("Compiled ~s\n", [unit_source(Unit)]); ?CONSOLE("Compiled ~s\n", [unit_source(Unit)]);
{ok, Warnings} -> {ok, Warnings} ->
@ -224,7 +213,7 @@ compile_worker(QueuePid, Config, CompileFn) ->
QueuePid ! {next, self()}, QueuePid ! {next, self()},
receive receive
{compile, Source} -> {compile, Source} ->
case catch(compile(Source, Config, CompileFn)) of case catch(CompileFn(Source, Config)) of
{ok, Ws} -> {ok, Ws} ->
QueuePid ! {compiled, Source, Ws}, QueuePid ! {compiled, Source, Ws},
compile_worker(QueuePid, Config, CompileFn); compile_worker(QueuePid, Config, CompileFn);

View file

@ -36,25 +36,17 @@
-include("rebar.hrl"). -include("rebar.hrl").
-include_lib("stdlib/include/erl_compile.hrl"). -include_lib("stdlib/include/erl_compile.hrl").
-define(ERLCINFO_VSN, 1). -define(ERLCINFO_VSN, 2).
-define(ERLCINFO_FILE, "erlcinfo"). -define(ERLCINFO_FILE, "erlcinfo").
-type erlc_info_v() :: {digraph:vertex(), term()} | 'false'. -type erlc_info_v() :: {digraph:vertex(), term()} | 'false'.
-type erlc_info_e() :: {digraph:vertex(), digraph:vertex()}. -type erlc_info_e() :: {digraph:vertex(), digraph:vertex()}.
-type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e())}. -type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e()), list(string())}.
-record(erlcinfo, -record(erlcinfo,
{ {
vsn = ?ERLCINFO_VSN :: pos_integer(), vsn = ?ERLCINFO_VSN :: pos_integer(),
info = {[], []} :: erlc_info() info = {[], [], []} :: erlc_info()
}). }).
-ifdef(namespaced_types).
%% digraph:graph() exists starting from Erlang 17.
-type rebar_digraph() :: digraph:graph().
-else.
%% digraph() has been obsoleted in Erlang 17 and deprecated in 18.
-type rebar_digraph() :: digraph().
-endif.
%% =================================================================== %% ===================================================================
%% Public API %% Public API
%% =================================================================== %% ===================================================================
@ -109,7 +101,7 @@ compile(Config, _AppFile) ->
doterl_compile(Config, "ebin"). doterl_compile(Config, "ebin").
-spec clean(rebar_config:config(), file:filename()) -> 'ok'. -spec clean(rebar_config:config(), file:filename()) -> 'ok'.
clean(Config, _AppFile) -> clean(_Config, _AppFile) ->
MibFiles = rebar_utils:find_files_by_ext("mibs", ".mib"), MibFiles = rebar_utils:find_files_by_ext("mibs", ".mib"),
MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles], MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles],
rebar_file_utils:delete_each( rebar_file_utils:delete_each(
@ -123,7 +115,7 @@ clean(Config, _AppFile) ->
|| F <- YrlFiles ]), || F <- YrlFiles ]),
%% Delete the build graph, if any %% Delete the build graph, if any
rebar_file_utils:rm_rf(erlcinfo_file(Config)), rebar_file_utils:rm_rf(erlcinfo_file()),
%% Erlang compilation is recursive, so it's possible that we have a nested %% Erlang compilation is recursive, so it's possible that we have a nested
%% directory structure in ebin with .beam files within. As such, we want %% directory structure in ebin with .beam files within. As such, we want
@ -292,144 +284,85 @@ doterl_compile(Config, OutDir) ->
doterl_compile(Config, OutDir, [], ErlOpts). doterl_compile(Config, OutDir, [], ErlOpts).
doterl_compile(Config, OutDir, MoreSources, ErlOpts) -> doterl_compile(Config, OutDir, MoreSources, ErlOpts) ->
ErlFirstFilesConf = rebar_config:get_list(Config, erl_first_files, []),
?DEBUG("erl_opts ~p~n", [ErlOpts]), ?DEBUG("erl_opts ~p~n", [ErlOpts]),
%% Support the src_dirs option allowing multiple directories to %% Support the src_dirs option allowing multiple directories to
%% contain erlang source. This might be used, for example, should %% contain erlang source. This might be used, for example, should
%% eunit tests be separated from the core application source. %% eunit tests be separated from the core application source.
SrcDirs = rebar_utils:src_dirs(proplists:append_values(src_dirs, ErlOpts)), SrcDirs = rebar_utils:src_dirs(proplists:append_values(src_dirs, ErlOpts)),
AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources, AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources,
%% NOTE: If and when erl_first_files is not inherited anymore
%% (rebar_config:get_local instead of rebar_config:get_list), consider
%% logging a warning message for any file listed in erl_first_files which
%% wasn't found via gather_src.
RestErls = [File || File <- AllErlFiles,
not lists:member(File, ErlFirstFilesConf)],
%% NOTE: order of files in ErlFirstFiles is important!
ErlFirstFiles = [File || File <- ErlFirstFilesConf,
lists:member(File, AllErlFiles)],
%% Make sure that ebin/ exists and is on the path %% Make sure that ebin/ exists and is on the path
ok = filelib:ensure_dir(filename:join("ebin", "dummy.beam")), ok = filelib:ensure_dir(filename:join("ebin", "dummy.beam")),
CurrPath = code:get_path(), CurrPath = code:get_path(),
true = code:add_path(filename:absname("ebin")), true = code:add_path(filename:absname("ebin")),
OutDir1 = proplists:get_value(outdir, ErlOpts, OutDir), OutDir1 = proplists:get_value(outdir, ErlOpts, OutDir),
G = init_erlcinfo(Config, AllErlFiles),
%% Split RestErls so that files which are depended on are treated
%% like erl_first_files.
{OtherFirstErls, OtherErls} =
lists:partition(
fun(F) ->
Children = get_children(G, F),
log_files(?FMT("Files dependent on ~s", [F]), Children),
case erls(Children) of G = init_erlcinfo(proplists:get_all_values(i, ErlOpts), AllErlFiles),
[] -> NeededErlFiles = needed_files(G, OutDir1, AllErlFiles),
%% There are no files dependent on this file. ErlFirstFiles = erl_first_files(Config, NeededErlFiles),
false; {DepErls, OtherErls} = lists:partition(
_ -> fun(Source) -> digraph:in_degree(G, Source) > 0 end,
%% There are some files dependent on the file. [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]),
%% Thus the file has higher priority DepErlsOrdered = digraph_utils:topsort(digraph_utils:subgraph(G, DepErls)),
%% and should be compiled in the first place. FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered),
true
end
end, RestErls),
%% Dependencies of OtherFirstErls that must be compiled first.
OtherFirstErlsDeps = lists:flatmap(
fun(Erl) -> erls(get_parents(G, Erl)) end,
OtherFirstErls),
%% NOTE: In case the way we retrieve OtherFirstErlsDeps or merge
%% it with OtherFirstErls does not result in the correct compile
%% priorities, or the method in use proves to be too slow for
%% certain projects, consider using a more elaborate method (maybe
%% digraph_utils) or alternatively getting and compiling the .erl
%% parents of an individual Source in internal_erl_compile. By not
%% handling this in internal_erl_compile, we also avoid extra
%% needs_compile/2 calls.
FirstErls = ErlFirstFiles ++ uo_merge(OtherFirstErlsDeps, OtherFirstErls),
?DEBUG("Files to compile first: ~p~n", [FirstErls]), ?DEBUG("Files to compile first: ~p~n", [FirstErls]),
rebar_base_compiler:run( rebar_base_compiler:run(
Config, FirstErls, OtherErls, Config, FirstErls, OtherErls,
fun(S, C) -> fun(S, C) -> internal_erl_compile(C, S, OutDir1, ErlOpts) end),
internal_erl_compile(C, S, OutDir1, ErlOpts, G)
end),
true = rebar_utils:cleanup_code_path(CurrPath), true = rebar_utils:cleanup_code_path(CurrPath),
ok. ok.
%% erl_first_files(Config, NeededErlFiles) ->
%% Return all .erl files from a list of files %% NOTE: rebar_config:get_local perhaps?
%% ErlFirstFilesConf = rebar_config:get_list(Config, erl_first_files, []),
erls(Files) -> %% NOTE: order of files in ErlFirstFiles is important!
[Erl || Erl <- Files, filename:extension(Erl) =:= ".erl"]. [File || File <- ErlFirstFilesConf, lists:member(File, NeededErlFiles)].
%% %% Get subset of SourceFiles which need to be recompiled, respecting
%% Return a list without duplicates while preserving order %% dependencies induced by given graph G.
%% needed_files(G, OutDir, SourceFiles) ->
ulist(L) -> lists:filter(fun(Source) ->
ulist(L, []). Target = target_base(OutDir, Source) ++ ".beam",
digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)}
end, SourceFiles).
ulist([H|T], Acc) -> target_base(OutDir, Source) ->
case lists:member(H, T) of filename:join(OutDir, filename:basename(Source, ".erl")).
true ->
ulist(T, Acc);
false ->
ulist(T, [H|Acc])
end;
ulist([], Acc) ->
lists:reverse(Acc).
%% erlcinfo_file() ->
%% Merge two lists without duplicates while preserving order
%%
uo_merge(L1, L2) ->
lists:foldl(fun(E, Acc) -> u_add_element(E, Acc) end, ulist(L1), L2).
u_add_element(Elem, [Elem|_]=Set) -> Set;
u_add_element(Elem, [E1|Set]) -> [E1|u_add_element(Elem, Set)];
u_add_element(Elem, []) -> [Elem].
-spec include_path(file:filename(),
rebar_config:config()) -> [file:filename(), ...].
include_path(Source, Config) ->
ErlOpts = rebar_config:get(Config, erl_opts, []),
lists:usort(["include", filename:dirname(Source)]
++ proplists:get_all_values(i, ErlOpts)).
-spec needs_compile(file:filename(), file:filename(),
[string()]) -> boolean().
needs_compile(Source, Target, Parents) ->
TargetLastMod = filelib:last_modified(Target),
lists:any(fun(I) -> TargetLastMod < filelib:last_modified(I) end,
[Source] ++ Parents).
check_erlcinfo(_Config, #erlcinfo{vsn=?ERLCINFO_VSN}) ->
ok;
check_erlcinfo(Config, #erlcinfo{vsn=Vsn}) ->
?ABORT("~s file version is incompatible. expected: ~b got: ~b~n",
[erlcinfo_file(Config), ?ERLCINFO_VSN, Vsn]);
check_erlcinfo(Config, _) ->
?ABORT("~s file is invalid. Please delete before next run.~n",
[erlcinfo_file(Config)]).
erlcinfo_file(_Config) ->
filename:join([rebar_utils:get_cwd(), ".rebar", ?ERLCINFO_FILE]). filename:join([rebar_utils:get_cwd(), ".rebar", ?ERLCINFO_FILE]).
init_erlcinfo(Config, Erls) -> %% Get dependency graph of given Erls files and their dependencies (header files,
G = restore_erlcinfo(Config), %% parse transforms, behaviours etc.) located in their directories or given
%% Get a unique list of dirs based on the source files' locations. %% InclDirs. Note that last modification times stored in vertices already respect
%% This is used for finding files in sub dirs of the configured %% dependencies induced by given graph G.
%% src_dirs. For example, src/sub_dir/foo.erl. init_erlcinfo(InclDirs, Erls) ->
Dirs = sets:to_list(lists:foldl( G = digraph:new([acyclic]),
fun(Erl, Acc) -> try restore_erlcinfo(G, InclDirs)
Dir = filename:dirname(Erl), catch
sets:add_element(Dir, Acc) _:_ ->
end, sets:new(), Erls)), ?WARN("Failed to restore ~s file. Discarding it.~n", [erlcinfo_file()]),
Updates = [update_erlcinfo(G, Erl, include_path(Erl, Config) ++ Dirs) ok = file:delete(erlcinfo_file())
|| Erl <- Erls], end,
Modified = lists:member(modified, Updates), Dirs = source_and_include_dirs(InclDirs, Erls),
ok = store_erlcinfo(G, Config, Modified), Modified = lists:foldl(update_erlcinfo_fun(G, Dirs), false, Erls),
if Modified -> store_erlcinfo(G, InclDirs); not Modified -> ok end,
G. G.
update_erlcinfo(G, Source, Dirs) -> source_and_include_dirs(InclDirs, Erls) ->
SourceDirs = lists:map(fun filename:dirname/1, Erls),
lists:usort(["include" | InclDirs ++ SourceDirs]).
update_erlcinfo_fun(G, Dirs) ->
fun(Erl, Modified) ->
case update_erlcinfo(G, Dirs, Erl) of
modified -> true;
unmodified -> Modified
end
end.
update_erlcinfo(G, Dirs, Source) ->
case digraph:vertex(G, Source) of case digraph:vertex(G, Source) of
{_, LastUpdated} -> {_, LastUpdated} ->
case filelib:last_modified(Source) of case filelib:last_modified(Source) of
@ -440,78 +373,68 @@ update_erlcinfo(G, Source, Dirs) ->
digraph:del_vertex(G, Source), digraph:del_vertex(G, Source),
modified; modified;
LastModified when LastUpdated < LastModified -> LastModified when LastUpdated < LastModified ->
modify_erlcinfo(G, Source, Dirs), modify_erlcinfo(G, Source, LastModified, Dirs);
modified;
_ -> _ ->
unmodified Modified = lists:foldl(
update_erlcinfo_fun(G, Dirs),
false, digraph:out_neighbours(G, Source)),
MaxModified = update_max_modified_deps(G, Source),
case Modified orelse MaxModified > LastUpdated of
true -> modified;
false -> unmodified
end
end; end;
false -> false ->
modify_erlcinfo(G, Source, Dirs), modify_erlcinfo(G, Source, filelib:last_modified(Source), Dirs)
modified
end. end.
modify_erlcinfo(G, Source, Dirs) -> update_max_modified_deps(G, Source) ->
MaxModified = lists:max(lists:map(
fun(File) -> {_, MaxModified} = digraph:vertex(G, File), MaxModified end,
[Source|digraph:out_neighbours(G, Source)])),
digraph:add_vertex(G, Source, MaxModified),
MaxModified.
modify_erlcinfo(G, Source, LastModified, Dirs) ->
{ok, Fd} = file:open(Source, [read]), {ok, Fd} = file:open(Source, [read]),
Incls = parse_attrs(Fd, []), Incls = parse_attrs(Fd, []),
AbsIncls = expand_file_names(Incls, Dirs), AbsIncls = expand_file_names(Incls, Dirs),
ok = file:close(Fd), ok = file:close(Fd),
LastUpdated = {date(), time()}, digraph:add_vertex(G, Source, LastModified),
digraph:add_vertex(G, Source, LastUpdated), digraph:del_edges(G, digraph:out_edges(G, Source)),
lists:foreach( lists:foreach(
fun(Incl) -> fun(Incl) ->
update_erlcinfo(G, Incl, Dirs), update_erlcinfo(G, Dirs, Incl),
digraph:add_edge(G, Source, Incl) digraph:add_edge(G, Source, Incl)
end, AbsIncls). end, AbsIncls),
modified.
restore_erlcinfo(Config) -> restore_erlcinfo(G, InclDirs) ->
File = erlcinfo_file(Config), case file:read_file(erlcinfo_file()) of
G = digraph:new(),
case file:read_file(File) of
{ok, Data} -> {ok, Data} ->
try binary_to_term(Data) of %% Since externally passed InclDirs can influence erlcinfo graph (see
Erlcinfo -> %% modify_erlcinfo), we have to check here that they didn't change.
ok = check_erlcinfo(Config, Erlcinfo), #erlcinfo{vsn=?ERLCINFO_VSN, info={Vs, Es, InclDirs}} =
#erlcinfo{info=ErlcInfo} = Erlcinfo, binary_to_term(Data),
{Vs, Es} = ErlcInfo, lists:foreach(
lists:foreach( fun({V, LastUpdated}) ->
fun({V, LastUpdated}) -> digraph:add_vertex(G, V, LastUpdated)
digraph:add_vertex(G, V, LastUpdated) end, Vs),
end, Vs), lists:foreach(
lists:foreach( fun({_, V1, V2, _}) ->
fun({V1, V2}) -> digraph:add_edge(G, V1, V2)
digraph:add_edge(G, V1, V2) end, Es);
end, Es) {error, _} ->
catch
error:badarg ->
?ERROR(
"Failed (binary_to_term) to restore rebar info file."
" Discard file.~n", []),
ok
end;
_Err ->
ok ok
end, end.
G.
store_erlcinfo(_G, _Config, _Modified = false) -> store_erlcinfo(G, InclDirs) ->
ok; Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
store_erlcinfo(G, Config, _Modified) -> Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
Vs = lists:map( File = erlcinfo_file(),
fun(V) ->
digraph:vertex(G, V)
end, digraph:vertices(G)),
Es = lists:flatmap(
fun({V, _}) ->
lists:map(
fun(E) ->
{_, V1, V2, _} = digraph:edge(G, E),
{V1, V2}
end, digraph:out_edges(G, V))
end, Vs),
File = erlcinfo_file(Config),
ok = filelib:ensure_dir(File), ok = filelib:ensure_dir(File),
Data = term_to_binary(#erlcinfo{info={Vs, Es}}, [{compressed, 9}]), Data = term_to_binary(#erlcinfo{info={Vs, Es, InclDirs}}, [{compressed, 2}]),
file:write_file(File, Data). ok = file:write_file(File, Data).
%% NOTE: If, for example, one of the entries in Files refers to %% NOTE: If, for example, one of the entries in Files refers to
%% gen_server.erl, that entry will be dropped. It is dropped because %% gen_server.erl, that entry will be dropped. It is dropped because
@ -545,46 +468,18 @@ expand_file_names(Files, Dirs) ->
end end
end, Files). end, Files).
-spec get_parents(rebar_digraph(), file:filename()) -> [file:filename()].
get_parents(G, Source) ->
%% Return all files which the Source depends upon.
digraph_utils:reachable_neighbours([Source], G).
-spec get_children(rebar_digraph(), file:filename()) -> [file:filename()].
get_children(G, Source) ->
%% Return all files dependent on the Source.
digraph_utils:reaching_neighbours([Source], G).
-spec internal_erl_compile(rebar_config:config(), file:filename(), -spec internal_erl_compile(rebar_config:config(), file:filename(),
file:filename(), list(), file:filename(), list()) -> ok | {ok, any()} | {error, any(), any()}.
rebar_digraph()) -> 'ok' | 'skipped'. internal_erl_compile(Config, Source, OutDir, ErlOpts) ->
internal_erl_compile(Config, Source, OutDir, ErlOpts, G) -> ok = filelib:ensure_dir(OutDir),
%% Determine the target name and includes list by inspecting the source file Opts = [{outdir, OutDir}] ++ ErlOpts ++ [{i, "include"}, return],
Module = filename:basename(Source, ".erl"), case compile:file(Source, Opts) of
Parents = get_parents(G, Source), {ok, _Mod} ->
log_files(?FMT("Dependencies of ~s", [Source]), Parents), ok;
{ok, _Mod, Ws} ->
%% Construct the target filename rebar_base_compiler:ok_tuple(Config, Source, Ws);
Target = filename:join([OutDir | string:tokens(Module, ".")]) ++ ".beam", {error, Es, Ws} ->
ok = filelib:ensure_dir(Target), rebar_base_compiler:error_tuple(Config, Source, Es, Ws, Opts)
%% If the file needs compilation, based on last mod date of includes or
%% the target
case needs_compile(Source, Target, Parents) of
true ->
Opts = [{outdir, filename:dirname(Target)}] ++
ErlOpts ++ [{i, "include"}, return],
case compile:file(Source, Opts) of
{ok, _Mod} ->
ok;
{ok, _Mod, Ws} ->
rebar_base_compiler:ok_tuple(Config, Source, Ws);
{error, Es, Ws} ->
rebar_base_compiler:error_tuple(Config, Source,
Es, Ws, Opts)
end;
false ->
skipped
end. end.
-spec compile_mib(file:filename(), file:filename(), -spec compile_mib(file:filename(), file:filename(),
@ -627,7 +522,7 @@ compile_yrl(Source, Target, Config) ->
-spec compile_xrl_yrl(rebar_config:config(), file:filename(), -spec compile_xrl_yrl(rebar_config:config(), file:filename(),
file:filename(), list(), module()) -> 'ok'. file:filename(), list(), module()) -> 'ok'.
compile_xrl_yrl(Config, Source, Target, Opts, Mod) -> compile_xrl_yrl(Config, Source, Target, Opts, Mod) ->
case needs_compile(Source, Target, []) of case needs_compile(Source, Target) of
true -> true ->
case Mod:file(Source, Opts ++ [{return, true}]) of case Mod:file(Source, Opts ++ [{return, true}]) of
{ok, _} -> {ok, _} ->
@ -642,6 +537,9 @@ compile_xrl_yrl(Config, Source, Target, Opts, Mod) ->
skipped skipped
end. end.
needs_compile(Source, Target) ->
filelib:last_modified(Source) > filelib:last_modified(Target).
gather_src([], Srcs) -> gather_src([], Srcs) ->
Srcs; Srcs;
gather_src([Dir|Rest], Srcs) -> gather_src([Dir|Rest], Srcs) ->
@ -676,21 +574,15 @@ parse_attrs(Fd, Includes) ->
end. end.
process_attr(Form, Includes) -> process_attr(Form, Includes) ->
try AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)),
AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)), process_attr(AttrName, Form, Includes).
process_attr(AttrName, Form, Includes)
catch _:_ ->
%% TODO: We should probably try to be more specific here
%% and not suppress all errors.
Includes
end.
process_attr(import, Form, Includes) -> process_attr(import, Form, Includes) ->
case erl_syntax_lib:analyze_import_attribute(Form) of case erl_syntax_lib:analyze_import_attribute(Form) of
{Mod, _Funs} -> {Mod, _Funs} ->
[atom_to_list(Mod) ++ ".erl"|Includes]; [module_to_erl(Mod)|Includes];
Mod -> Mod ->
[atom_to_list(Mod) ++ ".erl"|Includes] [module_to_erl(Mod)|Includes]
end; end;
process_attr(file, Form, Includes) -> process_attr(file, Form, Includes) ->
{File, _} = erl_syntax_lib:analyze_file_attribute(Form), {File, _} = erl_syntax_lib:analyze_file_attribute(Form),
@ -702,29 +594,35 @@ process_attr(include, Form, Includes) ->
process_attr(include_lib, Form, Includes) -> process_attr(include_lib, Form, Includes) ->
[FileNode] = erl_syntax:attribute_arguments(Form), [FileNode] = erl_syntax:attribute_arguments(Form),
RawFile = erl_syntax:string_value(FileNode), RawFile = erl_syntax:string_value(FileNode),
File = maybe_expand_include_lib_path(RawFile), maybe_expand_include_lib_path(RawFile) ++ Includes;
[File|Includes];
process_attr(behaviour, Form, Includes) -> process_attr(behaviour, Form, Includes) ->
[FileNode] = erl_syntax:attribute_arguments(Form), [FileNode] = erl_syntax:attribute_arguments(Form),
File = erl_syntax:atom_name(FileNode) ++ ".erl", File = module_to_erl(erl_syntax:atom_value(FileNode)),
[File|Includes]; [File|Includes];
process_attr(compile, Form, Includes) -> process_attr(compile, Form, Includes) ->
[Arg] = erl_syntax:attribute_arguments(Form), [Arg] = erl_syntax:attribute_arguments(Form),
case erl_syntax:concrete(Arg) of case erl_syntax:concrete(Arg) of
{parse_transform, Mod} -> {parse_transform, Mod} ->
[atom_to_list(Mod) ++ ".erl"|Includes]; [module_to_erl(Mod)|Includes];
{core_transform, Mod} -> {core_transform, Mod} ->
[atom_to_list(Mod) ++ ".erl"|Includes]; [module_to_erl(Mod)|Includes];
L when is_list(L) -> L when is_list(L) ->
lists:foldl( lists:foldl(
fun({parse_transform, M}, Acc) -> fun({parse_transform, Mod}, Acc) ->
[atom_to_list(M) ++ ".erl"|Acc]; [module_to_erl(Mod)|Acc];
({core_transform, M}, Acc) -> ({core_transform, Mod}, Acc) ->
[atom_to_list(M) ++ ".erl"|Acc]; [module_to_erl(Mod)|Acc];
(_, Acc) -> (_, Acc) ->
Acc Acc
end, Includes, L) end, Includes, L);
end. _ ->
Includes
end;
process_attr(_, _Form, Includes) ->
Includes.
module_to_erl(Mod) ->
atom_to_list(Mod) ++ ".erl".
%% Given the filename from an include_lib attribute, if the path %% Given the filename from an include_lib attribute, if the path
%% exists, return unmodified, or else get the absolute ERL_LIBS %% exists, return unmodified, or else get the absolute ERL_LIBS
@ -732,7 +630,7 @@ process_attr(compile, Form, Includes) ->
maybe_expand_include_lib_path(File) -> maybe_expand_include_lib_path(File) ->
case filelib:is_regular(File) of case filelib:is_regular(File) of
true -> true ->
File; [File];
false -> false ->
expand_include_lib_path(File) expand_include_lib_path(File)
end. end.
@ -747,8 +645,10 @@ expand_include_lib_path(File) ->
Split = filename:split(filename:dirname(File)), Split = filename:split(filename:dirname(File)),
Lib = hd(Split), Lib = hd(Split),
SubDir = filename:join(tl(Split)), SubDir = filename:join(tl(Split)),
Dir = code:lib_dir(list_to_atom(Lib), list_to_atom(SubDir)), case code:lib_dir(list_to_atom(Lib), list_to_atom(SubDir)) of
filename:join(Dir, File1). {error, bad_name} -> [];
Dir -> [filename:join(Dir, File1)]
end.
%% %%
%% Ensure all files in a list are present and abort if one is missing %% Ensure all files in a list are present and abort if one is missing
@ -762,13 +662,3 @@ check_file(File) ->
false -> ?ABORT("File ~p is missing, aborting\n", [File]); false -> ?ABORT("File ~p is missing, aborting\n", [File]);
true -> File true -> File
end. end.
%% Print prefix followed by list of files. If the list is empty, print
%% on the same line, otherwise use a separate line.
log_files(Prefix, Files) ->
case Files of
[] ->
?DEBUG("~s: ~p~n", [Prefix, Files]);
_ ->
?DEBUG("~s:~n~p~n", [Prefix, Files])
end.