From c7a2b450bb26ed3fee2a6f23266ca4e336bce62a Mon Sep 17 00:00:00 2001 From: David Kubecka Date: Sat, 28 Mar 2015 23:37:37 +0100 Subject: [PATCH] Store also max modified times in erlcinfo These times already contain max modified info of their dependencies. Therefore we no longer have to check in internal_erl_compile whether the file really needs recompiling, which simplifies the flow somewhat, because the work with dependency graph is now localized to much smaller space then before. --- inttest/erlc_dep_graph/erlc_dep_graph_rt.erl | 3 + inttest/erlc_dep_graph/src/lisp.erl | 2 + inttest/erlc_dep_graph/src/perl.erl | 10 ++ src/rebar_erlc_compiler.erl | 126 ++++++++----------- 4 files changed, 66 insertions(+), 75 deletions(-) create mode 100644 inttest/erlc_dep_graph/src/perl.erl diff --git a/inttest/erlc_dep_graph/erlc_dep_graph_rt.erl b/inttest/erlc_dep_graph/erlc_dep_graph_rt.erl index fb128c6..384ce87 100644 --- a/inttest/erlc_dep_graph/erlc_dep_graph_rt.erl +++ b/inttest/erlc_dep_graph/erlc_dep_graph_rt.erl @@ -62,6 +62,9 @@ run(_Dir) -> %% Clean up ok = file:write_file(Java, OrigContent), + %% Check that changes propagate deeply through the dependency tree + modify_and_recompile_ok("include/lambda.hrl", "ebin/perl.beam"), + ok. check_beams_ok() -> diff --git a/inttest/erlc_dep_graph/src/lisp.erl b/inttest/erlc_dep_graph/src/lisp.erl index dd418d2..eceb157 100644 --- a/inttest/erlc_dep_graph/src/lisp.erl +++ b/inttest/erlc_dep_graph/src/lisp.erl @@ -4,5 +4,7 @@ -export([parse_transform/2]). +-include("lambda.hrl"). + parse_transform(Forms, _Options) -> Forms. diff --git a/inttest/erlc_dep_graph/src/perl.erl b/inttest/erlc_dep_graph/src/perl.erl new file mode 100644 index 0000000..9687948 --- /dev/null +++ b/inttest/erlc_dep_graph/src/perl.erl @@ -0,0 +1,10 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 ft=erlang et +-module(perl). + +-export(['$_'/0]). + +-compile({parse_transform, lisp}). + +'$_'() -> + anything. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 32e1083..31ed23e 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -47,14 +47,6 @@ info = {[], [], []} :: erlc_info() }). --ifdef(namespaced_types). -%% digraph:graph() exists starting from Erlang 17. --type rebar_digraph() :: digraph:graph(). --else. -%% digraph() has been obsoleted in Erlang 17 and deprecated in 18. --type rebar_digraph() :: digraph(). --endif. - %% =================================================================== %% Public API %% =================================================================== @@ -306,47 +298,42 @@ doterl_compile(Config, OutDir, MoreSources, ErlOpts) -> OutDir1 = proplists:get_value(outdir, ErlOpts, OutDir), G = init_erlcinfo(proplists:get_all_values(i, ErlOpts), AllErlFiles), - ErlFirstFiles = erl_first_files(Config, AllErlFiles), + NeededErlFiles = needed_files(G, OutDir1, AllErlFiles), + ErlFirstFiles = erl_first_files(Config, NeededErlFiles), {DepErls, OtherErls} = lists:partition( fun(Source) -> digraph:in_degree(G, Source) > 0 end, - [File || File <- AllErlFiles, not lists:member(File, ErlFirstFiles)]), + [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]), DepErlsOrdered = digraph_utils:topsort(digraph_utils:subgraph(G, DepErls)), FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered), ?DEBUG("Files to compile first: ~p~n", [FirstErls]), rebar_base_compiler:run( Config, FirstErls, OtherErls, - fun(S, C) -> - internal_erl_compile(C, S, OutDir1, ErlOpts, G) - end), + fun(S, C) -> internal_erl_compile(C, S, OutDir1, ErlOpts) end), true = rebar_utils:cleanup_code_path(CurrPath), ok. -erl_first_files(Config, AllErlFiles) -> +erl_first_files(Config, NeededErlFiles) -> + %% NOTE: rebar_config:get_local perhaps? ErlFirstFilesConf = rebar_config:get_list(Config, erl_first_files, []), - %% NOTE: If and when erl_first_files is not inherited anymore - %% (rebar_config:get_local instead of rebar_config:get_list), consider - %% logging a warning message for any file listed in erl_first_files which - %% is not in AllErlFiles. %% NOTE: order of files in ErlFirstFiles is important! - [File || File <- ErlFirstFilesConf, lists:member(File, AllErlFiles)]. + [File || File <- ErlFirstFilesConf, lists:member(File, NeededErlFiles)]. --spec needs_compile(file:filename(), file:filename(), - [string()]) -> boolean(). -needs_compile(Source, Target, Parents) -> - TargetLastMod = filelib:last_modified(Target), - lists:any(fun(I) -> TargetLastMod < filelib:last_modified(I) end, - [Source] ++ Parents). +%% Get subset of SourceFiles which need to be recompiled, respecting +%% dependencies induced by given graph G. +needed_files(G, OutDir, SourceFiles) -> + lists:filter(fun(Source) -> + Target = target_base(OutDir, Source) ++ ".beam", + digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)} + end, SourceFiles). erlcinfo_file() -> filename:join([rebar_utils:get_cwd(), ".rebar", ?ERLCINFO_FILE]). %% Get dependency graph of given Erls files and their dependencies (header files, %% parse transforms, behaviours etc.) located in their directories or given -%% InclDirs. Note that last modification times stored in vertices are only for -%% internal optimization and cannot be directly used for deciding whether to -%% recompile a file, since when the file itself doesn't change we don't check its -%% dependencies which might change. +%% InclDirs. Note that last modification times stored in vertices already respect +%% dependencies induced by given graph G. init_erlcinfo(InclDirs, Erls) -> G = digraph:new([acyclic]), try restore_erlcinfo(G, InclDirs) @@ -385,12 +372,26 @@ update_erlcinfo(G, Dirs, Source) -> LastModified when LastUpdated < LastModified -> modify_erlcinfo(G, Source, LastModified, Dirs); _ -> - unmodified + Modified = lists:foldl( + update_erlcinfo_fun(G, Dirs), + false, digraph:out_neighbours(G, Source)), + MaxModified = update_max_modified_deps(G, Source), + case Modified orelse MaxModified > LastUpdated of + true -> modified; + false -> unmodified + end end; false -> modify_erlcinfo(G, Source, filelib:last_modified(Source), Dirs) end. +update_max_modified_deps(G, Source) -> + MaxModified = lists:max(lists:map( + fun(File) -> {_, MaxModified} = digraph:vertex(G, File), MaxModified end, + [Source|digraph:out_neighbours(G, Source)])), + digraph:add_vertex(G, Source, MaxModified), + MaxModified. + modify_erlcinfo(G, Source, LastModified, Dirs) -> {ok, Fd} = file:open(Source, [read]), Incls = parse_attrs(Fd, []), @@ -464,43 +465,25 @@ expand_file_names(Files, Dirs) -> end end, Files). --spec get_parents(rebar_digraph(), file:filename()) -> [file:filename()]. -get_parents(G, Source) -> - %% Return all files which the Source depends upon. - digraph_utils:reachable_neighbours([Source], G). - -spec internal_erl_compile(rebar_config:config(), file:filename(), - file:filename(), list(), - rebar_digraph()) -> 'ok' | 'skipped'. -internal_erl_compile(Config, Source, OutDir, ErlOpts, G) -> - %% Determine the target name and includes list by inspecting the source file - Module = filename:basename(Source, ".erl"), - Parents = get_parents(G, Source), - log_files(?FMT("Dependencies of ~s", [Source]), Parents), - - %% Construct the target filename - Target = filename:join([OutDir | string:tokens(Module, ".")]) ++ ".beam", - ok = filelib:ensure_dir(Target), - - %% If the file needs compilation, based on last mod date of includes or - %% the target - case needs_compile(Source, Target, Parents) of - true -> - Opts = [{outdir, filename:dirname(Target)}] ++ - ErlOpts ++ [{i, "include"}, return], - case compile:file(Source, Opts) of - {ok, _Mod} -> - ok; - {ok, _Mod, Ws} -> - rebar_base_compiler:ok_tuple(Config, Source, Ws); - {error, Es, Ws} -> - rebar_base_compiler:error_tuple(Config, Source, - Es, Ws, Opts) - end; - false -> - skipped + file:filename(), list()) -> ok | {ok, any()} | {error, any(), any()}. +internal_erl_compile(Config, Source, OutDir, ErlOpts) -> + TargetDir = filename:dirname(target_base(OutDir, Source)), + ok = filelib:ensure_dir(TargetDir), + Opts = [{outdir, TargetDir}] ++ ErlOpts ++ [{i, "include"}, return], + case compile:file(Source, Opts) of + {ok, _Mod} -> + ok; + {ok, _Mod, Ws} -> + rebar_base_compiler:ok_tuple(Config, Source, Ws); + {error, Es, Ws} -> + rebar_base_compiler:error_tuple(Config, Source, Es, Ws, Opts) end. +target_base(OutDir, Source) -> + Module = filename:basename(Source, ".erl"), + filename:join([OutDir|string:tokens(Module, ".")]). + -spec compile_mib(file:filename(), file:filename(), rebar_config:config()) -> 'ok'. compile_mib(Source, Target, Config) -> @@ -541,7 +524,7 @@ compile_yrl(Source, Target, Config) -> -spec compile_xrl_yrl(rebar_config:config(), file:filename(), file:filename(), list(), module()) -> 'ok'. compile_xrl_yrl(Config, Source, Target, Opts, Mod) -> - case needs_compile(Source, Target, []) of + case needs_compile(Source, Target) of true -> case Mod:file(Source, Opts ++ [{return, true}]) of {ok, _} -> @@ -556,6 +539,9 @@ compile_xrl_yrl(Config, Source, Target, Opts, Mod) -> skipped end. +needs_compile(Source, Target) -> + filelib:last_modified(Source) > filelib:last_modified(Target). + gather_src([], Srcs) -> Srcs; gather_src([Dir|Rest], Srcs) -> @@ -677,13 +663,3 @@ check_file(File) -> false -> ?ABORT("File ~p is missing, aborting\n", [File]); true -> File end. - -%% Print prefix followed by list of files. If the list is empty, print -%% on the same line, otherwise use a separate line. -log_files(Prefix, Files) -> - case Files of - [] -> - ?DEBUG("~s: ~p~n", [Prefix, Files]); - _ -> - ?DEBUG("~s:~n~p~n", [Prefix, Files]) - end.