From 9c15d63191695a8121cb5566817ed2123c59567b Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Sat, 2 Jan 2010 14:17:59 -0700 Subject: [PATCH] Refactor of compilation logic to enable support for package modules and provide better modularity/reusability for other systems --- ebin/rebar.app | 3 +- src/rebar_base_compiler.erl | 197 +++++++++++++++++++++++++++++++++ src/rebar_erlc_compiler.erl | 186 ++++++++----------------------- src/rebar_erlydtl_compiler.erl | 20 ++-- src/rebar_eunit.erl | 15 ++- src/rebar_lfe_compiler.erl | 7 +- src/rebar_otp_app.erl | 12 +- 7 files changed, 274 insertions(+), 166 deletions(-) create mode 100644 src/rebar_base_compiler.erl diff --git a/ebin/rebar.app b/ebin/rebar.app index c2f7967..bd76ff7 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -3,7 +3,8 @@ {vsn, "2"}, {modules, [ rebar, rebar_app_utils, - rebar_config, + rebar_base_compiler, + rebar_config, rebar_core, rebar_ct, rebar_deps, diff --git a/src/rebar_base_compiler.erl b/src/rebar_base_compiler.erl new file mode 100644 index 0000000..b6172c2 --- /dev/null +++ b/src/rebar_base_compiler.erl @@ -0,0 +1,197 @@ +%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% ------------------------------------------------------------------- +%% +%% rebar: Erlang Build Tools +%% +%% Copyright (c) 2009 Dave Smith (dizzyd@dizzyd.com) +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ------------------------------------------------------------------- +-module(rebar_base_compiler). + +-include("rebar.hrl"). + +-export([run/8]). + + +%% =================================================================== +%% Public API +%% =================================================================== + +run(Config, SourceDir, SourceExt, TargetDir, TargetExt, + FirstFiles, CompileFn, Opts) -> + SourceExtRe = ".*\\" ++ SourceExt ++ [$$], + + %% Options: + %% recurse_source_dir + %% needs_compile_checks - [ fun/2 ] + Recursive = proplists:get_bool(recurse_source_dir, Opts), + + %% Find all the source files we can + FoundFiles = filelib:fold_files(SourceDir, SourceExtRe, Recursive, + fun(F, Acc) -> [F | Acc] end, []), + + %% Construct two lists of targets. "FirstTargets" is the list of files which + %% must be compiled first and in strict order; "RestTargets" is all remaining files + %% that may be compiled in any order. + FirstTargets = [{Fs, target_file(Fs, SourceDir, SourceExt, TargetDir, TargetExt)} || + Fs <- FirstFiles], + RestTargets = [{Fs, target_file(Fs, SourceDir, SourceExt, TargetDir, TargetExt)} || + Fs <- drop_each(FirstFiles, FoundFiles)], + + %% Setup list of functions which determine if a file needs compilation or not. By + %% default we just check the last modified date + NeedsCompileFns = [ fun check_source_lastmod/3 ] ++ + rebar_config:get(Config, needs_compile_checks, []), + + %% Compile the first targets in sequence + compile_each(FirstTargets, Config, NeedsCompileFns, CompileFn), + + %% Spin up workers + case RestTargets of + [] -> + ok; + _ -> + Self = self(), + F = fun() -> compile_worker(Self, Config, NeedsCompileFns, CompileFn) end, + Pids = [spawn_monitor(F) || _I <- lists:seq(1,3)], + compile_queue(Pids, RestTargets) + end. + + +%% =================================================================== +%% Internal functions +%% =================================================================== + +target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> + %% Remove all leading components of the source dir from the file -- we want + %% to maintain the deeper structure (if any) of the source file path + BaseFile = remove_common_path(SourceFile, SourceDir), + filename:join([TargetDir, filename:dirname(BaseFile), + filename:basename(BaseFile, SourceExt) ++ TargetExt]). + + +remove_common_path(Fname, Path) -> + remove_common_path1(filename:split(Fname), filename:split(Path)). + +remove_common_path1([Part | RestFilename], [Part | RestPath]) -> + remove_common_path1(RestFilename, RestPath); +remove_common_path1(FilenameParts, _) -> + filename:join(FilenameParts). + + +drop_each([], List) -> + List; +drop_each([Member | Rest], List) -> + drop_each(Rest, lists:delete(Member, List)). + + +needs_compile(_SourceFile, _TargetFile, _Config, []) -> + false; +needs_compile(SourceFile, TargetFile, Config, [Fn | Rest]) -> + case Fn(SourceFile, TargetFile, Config) of + true -> + true; + false -> + needs_compile(SourceFile, TargetFile, Config, Rest) + end. + +check_source_lastmod(SourceFile, TargetFile, _Config) -> + filelib:last_modified(TargetFile) < filelib:last_modified(SourceFile). + +compile(Source, Target, Config, NeedsCompileFns, CompileFn) -> + case needs_compile(Source, Target, Config, NeedsCompileFns) of + true -> + ok = filelib:ensure_dir(Target), + CompileFn(Source, Target, Config); + false -> + skipped + end. + + + +compile_each([], _Config, _NeedsCompileFns, _CompileFn) -> + ok; +compile_each([{Source, Target} | Rest], Config, NeedsCompileFns, CompileFn) -> + case compile(Source, Target, Config, NeedsCompileFns, CompileFn) of + ok -> + ?CONSOLE("Compiled ~s\n", [Source]); + skipped -> + ?INFO("Skipped ~s\n", [Source]) + end, + compile_each(Rest, Config, NeedsCompileFns, CompileFn). + + + +compile_queue([], []) -> + ok; +compile_queue(Pids, Targets) -> + receive + {next, Worker} -> + case Targets of + [] -> + Worker ! empty, + compile_queue(Pids, Targets); + [{Source, Target} | Rest] -> + Worker ! {compile, Source, Target}, + compile_queue(Pids, Rest) + end; + + {fail, Error} -> + ?DEBUG("Worker compilation failed: ~p\n", [Error]), + ?FAIL; + + {compiled, Source} -> + ?CONSOLE("Compiled ~s\n", [Source]), + compile_queue(Pids, Targets); + + {skipped, Source} -> + ?INFO("Skipped ~s\n", [Source]), + compile_queue(Pids, Targets); + + {'DOWN', Mref, _, Pid, normal} -> + ?DEBUG("Worker exited cleanly\n", []), + Pids2 = lists:delete({Pid, Mref}, Pids), + compile_queue(Pids2, Targets); + + {'DOWN', _Mref, _, _Pid, Info} -> + ?DEBUG("Worker failed: ~p\n", [Info]), + ?FAIL + end. + +compile_worker(QueuePid, Config, NeedsCompileFns, CompileFn) -> + QueuePid ! {next, self()}, + receive + {compile, Source, Target} -> + case catch(compile(Source, Target, Config, NeedsCompileFns, CompileFn)) of + ok -> + QueuePid ! {compiled, Source}, + compile_worker(QueuePid, Config, NeedsCompileFns, CompileFn); + skipped -> + QueuePid ! {skipped, Source}, + compile_worker(QueuePid, Config, NeedsCompileFns, CompileFn); + Error -> + QueuePid ! {fail, Error}, + ok + end; + + empty -> + ok + end. diff --git a/src/rebar_erlc_compiler.erl b/src/rebar_erlc_compiler.erl index 68b1d75..4e083ab 100644 --- a/src/rebar_erlc_compiler.erl +++ b/src/rebar_erlc_compiler.erl @@ -30,7 +30,7 @@ clean/2]). %% make available for rebar_eunit until there is a better option --export([do_compile/8, compile_opts/2, list_hrls/2]). +-export([hrls_check/3]). -include("rebar.hrl"). @@ -39,18 +39,30 @@ %% =================================================================== compile(Config, _AppFile) -> - do_compile(Config, "src/*.erl", "ebin", ".erl", ".beam", - fun list_hrls/2, fun compile_erl/2, - rebar_config:get_list(Config, erl_first_files, [])), - do_compile(Config, "mibs/*.mib", "priv/mibs", ".mib", ".bin", - undefined, fun compile_mib/2, - rebar_config:get_list(Config, mib_first_files, [])). + rebar_base_compiler:run(Config, "src", ".erl", "ebin", ".beam", + rebar_config:get_list(Config, erl_first_files, []), + fun compile_erl/3, + [recurse_source_dir, + {needs_compile_checks, [fun hrls_check/3]}]), + + rebar_base_compiler:run(Config, "mibs", ".mib", "priv/mibs", ".bin", + rebar_config:get_list(Config, mib_first_files, []), + fun compile_mib/3, + []). clean(_Config, _AppFile) -> %% TODO: This would be more portable if it used Erlang to traverse %% the dir structure and delete each file; however it would also %% much slower. - [] = os:cmd("rm -f ebin/*.beam priv/mibs/*.bin"), + ok = rebar_file_utils:rm_rf("ebin/*.beam priv/mibs/*.bin"), + + %% Erlang compilation is recursive, so it's possible that we have a nested + %% directory structure in ebin with .beam files within. As such, we want + %% to scan whatever is left in the ebin/ directory for sub-dirs which + %% satisfy our criteria. TODO: Is there a better way to do this? + BeamFiles = filelib:fold_files("ebin", "^.*\\.beam\$", true, + fun(F, BeamFiles) -> BeamFiles ++ [F] end, []), + rebar_file_utils:delete_each(BeamFiles), ok. @@ -59,71 +71,11 @@ clean(_Config, _AppFile) -> %% Internal functions %% =================================================================== -do_compile(Config, SrcWildcard, OutDir, InExt, OutExt, - IncludeFn, CompileFn, FirstFiles) -> - case filelib:wildcard(SrcWildcard) of - [] -> - ok; - FoundFiles when is_list(FoundFiles) -> - %% Ensure that the FirstFiles are compiled first; drop them from the - %% FoundFiles and compile them in sequence - FirstTargets = [{Fs, target_file(Fs, OutDir, InExt, OutExt)} || Fs <- FirstFiles], - RestTargets = [{Fs, target_file(Fs, OutDir, InExt, OutExt)} || - Fs <- drop_each(FirstFiles, FoundFiles)], +hrls_check(Source, Target, Config) -> + TargetLastMod = filelib:last_modified(Target), + lists:any(fun(I) -> TargetLastMod < filelib:last_modified(I) end, + list_hrls(Source, Config)). - %% Make sure target directory exists - ok = filelib:ensure_dir(target_file(hd(FoundFiles), OutDir, InExt, OutExt)), - - %% Compile first targets in sequence - compile_each(FirstTargets, Config, IncludeFn, CompileFn), - - %% Spin up workers - Self = self(), - Pids = [spawn_monitor(fun() -> compile_worker(Self) end) || _I <- lists:seq(1,3)], - - %% Process rest of targets - compile_queue(Pids, RestTargets, Config, IncludeFn, CompileFn) - end. - -drop_each([], List) -> - List; -drop_each([Member | Rest], List) -> - drop_each(Rest, lists:delete(Member, List)). - -compile_each([], _Config, _IncludeFn, _CompileFn) -> - ok; -compile_each([{Src, Target} | Rest], Config, IncludeFn, CompileFn) -> - case needs_compile(Src, Target, IncludeFn, Config) of - true -> - ?CONSOLE("Compiling ~s\n", [Src]), - CompileFn(Src, Config); - false -> - ?INFO("Skipping ~s\n", [Src]), - ok - end, - compile_each(Rest, Config, IncludeFn, CompileFn). - -needs_compile(Src, Target, IncludeFn, Config) -> - TargetLM = filelib:last_modified(Target), - case TargetLM < filelib:last_modified(Src) of - true -> - true; - false -> - if is_function(IncludeFn) -> - lists:any(fun(I) -> - TargetLM < filelib:last_modified(I) - end, - IncludeFn(Src, Config)); - true -> - false - end - end. - -target_file(F, TargetDir, InExt, OutExt) -> - filename:join([TargetDir, filename:basename(F, InExt) ++ OutExt]). - -compile_opts(Config, Key) -> - rebar_config:get_list(Config, Key, []). list_hrls(Src, Config) -> case epp:open(Src, include_path(Src, Config)) of @@ -134,26 +86,31 @@ list_hrls(Src, Config) -> false end. -include_path(Src, Config) -> - [filename:dirname(Src)|compile_opts(Config, i)]. extract_includes(Epp, Src) -> case epp:parse_erl_form(Epp) of - {ok, {attribute, 1, file, {Src, 1}}} -> - extract_includes(Epp, Src); - {ok, {attribute, 1, file, {IncFile, 1}}} -> + {ok, {attribute, 1, file, {Src, 1}}} -> + extract_includes(Epp, Src); + {ok, {attribute, 1, file, {IncFile, 1}}} -> [IncFile|extract_includes(Epp, Src)]; - {ok, _} -> - extract_includes(Epp, Src); - {eof, _} -> - epp:close(Epp), - []; - {error, _Error} -> - extract_includes(Epp, Src) + {ok, _} -> + extract_includes(Epp, Src); + {eof, _} -> + epp:close(Epp), + []; + {error, _Error} -> + extract_includes(Epp, Src) end. -compile_erl(Source, Config) -> - Opts = [{i, "include"}, {outdir, "ebin"}, report, return] ++ compile_opts(Config, erl_opts), +include_path(Source, Config) -> + [filename:dirname(Source) | compile_opts(Config, i)]. + +compile_opts(Config, Key) -> + rebar_config:get_list(Config, Key, []). + +compile_erl(Source, Target, Config) -> + Opts = [{i, "include"}, {outdir, filename:dirname(Target)}, report, return] ++ + compile_opts(Config, erl_opts), case compile:file(Source, Opts) of {ok, _, []} -> ok; @@ -169,7 +126,7 @@ compile_erl(Source, Config) -> ?FAIL end. -compile_mib(Source, Config) -> +compile_mib(Source, _Target, Config) -> Opts = [{outdir, "priv/mibs"}, {i, ["priv/mibs"]}] ++ compile_opts(Config, mib_opts), case snmpc:compile(Source, Opts) of {ok, _} -> @@ -177,58 +134,3 @@ compile_mib(Source, Config) -> {error, compilation_failed} -> ?FAIL end. - -compile_queue([], [], _Config, _IncludeFn, _CompileFn) -> - ok; -compile_queue(Pids, Targets, Config, IncludeFn, CompileFn) -> - receive - {next, Worker} -> - case Targets of - [] -> - Worker ! empty, - compile_queue(Pids, Targets, Config, IncludeFn, CompileFn); - [{Src, Target} | Rest] -> - Worker ! {compile, Src, Target, Config, IncludeFn, CompileFn}, - compile_queue(Pids, Rest, Config, IncludeFn, CompileFn) - end; - - {fail, Error} -> - ?DEBUG("Worker compilation failed: ~p\n", [Error]), - ?FAIL; - - {compiled, Source} -> - ?CONSOLE("Compiled ~s\n", [Source]), - compile_queue(Pids, Targets, Config, IncludeFn, CompileFn); - - {'DOWN', Mref, _, Pid, normal} -> - ?DEBUG("Worker exited cleanly\n", []), - Pids2 = lists:delete({Pid, Mref}, Pids), - compile_queue(Pids2, Targets, Config, IncludeFn, CompileFn); - - {'DOWN', _Mref, _, _Pid, Info} -> - ?DEBUG("Worker failed: ~p\n", [Info]), - ?FAIL - end. - -compile_worker(QueuePid) -> - QueuePid ! {next, self()}, - receive - {compile, Src, Target, Config, IncludeFn, CompileFn} -> - case needs_compile(Src, Target, IncludeFn, Config) of - true -> - case catch(CompileFn(Src, Config)) of - ok -> - QueuePid ! {compiled, Src}, - compile_worker(QueuePid); - Error -> - QueuePid ! {fail, Error}, - ok - end; - false -> - ?INFO("Skipping ~s\n", [Src]), - compile_worker(QueuePid) - end; - - empty -> - ok - end. diff --git a/src/rebar_erlydtl_compiler.erl b/src/rebar_erlydtl_compiler.erl index 862c5ed..ecdc8c0 100644 --- a/src/rebar_erlydtl_compiler.erl +++ b/src/rebar_erlydtl_compiler.erl @@ -82,14 +82,10 @@ compile(Config, _AppFile) -> DtlOpts = erlydtl_opts(Config), - rebar_erlc_compiler:do_compile( - Config, - filename:join([option(doc_root, DtlOpts), - ["*",option(source_ext, DtlOpts)]]), - option(out_dir, DtlOpts), - option(source_ext, DtlOpts), - option(module_ext, DtlOpts)++".beam", - fun referenced_dtls/2, fun compile_dtl/2, []). + rebar_base_compiler:run(Config, option(doc_root, DtlOpts), option(source_ext, DtlOpts), + option(out_dir, DtlOpts), option(module_ext, DtlOpts) ++ ".beam", + [], fun compile_dtl/3, + [{needs_compile_checks, [fun referenced_dtls/3]}]). %% =================================================================== @@ -107,13 +103,13 @@ default(out_dir) -> "ebin"; default(source_ext) -> ".dtl"; default(module_ext) -> "_dtl". -referenced_dtls(Source, Config) -> - Set = referenced_dtls([Source], Config, +referenced_dtls(Source, _Target, Config) -> + Set = referenced_dtls1([Source], Config, sets:add_element(Source, sets:new())), Final = sets:to_list(sets:del_element(Source, Set)), Final. -referenced_dtls(Step, Config, Seen) -> +referenced_dtls1(Step, Config, Seen) -> DtlOpts = erlydtl_opts(Config), ExtMatch = re:replace(option(source_ext, DtlOpts), "\.", "\\\\.", [{return, list}]), @@ -132,7 +128,7 @@ referenced_dtls(Step, Config, Seen) -> sets:union(New, Seen)) end. -compile_dtl(Source, Config) -> +compile_dtl(Source, _Target, Config) -> case code:which(erlydtl) of non_existing -> ?CONSOLE( diff --git a/src/rebar_eunit.erl b/src/rebar_eunit.erl index e32aa72..74c58af 100644 --- a/src/rebar_eunit.erl +++ b/src/rebar_eunit.erl @@ -52,15 +52,19 @@ eunit(Config, _File) -> ok = filelib:ensure_dir(?EUNIT_DIR ++ "/foo"), %% Compile all erlang from src/ into ?EUNIT_DIR - rebar_erlc_compiler:do_compile(Config, "src/*.erl", ?EUNIT_DIR, ".erl", ".beam", - fun rebar_erlc_compiler:list_hrls/2, fun compile_erl/2, - rebar_config:get_list(Config, erl_first_files, [])), + rebar_base_compiler:run(Config, "src", ".erl", ?EUNIT_DIR, ".beam", + rebar_config:get_list(Config, erl_first_files, []), + fun compile_erl/3, + [recurse_source_dir, + {needs_compile_checks, [fun rebar_erlc_compiler:hrls_check/3]}]), %% Build a list of all the .beams in ?EUNIT_DIR -- use this for cover %% and eunit testing. Normally you can just tell cover and/or eunit to %% scan the directory for you, but eunit does a code:purge in conjunction %% with that scan and causes any cover compilation info to be lost. So, %% we do it by hand. :( + %% + %% TODO: Not currently compatible with package modules Modules = [list_to_atom(filename:basename(N, ".beam")) || N <- filelib:wildcard("*.beam", "ebin")], @@ -123,7 +127,7 @@ clean(_Config, _File) -> %% Internal functions %% =================================================================== -compile_erl(Source, Config) -> +compile_erl(Source, Target, Config) -> case is_quickcheck_avail() of true -> EqcOpts = [{d, 'EQC'}]; @@ -133,7 +137,8 @@ compile_erl(Source, Config) -> ErlOpts = rebar_config:get_list(Config, erl_opts, []), EunitOpts = rebar_config:get_list(Config, eunit_compile_opts, []), - Opts = [{i, "include"}, {outdir, ?EUNIT_DIR}, {d, 'TEST'}, debug_info, report] ++ + Opts = [{i, "include"}, {outdir, filename:dirname(Target)}, + {d, 'TEST'}, debug_info, report] ++ ErlOpts ++ EunitOpts ++ EqcOpts, case compile:file(Source, Opts) of {ok, _} -> diff --git a/src/rebar_lfe_compiler.erl b/src/rebar_lfe_compiler.erl index 5447028..78a2c3d 100644 --- a/src/rebar_lfe_compiler.erl +++ b/src/rebar_lfe_compiler.erl @@ -38,15 +38,16 @@ compile(Config, _AppFile) -> FirstFiles = rebar_config:get_list(Config, lfe_first_files, []), - rebar_erlc_compiler:do_compile(Config, "src/*.lfe", "ebin", ".lfe", ".beam", - undefined, fun compile_lfe/2, FirstFiles). + rebar_base_compiler:run(Config, "src", ".lfe", "ebin", ".beam", + FirstFiles, + fun compile_lfe/3, []). %% =================================================================== %% Internal functions %% =================================================================== -compile_lfe(Source, Config) -> +compile_lfe(Source, _Target, Config) -> case code:which(lfe_comp) of non_existing -> ?CONSOLE("~n===============================================~n" ++ diff --git a/src/rebar_otp_app.erl b/src/rebar_otp_app.erl index 6f4935f..6b330cf 100644 --- a/src/rebar_otp_app.erl +++ b/src/rebar_otp_app.erl @@ -119,7 +119,8 @@ validate_name(AppName, File) -> true -> ok; false -> - ?ERROR("Invalid ~s: name of application (~p) must match filename.\n", [File, AppName]), + ?ERROR("Invalid ~s: name of application (~p) must match filename.\n", + [File, AppName]), ?FAIL end. @@ -130,7 +131,7 @@ validate_modules(AppName, undefined) -> validate_modules(AppName, Mods) -> %% Construct two sets -- one for the actual .beam files in ebin/ and one for the modules %% listed in the .app file - EbinSet = ordsets:from_list([beam_to_mod(N) || N <- filelib:wildcard("ebin/*.beam")]), + EbinSet = ordsets:from_list([beam_to_mod(N) || N <- beams()]), ModSet = ordsets:from_list(Mods), %% Identify .beam files listed in the .app, but not present in ebin/ @@ -156,5 +157,10 @@ validate_modules(AppName, Mods) -> end. beam_to_mod(Filename) -> - list_to_atom(filename:basename(Filename, ".beam")). + ["ebin" | Rest] = filename:split(Filename), + list_to_atom(filename:basename(string:join(Rest, "."), ".beam")). + +beams() -> + filelib:fold_files("ebin", ".*\.beam\$", true, + fun(F, Acc) -> [F | Acc] end, []).