github/rebar
1
0
Fork 0
mirror of https://github.com/correl/rebar.git synced 2024-11-23 19:19:54 +00:00
Code Issues Projects Releases Packages Wiki Activity

Refactor of compilation logic to enable support for package modules and provide better modularity/reusability for other systems

Browse source
This commit is contained in:
Dave Smith 2010-01-02 14:17:59 -07:00
parent 18e1b37e6f
commit 9c15d63191
7 changed files with 274 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
ebin/rebar.app
View file

@ -3,7 +3,8 @@
{vsn, "2"},
{modules, [ rebar,
rebar_app_utils,
rebar_config,
rebar_base_compiler,
rebar_config,
rebar_core,
rebar_ct,
rebar_deps,

197
src/rebar_base_compiler.erl Normal file
View file

@ -0,0 +1,197 @@
%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 et
%% -------------------------------------------------------------------
%%
%% rebar: Erlang Build Tools
%%
%% Copyright (c) 2009 Dave Smith (dizzyd@dizzyd.com)
%%
%% Permission is hereby granted, free of charge, to any person obtaining a copy
%% of this software and associated documentation files (the "Software"), to deal
%% in the Software without restriction, including without limitation the rights
%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
%% copies of the Software, and to permit persons to whom the Software is
%% furnished to do so, subject to the following conditions:
%%
%% The above copyright notice and this permission notice shall be included in
%% all copies or substantial portions of the Software.
%%
%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
%% THE SOFTWARE.
%% -------------------------------------------------------------------
-module(rebar_base_compiler).
-include("rebar.hrl").
-export([run/8]).
%% ===================================================================
%% Public API
%% ===================================================================
run(Config, SourceDir, SourceExt, TargetDir, TargetExt,
FirstFiles, CompileFn, Opts) ->
SourceExtRe = ".*\\" ++ SourceExt ++ [$$],
%% Options:
%% recurse_source_dir
%% needs_compile_checks - [ fun/2 ]
Recursive = proplists:get_bool(recurse_source_dir, Opts),
%% Find all the source files we can
FoundFiles = filelib:fold_files(SourceDir, SourceExtRe, Recursive,
fun(F, Acc) -> [F | Acc] end, []),
%% Construct two lists of targets. "FirstTargets" is the list of files which
%% must be compiled first and in strict order; "RestTargets" is all remaining files
%% that may be compiled in any order.
FirstTargets = [{Fs, target_file(Fs, SourceDir, SourceExt, TargetDir, TargetExt)} ||
Fs <- FirstFiles],
RestTargets = [{Fs, target_file(Fs, SourceDir, SourceExt, TargetDir, TargetExt)} ||
Fs <- drop_each(FirstFiles, FoundFiles)],
%% Setup list of functions which determine if a file needs compilation or not. By
%% default we just check the last modified date
NeedsCompileFns = [ fun check_source_lastmod/3 ] ++
rebar_config:get(Config, needs_compile_checks, []),
%% Compile the first targets in sequence
compile_each(FirstTargets, Config, NeedsCompileFns, CompileFn),
%% Spin up workers
case RestTargets of
[] ->
ok;
_ ->
Self = self(),
F = fun() -> compile_worker(Self, Config, NeedsCompileFns, CompileFn) end,
Pids = [spawn_monitor(F) || _I <- lists:seq(1,3)],
compile_queue(Pids, RestTargets)
end.
%% ===================================================================
%% Internal functions
%% ===================================================================
target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) ->
%% Remove all leading components of the source dir from the file -- we want
%% to maintain the deeper structure (if any) of the source file path
BaseFile = remove_common_path(SourceFile, SourceDir),
filename:join([TargetDir, filename:dirname(BaseFile),
filename:basename(BaseFile, SourceExt) ++ TargetExt]).
remove_common_path(Fname, Path) ->
remove_common_path1(filename:split(Fname), filename:split(Path)).
remove_common_path1([Part | RestFilename], [Part | RestPath]) ->
remove_common_path1(RestFilename, RestPath);
remove_common_path1(FilenameParts, _) ->
filename:join(FilenameParts).
drop_each([], List) ->
List;
drop_each([Member | Rest], List) ->
drop_each(Rest, lists:delete(Member, List)).
needs_compile(_SourceFile, _TargetFile, _Config, []) ->
false;
needs_compile(SourceFile, TargetFile, Config, [Fn | Rest]) ->
case Fn(SourceFile, TargetFile, Config) of
true ->
true;
false ->
needs_compile(SourceFile, TargetFile, Config, Rest)
end.
check_source_lastmod(SourceFile, TargetFile, _Config) ->
filelib:last_modified(TargetFile) < filelib:last_modified(SourceFile).
compile(Source, Target, Config, NeedsCompileFns, CompileFn) ->
case needs_compile(Source, Target, Config, NeedsCompileFns) of
true ->
ok = filelib:ensure_dir(Target),
CompileFn(Source, Target, Config);
false ->
skipped
end.
compile_each([], _Config, _NeedsCompileFns, _CompileFn) ->
ok;
compile_each([{Source, Target} | Rest], Config, NeedsCompileFns, CompileFn) ->
case compile(Source, Target, Config, NeedsCompileFns, CompileFn) of
ok ->
?CONSOLE("Compiled ~s\n", [Source]);
skipped ->
?INFO("Skipped ~s\n", [Source])
end,
compile_each(Rest, Config, NeedsCompileFns, CompileFn).
compile_queue([], []) ->
ok;
compile_queue(Pids, Targets) ->
receive
{next, Worker} ->
case Targets of
[] ->
Worker ! empty,
compile_queue(Pids, Targets);
[{Source, Target} | Rest] ->
Worker ! {compile, Source, Target},
compile_queue(Pids, Rest)
end;
{fail, Error} ->
?DEBUG("Worker compilation failed: ~p\n", [Error]),
?FAIL;
{compiled, Source} ->
?CONSOLE("Compiled ~s\n", [Source]),
compile_queue(Pids, Targets);
{skipped, Source} ->
?INFO("Skipped ~s\n", [Source]),
compile_queue(Pids, Targets);
{'DOWN', Mref, _, Pid, normal} ->
?DEBUG("Worker exited cleanly\n", []),
Pids2 = lists:delete({Pid, Mref}, Pids),
compile_queue(Pids2, Targets);
{'DOWN', _Mref, _, _Pid, Info} ->
?DEBUG("Worker failed: ~p\n", [Info]),
?FAIL
end.
compile_worker(QueuePid, Config, NeedsCompileFns, CompileFn) ->
QueuePid ! {next, self()},
receive
{compile, Source, Target} ->
case catch(compile(Source, Target, Config, NeedsCompileFns, CompileFn)) of
ok ->
QueuePid ! {compiled, Source},
compile_worker(QueuePid, Config, NeedsCompileFns, CompileFn);
skipped ->
QueuePid ! {skipped, Source},
compile_worker(QueuePid, Config, NeedsCompileFns, CompileFn);
Error ->
QueuePid ! {fail, Error},
ok
end;
empty ->
ok
end.

186
src/rebar_erlc_compiler.erl
View file

@ -30,7 +30,7 @@
clean/2]).
%% make available for rebar_eunit until there is a better option
-export([do_compile/8, compile_opts/2, list_hrls/2]).
-export([hrls_check/3]).
-include("rebar.hrl").
@ -39,18 +39,30 @@
%% ===================================================================
compile(Config, _AppFile) ->
do_compile(Config, "src/*.erl", "ebin", ".erl", ".beam",
fun list_hrls/2, fun compile_erl/2,
rebar_config:get_list(Config, erl_first_files, [])),
do_compile(Config, "mibs/*.mib", "priv/mibs", ".mib", ".bin",
undefined, fun compile_mib/2,
rebar_config:get_list(Config, mib_first_files, [])).
rebar_base_compiler:run(Config, "src", ".erl", "ebin", ".beam",
rebar_config:get_list(Config, erl_first_files, []),
fun compile_erl/3,
[recurse_source_dir,
{needs_compile_checks, [fun hrls_check/3]}]),
rebar_base_compiler:run(Config, "mibs", ".mib", "priv/mibs", ".bin",
rebar_config:get_list(Config, mib_first_files, []),
fun compile_mib/3,
[]).
clean(_Config, _AppFile) ->
%% TODO: This would be more portable if it used Erlang to traverse
%% the dir structure and delete each file; however it would also
%% much slower.
[] = os:cmd("rm -f ebin/*.beam priv/mibs/*.bin"),
ok = rebar_file_utils:rm_rf("ebin/*.beam priv/mibs/*.bin"),
%% Erlang compilation is recursive, so it's possible that we have a nested
%% directory structure in ebin with .beam files within. As such, we want
%% to scan whatever is left in the ebin/ directory for sub-dirs which
%% satisfy our criteria. TODO: Is there a better way to do this?
BeamFiles = filelib:fold_files("ebin", "^.*\\.beam\$", true,
fun(F, BeamFiles) -> BeamFiles ++ [F] end, []),
rebar_file_utils:delete_each(BeamFiles),
ok.
@ -59,71 +71,11 @@ clean(_Config, _AppFile) ->
%% Internal functions
%% ===================================================================
do_compile(Config, SrcWildcard, OutDir, InExt, OutExt,
IncludeFn, CompileFn, FirstFiles) ->
case filelib:wildcard(SrcWildcard) of
[] ->
ok;
FoundFiles when is_list(FoundFiles) ->
%% Ensure that the FirstFiles are compiled first; drop them from the
%% FoundFiles and compile them in sequence
FirstTargets = [{Fs, target_file(Fs, OutDir, InExt, OutExt)} || Fs <- FirstFiles],
RestTargets = [{Fs, target_file(Fs, OutDir, InExt, OutExt)} ||
Fs <- drop_each(FirstFiles, FoundFiles)],
hrls_check(Source, Target, Config) ->
TargetLastMod = filelib:last_modified(Target),
lists:any(fun(I) -> TargetLastMod < filelib:last_modified(I) end,
list_hrls(Source, Config)).
%% Make sure target directory exists
ok = filelib:ensure_dir(target_file(hd(FoundFiles), OutDir, InExt, OutExt)),
%% Compile first targets in sequence
compile_each(FirstTargets, Config, IncludeFn, CompileFn),
%% Spin up workers
Self = self(),
Pids = [spawn_monitor(fun() -> compile_worker(Self) end) || _I <- lists:seq(1,3)],
%% Process rest of targets
compile_queue(Pids, RestTargets, Config, IncludeFn, CompileFn)
end.
drop_each([], List) ->
List;
drop_each([Member | Rest], List) ->
drop_each(Rest, lists:delete(Member, List)).
compile_each([], _Config, _IncludeFn, _CompileFn) ->
ok;
compile_each([{Src, Target} | Rest], Config, IncludeFn, CompileFn) ->
case needs_compile(Src, Target, IncludeFn, Config) of
true ->
?CONSOLE("Compiling ~s\n", [Src]),
CompileFn(Src, Config);
false ->
?INFO("Skipping ~s\n", [Src]),
ok
end,
compile_each(Rest, Config, IncludeFn, CompileFn).
needs_compile(Src, Target, IncludeFn, Config) ->
TargetLM = filelib:last_modified(Target),
case TargetLM < filelib:last_modified(Src) of
true ->
true;
false ->
if is_function(IncludeFn) ->
lists:any(fun(I) ->
TargetLM < filelib:last_modified(I)
end,
IncludeFn(Src, Config));
true ->
false
end
end.
target_file(F, TargetDir, InExt, OutExt) ->
filename:join([TargetDir, filename:basename(F, InExt) ++ OutExt]).
compile_opts(Config, Key) ->
rebar_config:get_list(Config, Key, []).
list_hrls(Src, Config) ->
case epp:open(Src, include_path(Src, Config)) of
@ -134,26 +86,31 @@ list_hrls(Src, Config) ->
false
end.
include_path(Src, Config) ->
[filename:dirname(Src)|compile_opts(Config, i)].
extract_includes(Epp, Src) ->
case epp:parse_erl_form(Epp) of
{ok, {attribute, 1, file, {Src, 1}}} ->
extract_includes(Epp, Src);
{ok, {attribute, 1, file, {IncFile, 1}}} ->
{ok, {attribute, 1, file, {Src, 1}}} ->
extract_includes(Epp, Src);
{ok, {attribute, 1, file, {IncFile, 1}}} ->
[IncFile|extract_includes(Epp, Src)];
{ok, _} ->
extract_includes(Epp, Src);
{eof, _} ->
epp:close(Epp),
[];
{error, _Error} ->
extract_includes(Epp, Src)
{ok, _} ->
extract_includes(Epp, Src);
{eof, _} ->
epp:close(Epp),
[];
{error, _Error} ->
extract_includes(Epp, Src)
end.
compile_erl(Source, Config) ->
Opts = [{i, "include"}, {outdir, "ebin"}, report, return] ++ compile_opts(Config, erl_opts),
include_path(Source, Config) ->
[filename:dirname(Source) | compile_opts(Config, i)].
compile_opts(Config, Key) ->
rebar_config:get_list(Config, Key, []).
compile_erl(Source, Target, Config) ->
Opts = [{i, "include"}, {outdir, filename:dirname(Target)}, report, return] ++
compile_opts(Config, erl_opts),
case compile:file(Source, Opts) of
{ok, _, []} ->
ok;
@ -169,7 +126,7 @@ compile_erl(Source, Config) ->
?FAIL
end.
compile_mib(Source, Config) ->
compile_mib(Source, _Target, Config) ->
Opts = [{outdir, "priv/mibs"}, {i, ["priv/mibs"]}] ++ compile_opts(Config, mib_opts),
case snmpc:compile(Source, Opts) of
{ok, _} ->
@ -177,58 +134,3 @@ compile_mib(Source, Config) ->
{error, compilation_failed} ->
?FAIL
end.
compile_queue([], [], _Config, _IncludeFn, _CompileFn) ->
ok;
compile_queue(Pids, Targets, Config, IncludeFn, CompileFn) ->
receive
{next, Worker} ->
case Targets of
[] ->
Worker ! empty,
compile_queue(Pids, Targets, Config, IncludeFn, CompileFn);
[{Src, Target} | Rest] ->
Worker ! {compile, Src, Target, Config, IncludeFn, CompileFn},
compile_queue(Pids, Rest, Config, IncludeFn, CompileFn)
end;
{fail, Error} ->
?DEBUG("Worker compilation failed: ~p\n", [Error]),
?FAIL;
{compiled, Source} ->
?CONSOLE("Compiled ~s\n", [Source]),
compile_queue(Pids, Targets, Config, IncludeFn, CompileFn);
{'DOWN', Mref, _, Pid, normal} ->
?DEBUG("Worker exited cleanly\n", []),
Pids2 = lists:delete({Pid, Mref}, Pids),
compile_queue(Pids2, Targets, Config, IncludeFn, CompileFn);
{'DOWN', _Mref, _, _Pid, Info} ->
?DEBUG("Worker failed: ~p\n", [Info]),
?FAIL
end.
compile_worker(QueuePid) ->
QueuePid ! {next, self()},
receive
{compile, Src, Target, Config, IncludeFn, CompileFn} ->
case needs_compile(Src, Target, IncludeFn, Config) of
true ->
case catch(CompileFn(Src, Config)) of
ok ->
QueuePid ! {compiled, Src},
compile_worker(QueuePid);
Error ->
QueuePid ! {fail, Error},
ok
end;
false ->
?INFO("Skipping ~s\n", [Src]),
compile_worker(QueuePid)
end;
empty ->
ok
end.

20
src/rebar_erlydtl_compiler.erl
View file

@ -82,14 +82,10 @@
compile(Config, _AppFile) ->
DtlOpts = erlydtl_opts(Config),
rebar_erlc_compiler:do_compile(
Config,
filename:join([option(doc_root, DtlOpts),
["*",option(source_ext, DtlOpts)]]),
option(out_dir, DtlOpts),
option(source_ext, DtlOpts),
option(module_ext, DtlOpts)++".beam",
fun referenced_dtls/2, fun compile_dtl/2, []).
rebar_base_compiler:run(Config, option(doc_root, DtlOpts), option(source_ext, DtlOpts),
option(out_dir, DtlOpts), option(module_ext, DtlOpts) ++ ".beam",
[], fun compile_dtl/3,
[{needs_compile_checks, [fun referenced_dtls/3]}]).
%% ===================================================================
@ -107,13 +103,13 @@ default(out_dir) -> "ebin";
default(source_ext) -> ".dtl";
default(module_ext) -> "_dtl".
referenced_dtls(Source, Config) ->
Set = referenced_dtls([Source], Config,
referenced_dtls(Source, _Target, Config) ->
Set = referenced_dtls1([Source], Config,
sets:add_element(Source, sets:new())),
Final = sets:to_list(sets:del_element(Source, Set)),
Final.
referenced_dtls(Step, Config, Seen) ->
referenced_dtls1(Step, Config, Seen) ->
DtlOpts = erlydtl_opts(Config),
ExtMatch = re:replace(option(source_ext, DtlOpts), "\.", "\\\\.",
[{return, list}]),
@ -132,7 +128,7 @@ referenced_dtls(Step, Config, Seen) ->
sets:union(New, Seen))
end.
compile_dtl(Source, Config) ->
compile_dtl(Source, _Target, Config) ->
case code:which(erlydtl) of
non_existing ->
?CONSOLE(

15
src/rebar_eunit.erl
View file

@ -52,15 +52,19 @@ eunit(Config, _File) ->
ok = filelib:ensure_dir(?EUNIT_DIR ++ "/foo"),
%% Compile all erlang from src/ into ?EUNIT_DIR
rebar_erlc_compiler:do_compile(Config, "src/*.erl", ?EUNIT_DIR, ".erl", ".beam",
fun rebar_erlc_compiler:list_hrls/2, fun compile_erl/2,
rebar_config:get_list(Config, erl_first_files, [])),
rebar_base_compiler:run(Config, "src", ".erl", ?EUNIT_DIR, ".beam",
rebar_config:get_list(Config, erl_first_files, []),
fun compile_erl/3,
[recurse_source_dir,
{needs_compile_checks, [fun rebar_erlc_compiler:hrls_check/3]}]),
%% Build a list of all the .beams in ?EUNIT_DIR -- use this for cover
%% and eunit testing. Normally you can just tell cover and/or eunit to
%% scan the directory for you, but eunit does a code:purge in conjunction
%% with that scan and causes any cover compilation info to be lost. So,
%% we do it by hand. :(
%%
%% TODO: Not currently compatible with package modules
Modules = [list_to_atom(filename:basename(N, ".beam")) ||
N <- filelib:wildcard("*.beam", "ebin")],
@ -123,7 +127,7 @@ clean(_Config, _File) ->
%% Internal functions
%% ===================================================================
compile_erl(Source, Config) ->
compile_erl(Source, Target, Config) ->
case is_quickcheck_avail() of
true ->
EqcOpts = [{d, 'EQC'}];
@ -133,7 +137,8 @@ compile_erl(Source, Config) ->
ErlOpts = rebar_config:get_list(Config, erl_opts, []),
EunitOpts = rebar_config:get_list(Config, eunit_compile_opts, []),
Opts = [{i, "include"}, {outdir, ?EUNIT_DIR}, {d, 'TEST'}, debug_info, report] ++
Opts = [{i, "include"}, {outdir, filename:dirname(Target)},
{d, 'TEST'}, debug_info, report] ++
ErlOpts ++ EunitOpts ++ EqcOpts,
case compile:file(Source, Opts) of
{ok, _} ->

7
src/rebar_lfe_compiler.erl
View file

@ -38,15 +38,16 @@
compile(Config, _AppFile) ->
FirstFiles = rebar_config:get_list(Config, lfe_first_files, []),
rebar_erlc_compiler:do_compile(Config, "src/*.lfe", "ebin", ".lfe", ".beam",
undefined, fun compile_lfe/2, FirstFiles).
rebar_base_compiler:run(Config, "src", ".lfe", "ebin", ".beam",
FirstFiles,
fun compile_lfe/3, []).
%% ===================================================================
%% Internal functions
%% ===================================================================
compile_lfe(Source, Config) ->
compile_lfe(Source, _Target, Config) ->
case code:which(lfe_comp) of
non_existing ->
?CONSOLE("~n===============================================~n" ++

12
src/rebar_otp_app.erl
View file

@ -119,7 +119,8 @@ validate_name(AppName, File) ->
true ->
ok;
false ->
?ERROR("Invalid ~s: name of application (~p) must match filename.\n", [File, AppName]),
?ERROR("Invalid ~s: name of application (~p) must match filename.\n",
[File, AppName]),
?FAIL
end.
@ -130,7 +131,7 @@ validate_modules(AppName, undefined) ->
validate_modules(AppName, Mods) ->
%% Construct two sets -- one for the actual .beam files in ebin/ and one for the modules
%% listed in the .app file
EbinSet = ordsets:from_list([beam_to_mod(N) || N <- filelib:wildcard("ebin/*.beam")]),
EbinSet = ordsets:from_list([beam_to_mod(N) || N <- beams()]),
ModSet = ordsets:from_list(Mods),
%% Identify .beam files listed in the .app, but not present in ebin/
@ -156,5 +157,10 @@ validate_modules(AppName, Mods) ->
end.
beam_to_mod(Filename) ->
list_to_atom(filename:basename(Filename, ".beam")).
["ebin" | Rest] = filename:split(Filename),
list_to_atom(filename:basename(string:join(Rest, "."), ".beam")).
beams() ->
filelib:fold_files("ebin", ".*\.beam\$", true,
fun(F, Acc) -> [F | Acc] end, []).