%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ts=4 sw=4 et %% ------------------------------------------------------------------- %% %% rebar: Erlang Build Tools %% %% Copyright (c) 2009, 2010 Dave Smith (dizzyd@dizzyd.com) %% %% Permission is hereby granted, free of charge, to any person obtaining a copy %% of this software and associated documentation files (the "Software"), to deal %% in the Software without restriction, including without limitation the rights %% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell %% copies of the Software, and to permit persons to whom the Software is %% furnished to do so, subject to the following conditions: %% %% The above copyright notice and this permission notice shall be included in %% all copies or substantial portions of the Software. %% %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR %% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, %% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE %% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER %% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN %% THE SOFTWARE. %% ------------------------------------------------------------------- -module(rebar_erlc_compiler). -export([compile/2, clean/2]). %% for internal use only -export([test_compile/3, info/2]). -include("rebar.hrl"). -include_lib("stdlib/include/erl_compile.hrl"). -define(REBAR_BUILD_INFO, ".rebar.build.info"). %% =================================================================== %% Public API %% =================================================================== %% Supported configuration variables: %% %% * erl_opts - Erlang list of options passed to compile:file/2 %% It is also possible to specify platform specific %% options by specifying a pair or a triplet where the %% first string is a regex that is checked against the %% string %% %% OtpRelease ++ "-" ++ SysArch ++ "-" ++ Words. %% %% where %% %% OtpRelease = erlang:system_info(otp_release). %% SysArch = erlang:system_info(system_architecture). %% Words = integer_to_list(8 * %% erlang:system_info({wordsize, external})). %% %% E.g. to define HAVE_SENDFILE only on systems with %% sendfile(), to define BACKLOG on Linux/FreeBSD as 128, %% and to define 'old_inets' for R13 OTP release do: %% %% {erl_opts, [{platform_define, %% "(linux|solaris|freebsd|darwin)", %% 'HAVE_SENDFILE'}, %% {platform_define, "(linux|freebsd)", %% 'BACKLOG', 128}, %% {platform_define, "R13", %% 'old_inets'}]}. %% -spec compile(rebar_config:config(), file:filename()) -> 'ok'. compile(Config, _AppFile) -> rebar_base_compiler:run(Config, check_files(rebar_config:get_local( Config, xrl_first_files, [])), "src", ".xrl", "src", ".erl", fun compile_xrl/3), rebar_base_compiler:run(Config, check_files(rebar_config:get_local( Config, yrl_first_files, [])), "src", ".yrl", "src", ".erl", fun compile_yrl/3), rebar_base_compiler:run(Config, check_files(rebar_config:get_local( Config, mib_first_files, [])), "mibs", ".mib", "priv/mibs", ".bin", fun compile_mib/3), doterl_compile(Config, "ebin"). -spec clean(rebar_config:config(), file:filename()) -> 'ok'. clean(_Config, _AppFile) -> MibFiles = rebar_utils:find_files("mibs", "^.*\\.mib\$"), MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles], rebar_file_utils:delete_each( [filename:join(["include",MIB++".hrl"]) || MIB <- MIBs]), lists:foreach(fun(F) -> ok = rebar_file_utils:rm_rf(F) end, ["ebin/*.beam", "priv/mibs/*.bin"]), YrlFiles = rebar_utils:find_files("src", "^.*\\.[x|y]rl\$"), rebar_file_utils:delete_each( [ binary_to_list(iolist_to_binary(re:replace(F, "\\.[x|y]rl$", ".erl"))) || F <- YrlFiles ]), %% Delete the build graph, if any rebar_file_utils:rm_rf(filename:join("ebin", ?REBAR_BUILD_INFO)), %% Erlang compilation is recursive, so it's possible that we have a nested %% directory structure in ebin with .beam files within. As such, we want %% to scan whatever is left in the ebin/ directory for sub-dirs which %% satisfy our criteria. BeamFiles = rebar_utils:find_files("ebin", "^.*\\.beam\$"), rebar_file_utils:delete_each(BeamFiles), lists:foreach(fun(Dir) -> delete_dir(Dir, dirs(Dir)) end, dirs("ebin")), ok. %% =================================================================== %% .erl Compilation API (externally used by only eunit and qc) %% =================================================================== test_compile(Config, Cmd, OutDir) -> %% Obtain all the test modules for inclusion in the compile stage. TestErls = rebar_utils:find_files("test", ".*\\.erl\$"), %% Copy source files to eunit dir for cover in case they are not directly %% in src but in a subdirectory of src. Cover only looks in cwd and ../src %% for source files. Also copy files from src_dirs. ErlOpts = rebar_utils:erl_opts(Config), SrcDirs = rebar_utils:src_dirs(proplists:append_values(src_dirs, ErlOpts)), SrcErls = lists:foldl( fun(Dir, Acc) -> Files = rebar_utils:find_files(Dir, ".*\\.erl\$"), lists:append(Acc, Files) end, [], SrcDirs), %% If it is not the first time rebar eunit is executed, there will be source %% files already present in OutDir. Since some SCMs (like Perforce) set %% the source files as being read only (unless they are checked out), we %% need to be sure that the files already present in OutDir are writable %% before doing the copy. This is done here by removing any file that was %% already present before calling rebar_file_utils:cp_r. %% Get the full path to a file that was previously copied in OutDir ToCleanUp = fun(F, Acc) -> F2 = filename:basename(F), F3 = filename:join([OutDir, F2]), case filelib:is_regular(F3) of true -> [F3|Acc]; false -> Acc end end, ok = rebar_file_utils:delete_each(lists:foldl(ToCleanUp, [], TestErls)), ok = rebar_file_utils:delete_each(lists:foldl(ToCleanUp, [], SrcErls)), ok = rebar_file_utils:cp_r(SrcErls ++ TestErls, OutDir), %% Compile erlang code to OutDir, using a tweaked config %% with appropriate defines for eunit, and include all the test modules %% as well. ok = doterl_compile(test_compile_config(Config, ErlOpts, Cmd), OutDir, TestErls), {ok, SrcErls}. %% =================================================================== %% Internal functions %% =================================================================== info(help, compile) -> info_help("Build *.erl, *.yrl, *.xrl, and *.mib sources"); info(help, clean) -> info_help("Delete *.erl, *.yrl, *.xrl, and *.mib build results"). info_help(Description) -> ?CONSOLE( "~s.~n" "~n" "Valid rebar.config options:~n" " ~p~n" " ~p~n" " ~p~n" " ~p~n" " ~p~n" " ~p~n" " ~p~n" " ~p~n", [ Description, {erl_opts, [no_debug_info, {i, "myinclude"}, {src_dirs, ["src", "src2", "src3"]}, {platform_define, "(linux|solaris|freebsd|darwin)", 'HAVE_SENDFILE'}, {platform_define, "(linux|freebsd)", 'BACKLOG', 128}, {platform_define, "R13", 'old_inets'}]}, {erl_first_files, ["mymib1", "mymib2"]}, {mib_opts, []}, {mib_first_files, []}, {xrl_opts, []}, {xrl_first_files, []}, {yrl_opts, []}, {yrl_first_files, []} ]). test_compile_config(Config, ErlOpts, Cmd) -> {Config1, TriqOpts} = triq_opts(Config), {Config2, PropErOpts} = proper_opts(Config1), {Config3, EqcOpts} = eqc_opts(Config2), OptsAtom = list_to_atom(Cmd ++ "_compile_opts"), EunitOpts = rebar_config:get_list(Config3, OptsAtom, []), Opts0 = [{d, 'TEST'}] ++ ErlOpts ++ EunitOpts ++ TriqOpts ++ PropErOpts ++ EqcOpts, Opts = [O || O <- Opts0, O =/= no_debug_info], Config4 = rebar_config:set(Config3, erl_opts, Opts), FirstFilesAtom = list_to_atom(Cmd ++ "_first_files"), FirstErls = rebar_config:get_list(Config4, FirstFilesAtom, []), rebar_config:set(Config4, erl_first_files, FirstErls). triq_opts(Config) -> {NewConfig, IsAvail} = is_lib_avail(Config, is_triq_avail, triq, "triq.hrl", "Triq"), Opts = define_if('TRIQ', IsAvail), {NewConfig, Opts}. proper_opts(Config) -> {NewConfig, IsAvail} = is_lib_avail(Config, is_proper_avail, proper, "proper.hrl", "PropEr"), Opts = define_if('PROPER', IsAvail), {NewConfig, Opts}. eqc_opts(Config) -> {NewConfig, IsAvail} = is_lib_avail(Config, is_eqc_avail, eqc, "eqc.hrl", "QuickCheck"), Opts = define_if('EQC', IsAvail), {NewConfig, Opts}. define_if(Def, true) -> [{d, Def}]; define_if(_Def, false) -> []. is_lib_avail(Config, DictKey, Mod, Hrl, Name) -> case rebar_config:get_xconf(Config, DictKey, undefined) of undefined -> IsAvail = case code:lib_dir(Mod, include) of {error, bad_name} -> false; Dir -> filelib:is_regular(filename:join(Dir, Hrl)) end, NewConfig = rebar_config:set_xconf(Config, DictKey, IsAvail), ?DEBUG("~s availability: ~p\n", [Name, IsAvail]), {NewConfig, IsAvail}; IsAvail -> {Config, IsAvail} end. -spec doterl_compile(rebar_config:config(), file:filename()) -> 'ok'. doterl_compile(Config, OutDir) -> doterl_compile(Config, OutDir, []). doterl_compile(Config, OutDir, MoreSources) -> FirstErls = rebar_config:get_list(Config, erl_first_files, []), ErlOpts = rebar_utils:erl_opts(Config), ?DEBUG("erl_opts ~p~n", [ErlOpts]), %% Support the src_dirs option allowing multiple directories to %% contain erlang source. This might be used, for example, should %% eunit tests be separated from the core application source. SrcDirs = rebar_utils:src_dirs(proplists:append_values(src_dirs, ErlOpts)), RestErls = [Source || Source <- gather_src(SrcDirs, []) ++ MoreSources, not lists:member(Source, FirstErls)], %% Make sure that ebin/ exists and is on the path ok = filelib:ensure_dir(filename:join("ebin", "dummy.beam")), CurrPath = code:get_path(), true = code:add_path(filename:absname("ebin")), OutDir1 = proplists:get_value(outdir, ErlOpts, OutDir), G = init_graph(Config, RestErls), %% Split RestErls so that parse_transforms and behaviours are instead added %% to erl_first_files, parse transforms first. {OtherFirstErls, OtherErls} = lists:partition( fun(F) -> case [Erl || Erl <- get_children(G, F), filename:extension(Erl) == ".erl"] of [] -> %% There are no files dependent on this file. false; _ -> %% There are some files dependent on the file. %% Thus the file has higher priority %% and should be compiled in the first place. true end end, RestErls), NewFirstErls = FirstErls ++ OtherFirstErls, rebar_base_compiler:run( Config, NewFirstErls, OtherErls, fun(S, C) -> internal_erl_compile(C, S, OutDir1, ErlOpts, G) end), true = code:set_path(CurrPath), ok. -spec include_path(file:filename(), rebar_config:config()) -> [file:filename(), ...]. include_path(Source, Config) -> ErlOpts = rebar_config:get(Config, erl_opts, []), lists:usort(["include", filename:dirname(Source)] ++ proplists:get_all_values(i, ErlOpts)). -spec needs_compile(file:filename(), file:filename(), [string()]) -> boolean(). needs_compile(Source, Target, Hrls) -> TargetLastMod = filelib:last_modified(Target), lists:any(fun(I) -> TargetLastMod < filelib:last_modified(I) end, [Source] ++ Hrls). init_graph(Config, Erls) -> KeepGraph = rebar_config:get_list(Config, keep_build_info, false), G = restore_graph("ebin", KeepGraph), lists:foreach( fun(Erl) -> update_graph(G, Erl, include_path(Erl, Config)) end, Erls), store_graph(G, "ebin", KeepGraph), G. update_graph(G, Source, IncludePath) -> case digraph:vertex(G, Source) of {_, LastUpdated} -> LastModified = filelib:last_modified(Source), if LastModified == 0 -> %% The file doesn't exist anymore, %% erase it from the graph. %% All the edges will be erased automatically. digraph:del_vertex(G, Source); LastUpdated < LastModified -> modify_graph(G, Source, IncludePath); true -> ok end; false -> modify_graph(G, Source, IncludePath) end. modify_graph(G, Source, IncludePath) -> case file:open(Source, [read]) of {ok, Fd} -> Incls = parse_attrs(Fd, []), AbsIncls = expand_file_names(Incls, IncludePath), catch file:close(Fd), LastUpdated = {date(), time()}, digraph:add_vertex(G, Source, LastUpdated), lists:foreach( fun(Incl) -> update_graph(G, Incl, IncludePath), digraph:add_edge(G, Source, Incl) end, AbsIncls); _Err -> ok end. restore_graph(_OutDir, _KeepGraph = false) -> digraph:new(); restore_graph(OutDir, _KeepGraph = true) -> File = filename:join(OutDir, ?REBAR_BUILD_INFO), G = digraph:new(), case file:read_file(File) of {ok, Data} -> case catch binary_to_term(Data) of {'EXIT', _} -> ok; {Vs, Es} -> lists:foreach( fun({V, LastUpdated}) -> digraph:add_vertex(G, V, LastUpdated) end, Vs), lists:foreach( fun({V1, V2}) -> digraph:add_edge(G, V1, V2) end, Es) end; _Err -> ok end, G. store_graph(_G, _OutDir, _KeepGraph = false) -> ok; store_graph(G, OutDir, _KeepGraph = true) -> Vs = lists:map( fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), Es = lists:flatmap( fun({V, _}) -> lists:map( fun(E) -> {_, V1, V2, _} = digraph:edge(G, E), {V1, V2} end, digraph:out_edges(G, V)) end, Vs), File = filename:join(OutDir, ?REBAR_BUILD_INFO), file:write_file(File, term_to_binary({Vs, Es})). -spec expand_file_names([file:filename()], [file:filename()]) -> [file:filename()]. expand_file_names(Files, IncludePath) -> %% We check if Files exist by itself or %% within the directories listed in IncludePath. %% Return the list of files matched. lists:flatmap( fun(Incl) -> case filelib:is_regular(Incl) of true -> [Incl]; false -> lists:flatmap( fun(Path) -> FullPath = filename:join(Path, Incl), case filelib:is_regular(FullPath) of true -> [FullPath]; false -> [] end end, IncludePath) end end, Files). -spec get_parents(digraph(), file:filename()) -> [file:filename()]. get_parents(G, Source) -> %% Return all files which the Source depends upon. digraph_utils:reachable_neighbours([Source], G). -spec get_children(digraph(), file:filename()) -> [file:filename()]. get_children(G, Source) -> %% Return all files dependent on the Source. digraph_utils:reaching_neighbours([Source], G). -spec internal_erl_compile(rebar_config:config(), file:filename(), file:filename(), list(), digraph()) -> 'ok' | 'skipped'. internal_erl_compile(Config, Source, Outdir, ErlOpts, G) -> %% Determine the target name and includes list by inspecting the source file Module = filename:basename(Source, ".erl"), Hrls = get_parents(G, Source), %% Construct the target filename Target = filename:join([Outdir | string:tokens(Module, ".")]) ++ ".beam", ok = filelib:ensure_dir(Target), %% If the file needs compilation, based on last mod date of includes or %% the target case needs_compile(Source, Target, Hrls) of true -> Opts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ [{i, "include"}, return], case compile:file(Source, Opts) of {ok, _Mod} -> ok; {ok, _Mod, Ws} -> rebar_base_compiler:ok_tuple(Config, Source, Ws); {error, Es, Ws} -> rebar_base_compiler:error_tuple(Config, Source, Es, Ws, Opts) end; false -> skipped end. -spec compile_mib(file:filename(), file:filename(), rebar_config:config()) -> 'ok'. compile_mib(Source, Target, Config) -> ok = rebar_utils:ensure_dir(Target), ok = rebar_utils:ensure_dir(filename:join("include", "dummy.hrl")), Opts = [{outdir, "priv/mibs"}, {i, ["priv/mibs"]}] ++ rebar_config:get(Config, mib_opts, []), case snmpc:compile(Source, Opts) of {ok, _} -> Mib = filename:rootname(Target), MibToHrlOpts = case proplists:get_value(verbosity, Opts, undefined) of undefined -> #options{specific = []}; Verbosity -> #options{specific = [{verbosity, Verbosity}]} end, ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts), Hrl_filename = Mib ++ ".hrl", rebar_file_utils:mv(Hrl_filename, "include"), ok; {error, compilation_failed} -> ?FAIL end. -spec compile_xrl(file:filename(), file:filename(), rebar_config:config()) -> 'ok'. compile_xrl(Source, Target, Config) -> Opts = [{scannerfile, Target} | rebar_config:get(Config, xrl_opts, [])], compile_xrl_yrl(Config, Source, Target, Opts, leex). -spec compile_yrl(file:filename(), file:filename(), rebar_config:config()) -> 'ok'. compile_yrl(Source, Target, Config) -> Opts = [{parserfile, Target} | rebar_config:get(Config, yrl_opts, [])], compile_xrl_yrl(Config, Source, Target, Opts, yecc). -spec compile_xrl_yrl(rebar_config:config(), file:filename(), file:filename(), list(), module()) -> 'ok'. compile_xrl_yrl(Config, Source, Target, Opts, Mod) -> case needs_compile(Source, Target, []) of true -> case Mod:file(Source, Opts ++ [{return, true}]) of {ok, _} -> ok; {ok, _Mod, Ws} -> rebar_base_compiler:ok_tuple(Config, Source, Ws); {error, Es, Ws} -> rebar_base_compiler:error_tuple(Config, Source, Es, Ws, Opts) end; false -> skipped end. gather_src([], Srcs) -> Srcs; gather_src([Dir|Rest], Srcs) -> gather_src(Rest, Srcs ++ rebar_utils:find_files(Dir, ".*\\.erl\$")). -spec dirs(file:filename()) -> [file:filename()]. dirs(Dir) -> [F || F <- filelib:wildcard(filename:join([Dir, "*"])), filelib:is_dir(F)]. -spec delete_dir(file:filename(), [string()]) -> 'ok' | {'error', atom()}. delete_dir(Dir, []) -> file:del_dir(Dir); delete_dir(Dir, Subdirs) -> lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, Subdirs), file:del_dir(Dir). parse_attrs(Fd, Includes) -> case io:parse_erl_form(Fd, "") of {ok, Form, _Line} -> case erl_syntax:type(Form) of attribute -> NewIncludes = process_attr(Form, Includes), parse_attrs(Fd, NewIncludes); _ -> parse_attrs(Fd, Includes) end; {eof, _} -> Includes; _Err -> parse_attrs(Fd, Includes) end. process_attr(Form, Includes) -> try AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)), process_attr(AttrName, Form, Includes) catch _:_ -> Includes end. process_attr(import, Form, Includes) -> case erl_syntax_lib:analyze_import_attribute(Form) of {Mod, _Funs} -> [atom_to_list(Mod) ++ ".erl"|Includes]; Mod -> [atom_to_list(Mod) ++ ".erl"|Includes] end; process_attr(file, Form, Includes) -> {File, _} = erl_syntax_lib:analyze_file_attribute(Form), [File|Includes]; process_attr(include, Form, Includes) -> [FileNode] = erl_syntax:attribute_arguments(Form), File = erl_syntax:string_value(FileNode), [File|Includes]; process_attr(include_lib, Form, Includes) -> [FileNode] = erl_syntax:attribute_arguments(Form), File = erl_syntax:string_value(FileNode), [File|Includes]; process_attr(behaviour, Form, Includes) -> [FileNode] = erl_syntax:attribute_arguments(Form), File = erl_syntax:atom_name(FileNode) ++ ".erl", [File|Includes]; process_attr(compile, Form, Includes) -> [Arg] = erl_syntax:attribute_arguments(Form), case erl_syntax:concrete(Arg) of {parse_transform, Mod} -> [atom_to_list(Mod) ++ ".erl"|Includes]; L when is_list(L) -> {_, Mod} = lists:keyfind(parse_transform, 1, L), [atom_to_list(Mod) ++ ".erl"|Includes] end. %% %% Ensure all files in a list are present and abort if one is missing %% -spec check_files([file:filename()]) -> [file:filename()]. check_files(FileList) -> [check_file(F) || F <- FileList]. check_file(File) -> case filelib:is_regular(File) of false -> ?ABORT("File ~p is missing, aborting\n", [File]); true -> File end.