From 4a6dc223b7a53acf99ffebeab95daef6f57f4be1 Mon Sep 17 00:00:00 2001 From: Andras Horvath Date: Wed, 24 Apr 2013 21:43:07 +0200 Subject: [PATCH] Add code coverage analysis functionality to `qc' - Use `cover' with QuickCheck testing - Reuse the `cover_*' rebar.config options - Refactor cover-related code to separate module (`qc_cover_utils') for use with both `eunit' and `qc' --- THANKS | 1 + dialyzer_reference | 4 +- ebin/rebar.app | 1 + src/rebar_cover_utils.erl | 257 ++++++++++++++++++++++++++++++++++++++ src/rebar_eunit.erl | 228 +-------------------------------- src/rebar_qc.erl | 39 ++++-- 6 files changed, 298 insertions(+), 232 deletions(-) create mode 100644 src/rebar_cover_utils.erl diff --git a/THANKS b/THANKS index ee95cee..ef359ac 100644 --- a/THANKS +++ b/THANKS @@ -125,3 +125,4 @@ YeJun Su Yuki Ito alisdair sullivan Alexander Verbitsky +Andras Horvath diff --git a/dialyzer_reference b/dialyzer_reference index 88909a6..c32104f 100644 --- a/dialyzer_reference +++ b/dialyzer_reference @@ -1,3 +1,3 @@ -rebar_eunit.erl:469: Call to missing or unexported function eunit_test:function_wrapper/2 -rebar_utils.erl:164: Call to missing or unexported function escript:foldl/3 +rebar_eunit.erl:471: Call to missing or unexported function eunit_test:function_wrapper/2 +rebar_utils.erl:184: Call to missing or unexported function escript:foldl/3 diff --git a/ebin/rebar.app b/ebin/rebar.app index 29ad8cf..cc9f751 100644 --- a/ebin/rebar.app +++ b/ebin/rebar.app @@ -14,6 +14,7 @@ rebar_cleaner, rebar_config, rebar_core, + rebar_cover_utils, rebar_ct, rebar_deps, rebar_edoc, diff --git a/src/rebar_cover_utils.erl b/src/rebar_cover_utils.erl new file mode 100644 index 0000000..a232867 --- /dev/null +++ b/src/rebar_cover_utils.erl @@ -0,0 +1,257 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% ------------------------------------------------------------------- +%% +%% rebar: Erlang Build Tools +%% +%% Copyright (c) 2009, 2010 Dave Smith (dizzyd@dizzyd.com) +%% Copyright (c) 2013 Andras Horvath (andras.horvath@erlang-solutions.com) +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ------------------------------------------------------------------- +-module(rebar_cover_utils). + +%% for internal use only +-export([init/3, + perform_cover/4, + close/1]). + +-include("rebar.hrl"). + +%% ==================================================================== +%% Internal functions +%% ==================================================================== + +perform_cover(Config, BeamFiles, SrcModules, TargetDir) -> + perform_cover(rebar_config:get(Config, cover_enabled, false), + Config, BeamFiles, SrcModules, TargetDir). + +perform_cover(false, _Config, _BeamFiles, _SrcModules, _TargetDir) -> + ok; +perform_cover(true, Config, BeamFiles, SrcModules, TargetDir) -> + analyze(Config, BeamFiles, SrcModules, TargetDir). + +close(not_enabled) -> + ok; +close(F) -> + ok = file:close(F). + +init(false, _BeamFiles, _TargetDir) -> + {ok, not_enabled}; +init(true, BeamFiles, TargetDir) -> + %% Attempt to start the cover server, then set its group leader to + %% TargetDir/cover.log, so all cover log messages will go there instead of + %% to stdout. If the cover server is already started, we'll kill that + %% server and start a new one in order not to inherit a polluted + %% cover_server state. + {ok, CoverPid} = case whereis(cover_server) of + undefined -> + cover:start(); + _ -> + cover:stop(), + cover:start() + end, + + {ok, F} = OkOpen = file:open( + filename:join([TargetDir, "cover.log"]), + [write]), + + group_leader(F, CoverPid), + + ?INFO("Cover compiling ~s\n", [rebar_utils:get_cwd()]), + + Compiled = [{Beam, cover:compile_beam(Beam)} || Beam <- BeamFiles], + case [Module || {_, {ok, Module}} <- Compiled] of + [] -> + %% No modules compiled successfully...fail + ?ERROR("Cover failed to compile any modules; aborting.~n", []), + ?FAIL; + _ -> + %% At least one module compiled successfully + + %% It's not an error for cover compilation to fail partially, + %% but we do want to warn about them + PrintWarning = + fun(Beam, Desc) -> + ?CONSOLE("Cover compilation warning for ~p: ~p", + [Beam, Desc]) + end, + _ = [PrintWarning(Beam, Desc) || {Beam, {error, Desc}} <- Compiled], + OkOpen + end; +init(Config, BeamFiles, TargetDir) -> + init(rebar_config:get(Config, cover_enabled, false), BeamFiles, TargetDir). + +analyze(_Config, [], _SrcModules, _TargetDir) -> + ok; +analyze(Config, FilteredModules, SrcModules, TargetDir) -> + %% Generate coverage info for all the cover-compiled modules + Coverage = lists:flatten([analyze_mod(M) + || M <- FilteredModules, + cover:is_compiled(M) =/= false]), + + %% Write index of coverage info + write_index(lists:sort(Coverage), SrcModules, TargetDir), + + %% Write coverage details for each file + lists:foreach( + fun({M, _, _}) -> + {ok, _} = cover:analyze_to_file(M, + cover_file(M, TargetDir), + [html]) + end, Coverage), + + Index = filename:join([rebar_utils:get_cwd(), TargetDir, "index.html"]), + ?CONSOLE("Cover analysis: ~s\n", [Index]), + + %% Export coverage data, if configured + case rebar_config:get(Config, cover_export_enabled, false) of + true -> + export_coverdata(TargetDir); + false -> + ok + end, + + %% Print coverage report, if configured + case rebar_config:get(Config, cover_print_enabled, false) of + true -> + print_coverage(lists:sort(Coverage)); + false -> + ok + end. + +analyze_mod(Module) -> + case cover:analyze(Module, coverage, module) of + {ok, {Module, {Covered, NotCovered}}} -> + %% Modules that include the eunit header get an implicit + %% test/0 fun, which cover considers a runnable line, but + %% eunit:test(TestRepresentation) never calls. Decrement + %% NotCovered in this case. + [align_notcovered_count(Module, Covered, NotCovered, + is_eunitized(Module))]; + {error, Reason} -> + ?ERROR("Cover analyze failed for ~p: ~p ~p\n", + [Module, Reason, code:which(Module)]), + [] + end. + +is_eunitized(Mod) -> + has_eunit_test_fun(Mod) andalso + has_header(Mod, "include/eunit.hrl"). + +has_eunit_test_fun(Mod) -> + [F || {exports, Funs} <- Mod:module_info(), + {F, 0} <- Funs, F =:= test] =/= []. + +has_header(Mod, Header) -> + Mod1 = case code:which(Mod) of + cover_compiled -> + {file, File} = cover:is_compiled(Mod), + File; + non_existing -> Mod; + preloaded -> Mod; + L -> L + end, + {ok, {_, [{abstract_code, {_, AC}}]}} = + beam_lib:chunks(Mod1, [abstract_code]), + [F || {attribute, 1, file, {F, 1}} <- AC, + string:str(F, Header) =/= 0] =/= []. + +align_notcovered_count(Module, Covered, NotCovered, false) -> + {Module, Covered, NotCovered}; +align_notcovered_count(Module, Covered, NotCovered, true) -> + {Module, Covered, NotCovered - 1}. + +write_index(Coverage, SrcModules, TargetDir) -> + {ok, F} = file:open(filename:join([TargetDir, "index.html"]), [write]), + ok = file:write(F, "\n" + "" + "Coverage Summary\n" + "\n"), + IsSrcCoverage = fun({Mod,_C,_N}) -> lists:member(Mod, SrcModules) end, + {SrcCoverage, TestCoverage} = lists:partition(IsSrcCoverage, Coverage), + write_index_section(F, "Source", SrcCoverage), + write_index_section(F, "Test", TestCoverage), + ok = file:write(F, ""), + ok = file:close(F). + +write_index_section(_F, _SectionName, []) -> + ok; +write_index_section(F, SectionName, Coverage) -> + %% Calculate total coverage + {Covered, NotCovered} = lists:foldl(fun({_Mod, C, N}, {CAcc, NAcc}) -> + {CAcc + C, NAcc + N} + end, {0, 0}, Coverage), + TotalCoverage = percentage(Covered, NotCovered), + + %% Write the report + ok = file:write(F, ?FMT("

~s Summary

\n", [SectionName])), + ok = file:write(F, ?FMT("

Total: ~s

\n", [TotalCoverage])), + ok = file:write(F, "\n"), + + FmtLink = + fun(Module, Cov, NotCov) -> + ?FMT("\n", + [Module, Module, percentage(Cov, NotCov)]) + end, + lists:foreach(fun({Module, Cov, NotCov}) -> + ok = file:write(F, FmtLink(Module, Cov, NotCov)) + end, Coverage), + ok = file:write(F, "
ModuleCoverage %
~s~s
\n"). + +print_coverage(Coverage) -> + {Covered, NotCovered} = lists:foldl(fun({_Mod, C, N}, {CAcc, NAcc}) -> + {CAcc + C, NAcc + N} + end, {0, 0}, Coverage), + TotalCoverage = percentage(Covered, NotCovered), + + %% Determine the longest module name for right-padding + Width = lists:foldl(fun({Mod, _, _}, Acc) -> + case length(atom_to_list(Mod)) of + N when N > Acc -> + N; + _ -> + Acc + end + end, 0, Coverage) * -1, + + %% Print the output the console + ?CONSOLE("~nCode Coverage:~n", []), + lists:foreach(fun({Mod, C, N}) -> + ?CONSOLE("~*s : ~3s~n", + [Width, Mod, percentage(C, N)]) + end, Coverage), + ?CONSOLE("~n~*s : ~s~n", [Width, "Total", TotalCoverage]). + +cover_file(Module, TargetDir) -> + filename:join([TargetDir, atom_to_list(Module) ++ ".COVER.html"]). + +export_coverdata(TargetDir) -> + ExportFile = filename:join(TargetDir, "cover.coverdata"), + case cover:export(ExportFile) of + ok -> + ?CONSOLE("Coverdata export: ~s~n", [ExportFile]); + {error, Reason} -> + ?ERROR("Coverdata export failed: ~p~n", [Reason]) + end. + +percentage(0, 0) -> + "not executed"; +percentage(Cov, NotCov) -> + integer_to_list(trunc((Cov / (Cov + NotCov)) * 100)) ++ "%". diff --git a/src/rebar_eunit.erl b/src/rebar_eunit.erl index 8532af1..a6fa0de 100644 --- a/src/rebar_eunit.erl +++ b/src/rebar_eunit.erl @@ -162,13 +162,15 @@ run_eunit(Config, CodePath, SrcErls) -> SrcModules = [rebar_utils:erl_to_mod(M) || M <- SrcErls], - {ok, CoverLog} = cover_init(Config, ModuleBeamFiles), + {ok, CoverLog} = rebar_cover_utils:init(Config, ModuleBeamFiles, + eunit_dir()), StatusBefore = status_before_eunit(), EunitResult = perform_eunit(Config, Tests), - perform_cover(Config, CoverageModules, SrcModules), - cover_close(CoverLog), + rebar_cover_utils:perform_cover(Config, CoverageModules, SrcModules, + eunit_dir()), + rebar_cover_utils:close(CoverLog), case proplists:get_value(reset_after_eunit, get_eunit_opts(Config), true) of @@ -498,226 +500,6 @@ get_eunit_opts(Config) -> BaseOpts ++ rebar_config:get_list(Config, eunit_opts, []). -%% -%% == code coverage == -%% - -perform_cover(Config, BeamFiles, SrcModules) -> - perform_cover(rebar_config:get(Config, cover_enabled, false), - Config, BeamFiles, SrcModules). - -perform_cover(false, _Config, _BeamFiles, _SrcModules) -> - ok; -perform_cover(true, Config, BeamFiles, SrcModules) -> - cover_analyze(Config, BeamFiles, SrcModules). - -cover_analyze(_Config, [], _SrcModules) -> - ok; -cover_analyze(Config, FilteredModules, SrcModules) -> - %% Generate coverage info for all the cover-compiled modules - Coverage = lists:flatten([cover_analyze_mod(M) - || M <- FilteredModules, - cover:is_compiled(M) =/= false]), - - %% Write index of coverage info - cover_write_index(lists:sort(Coverage), SrcModules), - - %% Write coverage details for each file - lists:foreach(fun({M, _, _}) -> - {ok, _} = cover:analyze_to_file(M, cover_file(M), - [html]) - end, Coverage), - - Index = filename:join([rebar_utils:get_cwd(), ?EUNIT_DIR, "index.html"]), - ?CONSOLE("Cover analysis: ~s\n", [Index]), - - %% Export coverage data, if configured - case rebar_config:get(Config, cover_export_enabled, false) of - true -> - cover_export_coverdata(); - false -> - ok - end, - - %% Print coverage report, if configured - case rebar_config:get(Config, cover_print_enabled, false) of - true -> - cover_print_coverage(lists:sort(Coverage)); - false -> - ok - end. - -cover_close(not_enabled) -> - ok; -cover_close(F) -> - ok = file:close(F). - -cover_init(false, _BeamFiles) -> - {ok, not_enabled}; -cover_init(true, BeamFiles) -> - %% Attempt to start the cover server, then set its group leader to - %% .eunit/cover.log, so all cover log messages will go there instead of - %% to stdout. If the cover server is already started, we'll kill that - %% server and start a new one in order not to inherit a polluted - %% cover_server state. - {ok, CoverPid} = case whereis(cover_server) of - undefined -> - cover:start(); - _ -> - cover:stop(), - cover:start() - end, - - {ok, F} = OkOpen = file:open( - filename:join([?EUNIT_DIR, "cover.log"]), - [write]), - - group_leader(F, CoverPid), - - ?INFO("Cover compiling ~s\n", [rebar_utils:get_cwd()]), - - Compiled = [{Beam, cover:compile_beam(Beam)} || Beam <- BeamFiles], - case [Module || {_, {ok, Module}} <- Compiled] of - [] -> - %% No modules compiled successfully...fail - ?ERROR("Cover failed to compile any modules; aborting.~n", []), - ?FAIL; - _ -> - %% At least one module compiled successfully - - %% It's not an error for cover compilation to fail partially, - %% but we do want to warn about them - PrintWarning = - fun(Beam, Desc) -> - ?CONSOLE("Cover compilation warning for ~p: ~p", - [Beam, Desc]) - end, - _ = [PrintWarning(Beam, Desc) || {Beam, {error, Desc}} <- Compiled], - OkOpen - end; -cover_init(Config, BeamFiles) -> - cover_init(rebar_config:get(Config, cover_enabled, false), BeamFiles). - -cover_analyze_mod(Module) -> - case cover:analyze(Module, coverage, module) of - {ok, {Module, {Covered, NotCovered}}} -> - %% Modules that include the eunit header get an implicit - %% test/0 fun, which cover considers a runnable line, but - %% eunit:test(TestRepresentation) never calls. Decrement - %% NotCovered in this case. - [align_notcovered_count(Module, Covered, NotCovered, - is_eunitized(Module))]; - {error, Reason} -> - ?ERROR("Cover analyze failed for ~p: ~p ~p\n", - [Module, Reason, code:which(Module)]), - [] - end. - -is_eunitized(Mod) -> - has_eunit_test_fun(Mod) andalso - has_header(Mod, "include/eunit.hrl"). - -has_eunit_test_fun(Mod) -> - [F || {exports, Funs} <- Mod:module_info(), - {F, 0} <- Funs, F =:= test] =/= []. - -has_header(Mod, Header) -> - Mod1 = case code:which(Mod) of - cover_compiled -> - {file, File} = cover:is_compiled(Mod), - File; - non_existing -> Mod; - preloaded -> Mod; - L -> L - end, - {ok, {_, [{abstract_code, {_, AC}}]}} = beam_lib:chunks(Mod1, - [abstract_code]), - [F || {attribute, 1, file, {F, 1}} <- AC, - string:str(F, Header) =/= 0] =/= []. - -align_notcovered_count(Module, Covered, NotCovered, false) -> - {Module, Covered, NotCovered}; -align_notcovered_count(Module, Covered, NotCovered, true) -> - {Module, Covered, NotCovered - 1}. - -cover_write_index(Coverage, SrcModules) -> - {ok, F} = file:open(filename:join([?EUNIT_DIR, "index.html"]), [write]), - ok = file:write(F, "\n" - "" - "Coverage Summary\n" - "\n"), - IsSrcCoverage = fun({Mod,_C,_N}) -> lists:member(Mod, SrcModules) end, - {SrcCoverage, TestCoverage} = lists:partition(IsSrcCoverage, Coverage), - cover_write_index_section(F, "Source", SrcCoverage), - cover_write_index_section(F, "Test", TestCoverage), - ok = file:write(F, ""), - ok = file:close(F). - -cover_write_index_section(_F, _SectionName, []) -> - ok; -cover_write_index_section(F, SectionName, Coverage) -> - %% Calculate total coverage - {Covered, NotCovered} = lists:foldl(fun({_Mod, C, N}, {CAcc, NAcc}) -> - {CAcc + C, NAcc + N} - end, {0, 0}, Coverage), - TotalCoverage = percentage(Covered, NotCovered), - - %% Write the report - ok = file:write(F, ?FMT("

~s Summary

\n", [SectionName])), - ok = file:write(F, ?FMT("

Total: ~s

\n", [TotalCoverage])), - ok = file:write(F, "\n"), - - FmtLink = - fun(Module, Cov, NotCov) -> - ?FMT("\n", - [Module, Module, percentage(Cov, NotCov)]) - end, - lists:foreach(fun({Module, Cov, NotCov}) -> - ok = file:write(F, FmtLink(Module, Cov, NotCov)) - end, Coverage), - ok = file:write(F, "
ModuleCoverage %
~s~s
\n"). - -cover_print_coverage(Coverage) -> - {Covered, NotCovered} = lists:foldl(fun({_Mod, C, N}, {CAcc, NAcc}) -> - {CAcc + C, NAcc + N} - end, {0, 0}, Coverage), - TotalCoverage = percentage(Covered, NotCovered), - - %% Determine the longest module name for right-padding - Width = lists:foldl(fun({Mod, _, _}, Acc) -> - case length(atom_to_list(Mod)) of - N when N > Acc -> - N; - _ -> - Acc - end - end, 0, Coverage) * -1, - - %% Print the output the console - ?CONSOLE("~nCode Coverage:~n", []), - lists:foreach(fun({Mod, C, N}) -> - ?CONSOLE("~*s : ~3s~n", - [Width, Mod, percentage(C, N)]) - end, Coverage), - ?CONSOLE("~n~*s : ~s~n", [Width, "Total", TotalCoverage]). - -cover_file(Module) -> - filename:join([?EUNIT_DIR, atom_to_list(Module) ++ ".COVER.html"]). - -cover_export_coverdata() -> - ExportFile = filename:join(eunit_dir(), "eunit.coverdata"), - case cover:export(ExportFile) of - ok -> - ?CONSOLE("Coverdata export: ~s~n", [ExportFile]); - {error, Reason} -> - ?ERROR("Coverdata export failed: ~p~n", [Reason]) - end. - -percentage(0, 0) -> - "not executed"; -percentage(Cov, NotCov) -> - integer_to_list(trunc((Cov / (Cov + NotCov)) * 100)) ++ "%". - %% %% == reset_after_eunit == %% diff --git a/src/rebar_qc.erl b/src/rebar_qc.erl index 1976722..cd3d288 100644 --- a/src/rebar_qc.erl +++ b/src/rebar_qc.erl @@ -4,7 +4,7 @@ %% %% rebar: Erlang Build Tools %% -%% Copyright (c) 2011-2012 Tuncer Ayaz +%% Copyright (c) 2011-2014 Tuncer Ayaz %% %% Permission is hereby granted, free of charge, to any person obtaining a copy %% of this software and associated documentation files (the "Software"), to deal @@ -68,11 +68,17 @@ info(help, qc) -> " {qc_opts, [{qc_mod, module()}, Options]}~n" " ~p~n" " ~p~n" + " ~p~n" + " ~p~n" + " ~p~n" "Valid command line options:~n" " compile_only=true (Compile but do not test properties)", [ {qc_compile_opts, []}, - {qc_first_files, []} + {qc_first_files, []}, + {cover_enabled, false}, + {cover_print_enabled, false}, + {cover_export_enabled, false} ]); info(help, clean) -> Description = ?FMT("Delete QuickCheck test dir (~s)", [?QC_DIR]), @@ -151,21 +157,40 @@ run(Config, QC, QCOpts) -> %% Compile erlang code to ?QC_DIR, using a tweaked config %% with appropriate defines, and include all the test modules %% as well. - {ok, _SrcErls} = rebar_erlc_compiler:test_compile(Config, "qc", ?QC_DIR), + {ok, SrcErls} = rebar_erlc_compiler:test_compile(Config, "qc", ?QC_DIR), case CompileOnly of "true" -> true = code:set_path(CodePath), ?CONSOLE("Compiled modules for qc~n", []); false -> - run1(QC, QCOpts, CodePath) + run1(QC, QCOpts, Config, CodePath, SrcErls) end. -run1(QC, QCOpts, CodePath) -> +run1(QC, QCOpts, Config, CodePath, SrcErls) -> + + AllBeamFiles = rebar_utils:beams(?QC_DIR), + AllModules = [rebar_utils:beam_to_mod(?QC_DIR, N) + || N <- AllBeamFiles], + PropMods = find_prop_mods(), + FilteredModules = AllModules -- PropMods, + + SrcModules = [rebar_utils:erl_to_mod(M) || M <- SrcErls], + + {ok, CoverLog} = rebar_cover_utils:init(Config, AllBeamFiles, qc_dir()), + TestModule = fun(M) -> qc_module(QC, QCOpts, M) end, - case lists:flatmap(TestModule, find_prop_mods()) of + QCResult = lists:flatmap(TestModule, PropMods), + + rebar_cover_utils:perform_cover(Config, FilteredModules, SrcModules, + qc_dir()), + rebar_cover_utils:close(CoverLog), + ok = cover:stop(), + + true = code:set_path(CodePath), + + case QCResult of [] -> - true = code:set_path(CodePath), ok; Errors -> ?ABORT("One or more QC properties didn't hold true:~n~p~n",