Merge commit 'upstream/master'

THANKS

@@ -43,3 +43,4 @@ Jesper Louis Andersen
 Richard Jones
 Tim Watson
 Anders 'andekar'
+Christopher Brown

include/rebar.hrl

@@ -15,4 +15,3 @@
 -define(ERROR(Str, Args), rebar_log:log(error, Str, Args)).
 
 -define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))).
-

inttest/ct2/ct2_rt.erl

@@ -0,0 +1,26 @@
+-module(ct2_rt).
+
+-compile(export_all).
+
+
+files() ->
+    [{create, "ebin/foo.app", app(foo)},
+     {copy, "../../rebar", "rebar"},
+     {copy, "foo.test.spec", "test/foo.test.spec"},
+     {copy, "foo_SUITE.erl", "test/foo_SUITE.erl"}].
+
+run(_Dir) ->
+    {ok, _} = retest:sh("./rebar compile ct -v"),
+    ok.
+
+%%
+%% Generate the contents of a simple .app file
+%%
+app(Name) ->
+    App = {application, Name,
+           [{description, atom_to_list(Name)},
+            {vsn, "1"},
+            {modules, []},
+            {registered, []},
+            {applications, [kernel, stdlib]}]},
+    io_lib:format("~p.\n", [App]).

inttest/ct2/foo.test.spec

@@ -0,0 +1 @@
+{suites, "test", all}.

inttest/ct2/foo_SUITE.erl

@@ -0,0 +1,10 @@
+-module(foo_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+all() -> [foo].
+
+foo(Config) ->
+    io:format("Test: ~p\n", [Config]).

src/rebar_appups.erl

@@ -55,14 +55,8 @@
              NewName == OldName,
              "Reltool and .rel release names do not match~n", []),
 
-    %% Get lists of the old and new app files
-    OldAppFiles = rebar_utils:find_files(
-                    filename:join([OldVerPath, "lib"]), "^.*.app$"),
-    NewAppFiles = rebar_utils:find_files(
-                    filename:join([NewName, "lib"]), "^.*.app$"),
-
     %% Find all the apps that have been upgraded
-    UpgradedApps = get_upgraded_apps(OldAppFiles, NewAppFiles),
+    UpgradedApps = get_upgraded_apps(Name, OldVerPath, NewVerPath),
 
     %% Get a list of any appup files that exist in the new release
     NewAppUpFiles = rebar_utils:find_files(
@@ -85,18 +79,24 @@
 %% Internal functions
 %% ===================================================================
 
-get_upgraded_apps(OldAppFiles, NewAppFiles) ->
+get_upgraded_apps(Name, OldVerPath, NewVerPath) ->
-    OldAppsVer = [{rebar_app_utils:app_name(AppFile),
+    OldApps = rebar_rel_utils:get_rel_apps(Name, OldVerPath),
-                   rebar_app_utils:app_vsn(AppFile)} || AppFile <- OldAppFiles],
+    NewApps = rebar_rel_utils:get_rel_apps(Name, NewVerPath),
-    NewAppsVer = [{rebar_app_utils:app_name(AppFile),
+
-                   rebar_app_utils:app_vsn(AppFile)} || AppFile <- NewAppFiles],
+    Sorted = lists:umerge(lists:sort(NewApps), lists:sort(OldApps)),
-    UpgradedApps = lists:subtract(NewAppsVer, OldAppsVer),
+    AddedorChanged = lists:subtract(Sorted, OldApps),
-    lists:map(
+    DeletedorChanged = lists:subtract(Sorted, NewApps),
-      fun({App, NewVer}) ->
+    ?DEBUG("Added or Changed: ~p~n", [AddedorChanged]),
-              {App, OldVer} = proplists:lookup(App, OldAppsVer),
+    ?DEBUG("Deleted or Changed: ~p~n", [DeletedorChanged]),
-              {App, {OldVer, NewVer}}
+
-      end,
+    AddedDeletedChanged = lists:ukeysort(1, lists:append(DeletedorChanged,
-      UpgradedApps).
+                                                         AddedorChanged)),
+    UpgradedApps = lists:subtract(AddedorChanged, AddedDeletedChanged),
+    ?DEBUG("Upgraded Apps:~p~n", [UpgradedApps]),
+
+    [{AppName, {proplists:get_value(AppName, OldApps), NewVer}}
+     || {AppName, NewVer} <- UpgradedApps].
+
 
 file_to_name(File) ->
     filename:rootname(filename:basename(File)).

src/rebar_core.erl

@@ -76,6 +76,10 @@ process_commands([Command | Rest]) ->
     lists:foreach(fun (D) -> erlang:erase({skip_dir, D}) end, skip_dirs()),
     Operations = erlang:get(operations),
 
+    %% Convert the code path so that all the entries are absolute paths.
+    %% If not, code:set_path() may choke on invalid relative paths when trying
+    %% to restore the code path from inside a subdirectory.
+    true = rebar_utils:expand_code_path(),
     _ = process_dir(rebar_utils:get_cwd(), rebar_config:new(),
                     Command, sets:new()),
     case erlang:get(operations) of
@@ -280,8 +284,8 @@ expand_lib_dirs([Dir | Rest], Root, Acc) ->
 select_modules([], _Command, Acc) ->
     lists:reverse(Acc);
 select_modules([Module | Rest], Command, Acc) ->
-    Exports = Module:module_info(exports),
+    {module, Module} = code:ensure_loaded(Module),
-    case lists:member({Command, 2}, Exports) of
+    case erlang:function_exported(Module, Command, 2) of
         true ->
             select_modules(Rest, Command, [Module | Acc]);
         false ->

src/rebar_ct.erl

@@ -141,32 +141,57 @@ make_cmd(TestDir, Config) ->
     CodeDirs = [io_lib:format("\"~s\"", [Dir]) ||
                    Dir <- [EbinDir|NonLibCodeDirs]],
     CodePathString = string:join(CodeDirs, " "),
-    Cmd = ?FMT("erl " % should we expand ERL_PATH?
+    Cmd = case get_ct_specs(Cwd) of
-               " -noshell -pa ~s ~s"
+        undefined ->
-               " -s ct_run script_start -s erlang halt"
+            ?FMT("erl " % should we expand ERL_PATH?
-               " -name test@~s"
+                       " -noshell -pa ~s ~s"
-               " -logdir \"~s\""
+                       " -s ct_run script_start -s erlang halt"
-               " -env TEST_DIR \"~s\"",
+                       " -name test@~s"
-               [CodePathString,
+                       " -logdir \"~s\""
-                Include,
+                       " -env TEST_DIR \"~s\"",
-                net_adm:localhost(),
+                       [CodePathString,
-                LogDir,
+                        Include,
-                filename:join(Cwd, TestDir)]) ++
+                        net_adm:localhost(),
-        get_cover_config(Config, Cwd) ++
+                        LogDir,
-        get_ct_config_file(TestDir) ++
+                        filename:join(Cwd, TestDir)]) ++
-        get_config_file(TestDir) ++
+                get_cover_config(Config, Cwd) ++
-        get_suite(TestDir) ++
+                get_ct_config_file(TestDir) ++
-        get_case(),
+                get_config_file(TestDir) ++
+                get_suite(TestDir) ++
+                get_case();
+        SpecFlags ->
+            ?FMT("erl " % should we expand ERL_PATH?
+                       " -noshell -pa ~s ~s"
+                       " -s ct_run script_start -s erlang halt"
+                       " -name test@~s"
+                       " -logdir \"~s\""
+                       " -env TEST_DIR \"~s\"",
+                       [CodePathString,
+                        Include,
+                        net_adm:localhost(),
+                        LogDir,
+                        filename:join(Cwd, TestDir)]) ++
+                SpecFlags ++ get_cover_config(Config, Cwd)
+    end,
     RawLog = filename:join(LogDir, "raw.log"),
     {Cmd, RawLog}.
 
+get_ct_specs(Cwd) ->
+    case collect_glob(Cwd, ".*\.test\.spec\$") of
+        [] -> undefined;
+        [Spec] ->
+            " -spec " ++ Spec;
+        Specs ->
+            " -spec " ++
+                lists:flatten([io_lib:format("~s ", [Spec]) || Spec <- Specs])
+    end.
+
 get_cover_config(Config, Cwd) ->
     case rebar_config:get_local(Config, cover_enabled, false) of
         false ->
             "";
         true ->
-            case filelib:fold_files(Cwd, ".*cover\.spec\$",
+            case collect_glob(Cwd, ".*cover\.spec\$") of
-                                    true, fun collect_ct_specs/2, []) of
                 [] ->
                     ?DEBUG("No cover spec found: ~s~n", [Cwd]),
                     "";
@@ -178,15 +203,18 @@ get_cover_config(Config, Cwd) ->
             end
     end.
 
-collect_ct_specs(F, Acc) ->
+collect_glob(Cwd, Glob) ->
+    filelib:fold_files(Cwd, Glob, true, fun collect_files/2, []).
+
+collect_files(F, Acc) ->
     %% Ignore any specs under the deps/ directory. Do this pulling
     %% the dirname off the the F and then splitting it into a list.
     Parts = filename:split(filename:dirname(F)),
     case lists:member("deps", Parts) of
         true ->
-            Acc;                     % There is a directory named "deps" in path
+            Acc;                % There is a directory named "deps" in path
         false ->
-            [F | Acc]                % No "deps" directory in path
+            [F | Acc]           % No "deps" directory in path
     end.
 
 get_ct_config_file(TestDir) ->

src/rebar_deps.erl

@@ -131,7 +131,7 @@ compile(Config, AppFile) ->
 
 'delete-deps'(Config, _) ->
     %% Delete all the available deps in our deps/ directory, if any
-    DepsDir = get_deps_dir(),
+    {true, DepsDir} = get_deps_dir(),
     Deps = rebar_config:get_local(Config, deps, []),
     {AvailableDeps, _} = find_deps(find, Deps),
     _ = [delete_dep(D)
@@ -154,9 +154,21 @@ set_global_deps_dir(_Config, _DepsDir) ->
     ok.
 
 get_deps_dir() ->
+    get_deps_dir("").
+
+get_deps_dir(App) ->
     BaseDir = rebar_config:get_global(base_dir, []),
     DepsDir = rebar_config:get_global(deps_dir, "deps"),
-    filename:join(BaseDir, DepsDir).
+    {true, filename:join([BaseDir, DepsDir, App])}.
+
+get_lib_dir(App) ->
+    % Find App amongst the reachable lib directories
+    % Returns either the found path or a tagged tuple with a boolean
+    % to match get_deps_dir's return type
+    case code:lib_dir(App) of
+        {error, bad_name} -> {false, bad_name};
+        Path -> {true, Path}
+    end.
 
 update_deps_code_path([]) ->
     ok;
@@ -189,24 +201,42 @@ find_deps(Mode, [{App, VsnRegex, Source} | Rest], Acc) ->
     Dep = #dep { app = App,
                  vsn_regex = VsnRegex,
                  source = Source },
-    case is_app_available(App, VsnRegex) of
+    {Availability, FoundDir} = find_dep(Dep),
-        {true, AppDir} ->
+    find_deps(Mode, Rest, acc_deps(Mode, Availability, Dep, FoundDir, Acc));
-            find_deps(Mode, Rest, acc_deps(Mode, avail, Dep, AppDir, Acc));
-        {false, _} ->
-            AppDir = filename:join(get_deps_dir(), Dep#dep.app),
-            case is_app_available(App, VsnRegex, AppDir) of
-                {true, AppDir} ->
-                    find_deps(Mode, Rest,
-                              acc_deps(Mode, avail, Dep, AppDir, Acc));
-                {false, _} ->
-                    find_deps(Mode, Rest,
-                              acc_deps(Mode, missing, Dep, AppDir, Acc))
-            end
-    end;
 find_deps(_Mode, [Other | _Rest], _Acc) ->
     ?ABORT("Invalid dependency specification ~p in ~s\n",
            [Other, rebar_utils:get_cwd()]).
 
+find_dep(Dep) ->
+    % Find a dep based on its source,
+    % e.g. {git, "https://github.com/mochi/mochiweb.git", "HEAD"}
+    % Deps with a source must be found (or fetched) locally.
+    % Those without a source may be satisfied from lib directories (get_lib_dir).
+    find_dep(Dep, Dep#dep.source).
+
+find_dep(Dep, undefined) ->
+    % 'source' is undefined.  If Dep is not satisfied locally,
+    % go ahead and find it amongst the lib_dir's.
+    case find_dep_in_dir(Dep, get_deps_dir(Dep#dep.app)) of
+        {avail, Dir} -> {avail, Dir};
+        {missing, _} -> find_dep_in_dir(Dep, get_lib_dir(Dep#dep.app))
+    end;
+find_dep(Dep, _Source) ->
+    % _Source is defined.  Regardless of what it is, we must find it
+    % locally satisfied or fetch it from the original source
+    % into the project's deps
+    find_dep_in_dir(Dep, get_deps_dir(Dep#dep.app)).
+
+find_dep_in_dir(_Dep, {false, Dir}) ->
+    {missing, Dir};
+find_dep_in_dir(Dep, {true, Dir}) ->
+    App = Dep#dep.app,
+    VsnRegex = Dep#dep.vsn_regex,
+    case is_app_available(App, VsnRegex, Dir) of
+        {true, _AppFile} -> {avail, Dir};
+        {false, _}       -> {missing, Dir}
+    end.
+
 acc_deps(find, avail, Dep, AppDir, {Avail, Missing}) ->
     {[Dep#dep { dir = AppDir } | Avail], Missing};
 acc_deps(find, missing, Dep, AppDir, {Avail, Missing}) ->
@@ -227,15 +257,8 @@ require_source_engine(Source) ->
     true = source_engine_avail(Source),
     ok.
 
-is_app_available(App, VsnRegex) ->
-    case code:lib_dir(App) of
-        {error, bad_name} ->
-            {false, bad_name};
-        Path ->
-            is_app_available(App, VsnRegex, Path)
-    end.
-
 is_app_available(App, VsnRegex, Path) ->
+    ?DEBUG("is_app_available, looking for App ~p  with Path ~p~n", [App, Path]),
     case rebar_app_utils:is_app_dir(Path) of
         {true, AppFile} ->
             case rebar_app_utils:app_name(AppFile) of
@@ -293,7 +316,7 @@ use_source(Dep, Count) ->
         false ->
             ?CONSOLE("Pulling ~p from ~p\n", [Dep#dep.app, Dep#dep.source]),
             require_source_engine(Dep#dep.source),
-            TargetDir = filename:join(get_deps_dir(), Dep#dep.app),
+            {true, TargetDir} = get_deps_dir(Dep#dep.app),
             download_source(TargetDir, Dep#dep.source),
             use_source(Dep#dep { dir = TargetDir }, Count-1)
     end.
@@ -335,7 +358,7 @@ update_source(Dep) ->
     %% VCS directory, such as when a source archive is built of a project, with
     %% all deps already downloaded/included. So, verify that the necessary VCS
     %% directory exists before attempting to do the update.
-    AppDir = filename:join(get_deps_dir(), Dep#dep.app),
+    {true, AppDir} = get_deps_dir(Dep#dep.app),
     case has_vcs_dir(element(1, Dep#dep.source), AppDir) of
         true ->
             ?CONSOLE("Updating ~p from ~p\n", [Dep#dep.app, Dep#dep.source]),

src/rebar_lfe_compiler.erl

@@ -41,27 +41,40 @@ compile(Config, _AppFile) ->
     rebar_base_compiler:run(Config, FirstFiles, "src", ".lfe", "ebin", ".beam",
                             fun compile_lfe/3).
 
-
 %% ===================================================================
 %% Internal functions
 %% ===================================================================
 
-compile_lfe(Source, _Target, Config) ->
+compile_lfe(Source, Target, Config) ->
     case code:which(lfe_comp) of
         non_existing ->
-            ?CONSOLE(
+            ?CONSOLE(<<
-               <<"~n===============================================~n"
+                       "~n"
-                 " You need to install LFE to compile LFE source files~n"
+                       "*** MISSING LFE COMPILER ***~n"
-                 "Download the latest tarball release from github~n"
+                       "  You must do one of the following:~n"
-                 "   https://github.com/rvirding/lfe/downloads~n"
+                       "    a) Install LFE globally in your erl libs~n"
-                 "  and install it into your erlang library dir~n"
+                       "    b) Add LFE as a dep for your project, eg:~n"
-                 "===============================================~n~n">>, []),
+                       "       {lfe, \"0.6.1\",~n"
+                       "        {git, \"git://github.com/rvirding/lfe\",~n"
+                       "         {tag, \"v0.6.1\"}}}~n"
+                       "~n"
+                     >>, []),
             ?FAIL;
         _ ->
             Opts = [{i, "include"}, {outdir, "ebin"}, report, return] ++
                 rebar_config:get_list(Config, lfe_opts, []),
-            case lfe_comp:file(Source,Opts) of
+            case lfe_comp:file(Source, Opts) of
-                {ok, _, []} -> ok;
+                {ok, _, []} ->
-                _ -> ?FAIL
+                    ok;
+                {ok, _, _Warnings} ->
+                    case lists:member(fail_on_warning, Opts) of
+                        true ->
+                            ok = file:delete(Target),
+                            ?FAIL;
+                        false ->
+                            ok
+                    end;
+                _ ->
+                    ?FAIL
             end
     end.

src/rebar_rel_utils.erl

@@ -31,7 +31,10 @@
          get_reltool_release_info/1,
          get_rel_release_info/1,
          get_rel_release_info/2,
-         get_previous_release_path/0]).
+         get_rel_apps/1,
+         get_rel_apps/2,
+         get_previous_release_path/0,
+         get_rel_file_path/2]).
 
 -include("rebar.hrl").
 
@@ -70,11 +73,29 @@ get_rel_release_info(RelFile) ->
 
 %% Get release name and version from a name and a path
 get_rel_release_info(Name, Path) ->
+    RelPath = get_rel_file_path(Name, Path),
+    get_rel_release_info(RelPath).
+
+%% Get list of apps included in a release from a rel file
+get_rel_apps(RelFile) ->
+    case file:consult(RelFile) of
+        {ok, [{release, _, _, Apps}]} ->
+            Apps;
+        _ ->
+            ?ABORT("Failed to parse ~s~n", [RelFile])
+    end.
+
+%% Get list of apps included in a release from a name and a path
+get_rel_apps(Name, Path) ->
+    RelPath = get_rel_file_path(Name, Path),
+    get_rel_apps(RelPath).
+
+%% Get rel file path from name and path
+get_rel_file_path(Name, Path) ->
     [RelFile] = filelib:wildcard(filename:join([Path, "releases", "*",
                                                 Name ++ ".rel"])),
     [BinDir|_] = re:replace(RelFile, Name ++ "\\.rel", ""),
-    get_rel_release_info(filename:join([binary_to_list(BinDir),
+    filename:join([binary_to_list(BinDir), Name ++ ".rel"]).
-                                        Name ++ ".rel"])).
 
 %% Get the previous release path from a global variable
 get_previous_release_path() ->

src/rebar_utils.erl

@@ -38,7 +38,8 @@
          abort/2,
          escript_foldl/3,
          find_executable/1,
-         prop_check/3]).
+         prop_check/3,
+         expand_code_path/0]).
 
 -include("rebar.hrl").
 
@@ -156,6 +157,14 @@ find_executable(Name) ->
 prop_check(true, _, _) -> true;
 prop_check(false, Msg, Args) -> ?ABORT(Msg, Args).
 
+%% Convert all the entries in the code path to absolute paths.
+expand_code_path() ->
+    CodePath = lists:foldl(fun (Path, Acc) ->
+                                   [filename:absname(Path) | Acc]
+                           end, [], code:get_path()),
+    code:set_path(lists:reverse(CodePath)).
+
+
 %% ====================================================================
 %% Internal functions
 %% ====================================================================

test/upgrade_project/rel/files/dummy

@@ -7,6 +7,7 @@ RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd)
 RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}
 RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc
 RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log
+# Note the trailing slash on $PIPE_DIR/
 PIPE_DIR=/tmp/$RUNNER_BASE_DIR/
 RUNNER_USER=
 
@@ -61,8 +62,7 @@ case "$1" in
         HEART_COMMAND="$RUNNER_BASE_DIR/bin/$SCRIPT start"
         export HEART_COMMAND
         mkdir -p $PIPE_DIR
-        # Note the trailing slash on $PIPE_DIR/
+        $ERTS_PATH/run_erl -daemon $PIPE_DIR $RUNNER_LOG_DIR "exec $RUNNER_BASE_DIR/bin/$SCRIPT console" 2>&1
-        $ERTS_PATH/run_erl -daemon $PIPE_DIR/ $RUNNER_LOG_DIR "exec $RUNNER_BASE_DIR/bin/$SCRIPT console" 2>&1
         ;;
 
     stop)
@@ -147,7 +147,7 @@ case "$1" in
         ;;
 
     *)
-        echo "Usage: $SCRIPT {start|stop|restart|reboot|ping|console|attach}"
+        echo "Usage: $SCRIPT {start|stop|restart|reboot|ping|console|console_clean|attach}"
         exit 1
         ;;
 esac