Fix #56 (always-on recursion)

Always-on recursive application of all rebar commands causes too many
issues. Recursive application is required for:
1. dealing with dependencies: get-deps, update-deps, and compile of deps
   right after get-deps or update-deps
2. projects with a riak-like apps/ project structure and dev process

The vast majority of projects are not structured like riak. Therefore,
moving forward it's best to (by default) restrict recursive behavior to
dealing with deps. This commit does that and also adds command line and
rebar.config options for controlling or configuring recursion. Also, we
introduce two meta commands: prepare-deps (equivalent to rebar -r
get-deps compile) and refresh-deps (equivalent to rebar -r update-deps
compile). riak-like projects can extend the list of recursive commands
(to include 'eunit' and 'compile') by adding
{recursive_cmds, [eunit, compile]} to rebar.config.

ebin/rebar.app

@@ -38,6 +38,7 @@
               rebar_upgrade,
               rebar_utils,
               rebar_xref,
+              rebar_metacmds,
               rebar_getopt,
               rebar_mustache ]},
   {registered, []},
@@ -80,7 +81,8 @@
                                rebar_escripter,
                                rebar_edoc,
                                rebar_shell,
-                               rebar_xref
+                               rebar_xref,
+                               rebar_metacmds
                               ]},
 
                     {rel_dir, [
@@ -88,6 +90,13 @@
                                rebar_reltool,
                                rebar_upgrade
                               ]}
-                   ]}
+                   ]},
+         {recursive_cmds, [
+                           'get-deps',
+                           'check-deps',
+                           'delete-deps',
+                           'list-deps',
+                           'update-deps'
+                          ]}
         ]}
  ]}.

inttest/depplugins/depplugins_rt.erl

@@ -39,7 +39,7 @@ files() ->
     ].
 
 run(_Dir) ->
-    ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])),
+    ?assertMatch({ok, _}, retest_sh:run("./rebar -r compile", [])),
 
     ?assertEqual(true, filelib:is_regular("base_dir_cwd_pre.compile")),
 

inttest/tdeps1/tdeps1_rt.erl

@@ -41,7 +41,8 @@ run(_Dir) ->
     apply_cmds(GitCmds, [{dir, "repo/b"}]),
     apply_cmds(GitCmds, [{dir, "repo/c"}]),
 
-    {ok, _} = retest_sh:run("./rebar get-deps compile", []),
+    {ok, _} = retest_sh:run("./rebar get-deps", []),
+    {ok, _} = retest_sh:run("./rebar -r compile", []),
 
     true = filelib:is_regular("ebin/a.beam"),
     ok.

inttest/tdeps2/tdeps2_rt.erl

@@ -49,7 +49,8 @@ run(_Dir) ->
     ok = apply_cmds(GitCmds, [{dir, "repo/b"}]),
     ok = apply_cmds(GitCmds, [{dir, "repo/c"}]),
 
-    {ok, _} = retest_sh:run("./rebar -v get-deps compile", []),
+    {ok, _} = retest_sh:run("./rebar -v get-deps", []),
+    {ok, _} = retest_sh:run("./rebar -v -r compile", []),
     ok.
 
 %%

inttest/tdeps_update/tdeps_update_rt.erl

@@ -122,7 +122,8 @@ run(_Dir) ->
     ok = apply_cmds(GitCmds++ECmds, [{dir, "repo/e"}]),
     ok = apply_cmds(GitCmds++FCmds, [{dir, "repo/f"}]),
 
-    {ok, _} = retest_sh:run("./rebar -v get-deps compile", []),
+    {ok, _} = retest_sh:run("./rebar -v get-deps", []),
+    {ok, _} = retest_sh:run("./rebar -v -r compile", []),
     os:cmd("cp a2.rebar.config apps/a1/rebar.config"),
     {ok, _} = retest_sh:run("./rebar -v update-deps", []),
     {ok, _} = retest_sh:run("./rebar -v compile", []),

priv/shell-completion/bash/rebar

@@ -6,7 +6,7 @@ _rebar()
     COMPREPLY=()
     cur="${COMP_WORDS[COMP_CWORD]}"
     prev="${COMP_WORDS[COMP_CWORD-1]}"
-    sopts="-h -c -v -V -f -D -j -C -p -k"
+    sopts="-h -c -v -V -f -D -j -C -p -k -r"
     lopts="--help \
         --commands \
         --verbose \
@@ -15,6 +15,7 @@ _rebar()
         --config \
         --profile \
         --keep-going \
+        --recursive \
         --version"
     cmdsnvars="check-deps \
         clean \
@@ -35,7 +36,9 @@ _rebar()
         help \
         list-deps \
         list-templates \
+        prepare-deps \
         qc \
+        refresh-deps \
         update-deps \
         version \
         xref \

priv/shell-completion/zsh/_rebar

@@ -17,6 +17,7 @@ _rebar_global_opts=(
   '(--config -C)'{--config,-C}'[Rebar config file to use]:files:_files'
   '(--profile -p)'{--profile,-p}'[Profile this run of rebar]'
   '(--keep-going -k)'{--keep-going,-k}'[Keep running after a command fails]'
+  '(--recursive -r)'{--recursive,-r}'[Apply commands to subdirs and dependencies]'
 )
 
 _rebar () {
@@ -36,6 +37,8 @@ _rebar () {
         'list-template[List avaiavle templates]' \
         'doc[Generate Erlang program documentation]' \
         'check-deps[Display to be fetched dependencies]' \
+        'prepare-deps[Fetch and build dependencies]' \
+        'refresh-deps[Update and build dependencies]' \
         'get-deps[Fetch dependencies]' \
         'update-deps[Update fetched dependencies]' \
         'delete-deps[Delete fetched dependencies]' \

rebar.config.sample

@@ -5,6 +5,9 @@
 
 %% == Core ==
 
+%% Extend list of always recursive commands
+{recursive_cmds, []}.
+
 %% Check required ERTS or OTP release version
 {require_erts_vsn, ".*"}.
 {require_otp_vsn, ".*"}.

src/rebar.erl

@@ -206,8 +206,10 @@ help() ->
        "  ~p~n"
        "  ~p~n"
        "  ~p~n"
+       "  ~p~n"
        "  ~p~n",
        [
+        {recursive_cmds, []},
         {lib_dirs, []},
         {sub_dirs, ["dir1", "dir2"]},
         {plugins, [plugin1, plugin2]},
@@ -254,19 +256,23 @@ save_options(Config, {Options, NonOptArgs}) ->
     Config3 = rebar_config:set_xconf(Config2, keep_going,
                                      proplists:get_bool(keep_going, Options)),
 
+    %% Setup flag to enable recursive application of commands
+    Config4 = rebar_config:set_xconf(Config3, recursive,
+                                     proplists:get_bool(recursive, Options)),
+
     %% Set global variables based on getopt options
-    Config4 = set_global_flag(Config3, Options, force),
+    Config5 = set_global_flag(Config4, Options, force),
-    Config5 = case proplists:get_value(jobs, Options, ?DEFAULT_JOBS) of
+    Config6 = case proplists:get_value(jobs, Options, ?DEFAULT_JOBS) of
                   ?DEFAULT_JOBS ->
-                      Config4;
+                      Config5;
                   Jobs ->
-                      rebar_config:set_global(Config4, jobs, Jobs)
+                      rebar_config:set_global(Config5, jobs, Jobs)
               end,
 
     %% Filter all the flags (i.e. strings of form key=value) from the
     %% command line arguments. What's left will be the commands to run.
-    {Config6, RawCmds} = filter_flags(Config5, NonOptArgs, []),
+    {Config7, RawCmds} = filter_flags(Config6, NonOptArgs, []),
-    {Config6, unabbreviate_command_names(RawCmds)}.
+    {Config7, unabbreviate_command_names(RawCmds)}.
 
 %%
 %% set log level based on getopt option
@@ -358,6 +364,9 @@ list-templates                           List available templates
 
 doc                                      Generate Erlang program documentation
 
+prepare-deps                             Run 'rebar -r get-deps compile'
+refresh-deps                             Run 'rebar -r update-deps compile'
+
 check-deps                               Display to be fetched dependencies
 get-deps                                 Fetch dependencies
 update-deps                              Update fetched dependencies
@@ -420,7 +429,9 @@ option_spec_list() ->
      {config,   $C, "config",   string,    "Rebar config file to use"},
      {profile,  $p, "profile",  undefined, "Profile this run of rebar"},
      {keep_going, $k, "keep-going", undefined,
-      "Keep running after a command fails"}
+      "Keep running after a command fails"},
+     {recursive, $r, "recursive", boolean,
+      "Apply commands to subdirs and dependencies"}
     ].
 
 %%
@@ -469,7 +480,9 @@ command_names() ->
      "help",
      "list-deps",
      "list-templates",
+     "prepare-deps",
      "qc",
+     "refresh-deps",
      "update-deps",
      "overlay",
      "shell",

src/rebar_config.erl

@@ -31,6 +31,7 @@
          get_all/2,
          set/3,
          set_global/3, get_global/3,
+         is_recursive/1,
          save_env/3, get_env/2, reset_envs/1,
          set_skip_dir/2, is_skip_dir/2, reset_skip_dirs/1,
          clean_config/2,
@@ -109,6 +110,9 @@ get_global(Config, Key, Default) ->
             Value
     end.
 
+is_recursive(Config) ->
+    get_xconf(Config, recursive, false).
+
 consult_file(File) ->
     case filename:extension(File) of
         ".script" ->

src/rebar_core.erl

@@ -122,26 +122,44 @@ process_dir(Dir, ParentConfig, Command, DirSet) ->
         false ->
             ?WARN("Skipping non-existent sub-dir: ~p\n", [Dir]),
             {ParentConfig, DirSet};
+        true ->
+            maybe_process_dir(Dir, ParentConfig, Command, DirSet)
+    end.
 
+maybe_process_dir(Dir, ParentConfig, Command, DirSet) ->
+    case should_cd_into_dir(Dir, ParentConfig, Command) of
         true ->
             ok = file:set_cwd(Dir),
             Config = maybe_load_local_config(Dir, ParentConfig),
 
             %% Save the current code path and then update it with
-            %% lib_dirs. Children inherit parents code path, but we
+            %% lib_dirs. Children inherit parents code path, but we also
-            %% also want to ensure that we restore everything to pristine
+            %% want to ensure that we restore everything to pristine
             %% condition after processing this child
             CurrentCodePath = update_code_path(Config),
 
-            %% Get the list of processing modules and check each one against
+            %% Get the list of processing modules and check each one
-            %% CWD to see if it's a fit -- if it is, use that set of modules
+            %% against CWD to see if it's a fit -- if it is, use that
-            %% to process this dir.
+            %% set of modules to process this dir.
             {ok, AvailModuleSets} = application:get_env(rebar, modules),
             ModuleSet = choose_module_set(AvailModuleSets, Dir),
             skip_or_process_dir(ModuleSet, Config, CurrentCodePath,
-                                Dir, Command, DirSet)
+                                Dir, Command, DirSet);
+        false ->
+            {ParentConfig, DirSet}
     end.
 
+should_cd_into_dir(Dir, Config, Command) ->
+    rebar_utils:processing_base_dir(Config, Dir) orelse
+        rebar_config:is_recursive(Config) orelse
+        is_recursive_command(Config, Command).
+
+is_recursive_command(Config, Command) ->
+    {ok, AppCmds} = application:get_env(rebar, recursive_cmds),
+    ConfCmds = rebar_config:get_local(Config, recursive_cmds, []),
+    RecursiveCmds = AppCmds ++ ConfCmds,
+    lists:member(Command, RecursiveCmds).
+
 skip_or_process_dir({[], undefined}=ModuleSet, Config, CurrentCodePath,
                     Dir, Command, DirSet) ->
     process_dir1(Dir, Command, DirSet, Config, CurrentCodePath, ModuleSet);

src/rebar_metacmds.erl

@@ -0,0 +1,56 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+%% -------------------------------------------------------------------
+%%
+%% rebar: Erlang Build Tools
+%%
+%% Copyright (c) 2013-2014 Tuncer Ayaz
+%%
+%% Permission is hereby granted, free of charge, to any person obtaining a copy
+%% of this software and associated documentation files (the "Software"), to deal
+%% in the Software without restriction, including without limitation the rights
+%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the Software is
+%% furnished to do so, subject to the following conditions:
+%%
+%% The above copyright notice and this permission notice shall be included in
+%% all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+%% THE SOFTWARE.
+%% -------------------------------------------------------------------
+-module(rebar_metacmds).
+
+-export(['prepare-deps'/2,
+         'refresh-deps'/2]).
+
+%% for internal use only
+-export([info/2]).
+
+-include("rebar.hrl").
+
+%% ===================================================================
+%% Public API
+%% ===================================================================
+'prepare-deps'(Config, _AppFile) ->
+    rebar:run(enable_recursion(Config), ["get-deps", "compile"]).
+
+'refresh-deps'(Config, _AppFile) ->
+    rebar:run(enable_recursion(Config), ["update-deps", "compile"]).
+
+%% ===================================================================
+%% Internal functions
+%% ===================================================================
+
+info(help, 'prepare-deps') ->
+    ?CONSOLE("Meta command to run 'rebar -r get-deps compile'.~n", []);
+info(help, 'refresh-deps') ->
+    ?CONSOLE("Meta command to run 'rebar -r update-deps compile'.~n", []).
+
+enable_recursion(Config) ->
+    rebar_config:set_xconf(Config, recursive, true).