Add support for non-Erlang/OTP (raw) dependencies

Introduce a new 'raw' option for dependency specs in rebar.config file.
For example:

{deps,
    {dependency_name, "1.0.*",
	 {git, "<...>", {branch, "master"}},
	 [raw]
    }
]}.

When this option is specified, rebar does not require the dependency to
have a standard Erlang/OTP layout which assumes presence of either
"src/dependency_name.app.src" or "ebin/dependency_name.app" files.

'raw' dependencies can still contain 'rebar.config' and even can have
the proper OTP directory layout, but they won't be compiled.

Only a subset of rebar commands will be executed on the 'raw'
subdirectories:

	get-deps, update-deps, check-deps, list-deps and delete-deps.

inttest/t_custom_config/t_custom_config_rt.erl

@@ -20,7 +20,7 @@ run(Dir) ->
     {ok, Missing} =
         retest:sh_expect(Ref,
                          "DEBUG: Missing deps  : \\[\\{dep,bad_name,"
-                         "boo,\"\\.\",undefined\\}\\]",
+                         "boo,\"\\.\",undefined,false\\}\\]",
                          [{capture, all, list}]),
     retest_log:log(debug, "[CAPTURED]: ~s~n", [Captured]),
     retest_log:log(debug, "[Missing]: ~s~n", [Missing]),

rebar.config.sample

@@ -138,7 +138,20 @@
 {deps, [application_name,
         {application_name, "1.0.*"},
         {application_name, "1.0.*",
-         {git, "git://github.com/basho/rebar.git", {branch, "master"}}}]}.
+         {git, "git://github.com/basho/rebar.git", {branch, "master"}}},
+
+%% Dependencies can be marked as 'raw'. Rebar does not require such dependencies
+%% to have a standard Erlang/OTP layout which assumes the presence of either
+%% "src/dependency_name.app.src" or "ebin/dependency_name.app" files.
+%%
+%% 'raw' dependencies can still contain 'rebar.config' and even can have the
+%% proper OTP directory layout, but they won't be compiled.
+%%
+%% Only a subset of rebar commands will be executed on the 'raw' subdirectories:
+%% get-deps, update-deps, check-deps, list-deps and delete-deps.
+        {application_name, "",
+         {git, "git://github.com/basho/rebar.git", {branch, "master"}},
+         [raw]}]}.
 
 %% == Subdirectories ==
 

src/rebar_deps.erl

@@ -42,7 +42,8 @@
 -record(dep, { dir,
                app,
                vsn_regex,
-               source }).
+               source,
+               is_raw }). %% is_raw = true means non-Erlang/OTP dependency
 
 %% ===================================================================
 %% Public API
@@ -78,8 +79,12 @@ preprocess(Config, _) ->
                         Config3
                 end,
 
+    %% Filtering out 'raw' dependencies so that no commands other than
+    %% deps-related can be executed on their directories.
+    NonRawAvailableDeps = [D || D <- AvailableDeps, not D#dep.is_raw],
+
     %% Return all the available dep directories for process
-    {ok, NewConfig, dep_dirs(AvailableDeps)}.
+    {ok, NewConfig, dep_dirs(NonRawAvailableDeps)}.
 
 postprocess(Config, _) ->
     case rebar_config:get_xconf(Config, ?MODULE, undefined) of
@@ -90,8 +95,9 @@ postprocess(Config, _) ->
             {ok, NewConfig, Dirs}
     end.
 
-compile(Config, AppFile) ->
+compile(Config, _) ->
-    'check-deps'(Config, AppFile).
+    {Config1, _AvailDeps} = do_check_deps(Config),
+    {ok, Config1}.
 
 %% set REBAR_DEPS_DIR and ERL_LIBS environment variables
 setup_env(Config) ->
@@ -111,13 +117,14 @@ setup_env(Config) ->
                end,
     [{"REBAR_DEPS_DIR", DepsDir}, ERL_LIBS].
 
-'check-deps'(Config, _) ->
+%% common function used by 'check-deps' and 'compile'
+do_check_deps(Config) ->
     %% Get the list of immediate (i.e. non-transitive) deps that are missing
     Deps = rebar_config:get_local(Config, deps, []),
     case find_deps(Config, find, Deps) of
-        {Config1, {_, []}} ->
+        {Config1, {AvailDeps, []}} ->
             %% No missing deps
-            {ok, Config1};
+            {Config1, AvailDeps};
         {_Config1, {_, MissingDeps}} ->
             lists:foreach(fun (#dep{app=App, vsn_regex=Vsn, source=Src}) ->
                                   ?CONSOLE("Dependency not available: "
@@ -126,6 +133,10 @@ setup_env(Config) ->
             ?FAIL
     end.
 
+'check-deps'(Config, _) ->
+    {Config1, AvailDeps} = do_check_deps(Config),
+    {ok, save_dep_dirs(Config1, AvailDeps)}.
+
 'get-deps'(Config, _) ->
     %% Determine what deps are available and missing
     Deps = rebar_config:get_local(Config, deps, []),
@@ -171,7 +182,7 @@ setup_env(Config) ->
     case find_deps(Config, find, Deps) of
         {Config1, {AvailDeps, []}} ->
             lists:foreach(fun(Dep) -> print_source(Dep) end, AvailDeps),
-            {ok, Config1};
+            {ok, save_dep_dirs(Config1, AvailDeps)};
         {_, MissingDeps} ->
             ?ABORT("Missing dependencies: ~p\n", [MissingDeps])
     end.
@@ -221,7 +232,7 @@ update_deps_code_path(Config, []) ->
 update_deps_code_path(Config, [Dep | Rest]) ->
     Config2 =
         case is_app_available(Config, Dep#dep.app,
-                              Dep#dep.vsn_regex, Dep#dep.dir) of
+                              Dep#dep.vsn_regex, Dep#dep.dir, Dep#dep.is_raw) of
             {Config1, {true, _}} ->
                 Dir = filename:join(Dep#dep.dir, "ebin"),
                 ok = filelib:ensure_dir(filename:join(Dir, "dummy")),
@@ -247,9 +258,14 @@ find_deps(Config, Mode, [App | Rest], Acc) when is_atom(App) ->
 find_deps(Config, Mode, [{App, VsnRegex} | Rest], Acc) when is_atom(App) ->
     find_deps(Config, Mode, [{App, VsnRegex, undefined} | Rest], Acc);
 find_deps(Config, Mode, [{App, VsnRegex, Source} | Rest], Acc) ->
+    find_deps(Config, Mode, [{App, VsnRegex, Source, []} | Rest], Acc);
+find_deps(Config, Mode, [{App, VsnRegex, Source, Opts} | Rest], Acc) when is_list(Opts) ->
     Dep = #dep { app = App,
                  vsn_regex = VsnRegex,
-                 source = Source },
+                 source = Source,
+                 %% dependency is considered raw (i.e. non-Erlang/OTP) when
+                 %% 'raw' option is present
+                 is_raw = proplists:get_value(raw, Opts, false) },
     {Config1, {Availability, FoundDir}} = find_dep(Config, Dep),
     find_deps(Config1, Mode, Rest,
               acc_deps(Mode, Availability, Dep, FoundDir, Acc));
@@ -284,7 +300,8 @@ find_dep_in_dir(Config, _Dep, {false, Dir}) ->
 find_dep_in_dir(Config, Dep, {true, Dir}) ->
     App = Dep#dep.app,
     VsnRegex = Dep#dep.vsn_regex,
-    case is_app_available(Config, App, VsnRegex, Dir) of
+    IsRaw = Dep#dep.is_raw,
+    case is_app_available(Config, App, VsnRegex, Dir, IsRaw) of
         {Config1, {true, _AppFile}} -> {Config1, {avail, Dir}};
         {Config1, {false, _}}       -> {Config1, {missing, Dir}}
     end.
@@ -309,7 +326,11 @@ require_source_engine(Source) ->
     true = source_engine_avail(Source),
     ok.
 
-is_app_available(Config, App, VsnRegex, Path) ->
+%% IsRaw = false means regular Erlang/OTP dependency
+%%
+%% IsRaw = true means non-Erlang/OTP dependency, e.g. the one that does not
+%% have a proper .app file
+is_app_available(Config, App, VsnRegex, Path, _IsRaw = false) ->
     ?DEBUG("is_app_available, looking for App ~p with Path ~p~n", [App, Path]),
     case rebar_app_utils:is_app_dir(Path) of
         {true, AppFile} ->
@@ -340,6 +361,19 @@ is_app_available(Config, App, VsnRegex, Path) ->
             ?WARN("Expected ~s to be an app dir (containing ebin/*.app), "
                   "but no .app found.\n", [Path]),
             {Config, {false, {missing_app_file, Path}}}
+    end;
+is_app_available(Config, App, _VsnRegex, Path, _IsRaw = true) ->
+    ?DEBUG("is_app_available, looking for Raw Depencency ~p with Path ~p~n", [App, Path]),
+    case filelib:is_dir(Path) of
+        true ->
+            %% TODO: look for version string in <Path>/VERSION file? Not clear
+            %% how to detect git/svn/hg/{cmd, ...} settings that can be passed
+            %% to rebar_utils:vcs_vsn/2 to obtain version dynamically
+            {Config, {true, Path}};
+        false ->
+            ?WARN("Expected ~s to be a raw dependency directory, "
+                  "but no directory found.\n", [Path]),
+            {Config, {false, {missing_raw_dependency_directory, Path}}}
     end.
 
 use_source(Config, Dep) ->
@@ -353,7 +387,7 @@ use_source(Config, Dep, Count) ->
         true ->
             %% Already downloaded -- verify the versioning matches the regex
             case is_app_available(Config, Dep#dep.app,
-                                  Dep#dep.vsn_regex, Dep#dep.dir) of
+                                  Dep#dep.vsn_regex, Dep#dep.dir, Dep#dep.is_raw) of
                 {Config1, {true, _}} ->
                     Dir = filename:join(Dep#dep.dir, "ebin"),
                     ok = filelib:ensure_dir(filename:join(Dir, "dummy")),