support for hex v2, multiple repository fetching, private organizations

.travis.yml

@@ -22,6 +22,7 @@ script: "./bootstrap && ./rebar3 ct"
 branches:
   only:
   - master
+  - hex_core
 cache:
   directories:
   - "$HOME/.cache/rebar3/hex/default"

bootstrap

@@ -18,7 +18,8 @@ main(_) ->
                ,{erlware_commons, ["ec_dictionary.erl", "ec_vsn.erl"]}
                ,{parse_trans, ["parse_trans.erl", "parse_trans_pp.erl",
                                "parse_trans_codegen.erl"]}
-               ,{certifi, []}],
+               ,{certifi, []}
+               ,{hex_core, []}],
     Deps = get_deps(),
     [fetch_and_compile(Dep, Deps) || Dep <- BaseDeps],
 
@@ -35,14 +36,6 @@ main(_) ->
 
     setup_env(),
     os:putenv("REBAR_PROFILE", "bootstrap"),
-    RegistryFile = default_registry_file(),
-    case filelib:is_file(RegistryFile) of
-        true ->
-            ok;
-        false ->
-            rebar3:run(["update"])
-    end,
-
     {ok, State} = rebar3:run(["compile"]),
     reset_env(),
     os:putenv("REBAR_PROFILE", ""),
@@ -56,11 +49,6 @@ main(_) ->
     %% Done with compile, can turn back on error logger
     error_logger:tty(true).
 
-default_registry_file() ->
-    {ok, [[Home]]} = init:get_argument(home),
-    CacheDir = filename:join([Home, ".cache", "rebar3"]),
-    filename:join([CacheDir, "hex", "default", "registry"]).
-
 fetch_and_compile({Name, ErlFirstFiles}, Deps) ->
     case lists:keyfind(Name, 1, Deps) of
         {Name, Vsn} ->
@@ -173,7 +161,7 @@ bootstrap_rebar3() ->
     Res = symlink_or_copy(filename:absname("src"),
                           filename:absname("_build/default/lib/rebar/src")),
     true = Res == ok orelse Res == exists,
-    Sources = ["src/rebar_resource.erl" | filelib:wildcard("src/*.erl")],
+    Sources = ["src/rebar_resource_v2.erl", "src/rebar_resource.erl" | filelib:wildcard("src/*.erl")],
     [compile_file(X, [{outdir, "_build/default/lib/rebar/ebin/"}
                      ,return | additional_defines()]) || X <- Sources],
     code:add_patha(filename:absname("_build/default/lib/rebar/ebin")).

rebar.config

@@ -11,6 +11,7 @@
         {relx,                "3.26.0"},
         {cf,                  "0.2.2"},
         {cth_readable,        "1.4.2"},
+        {hex_core,            "0.2.0"},
         {eunit_formatters,    "0.5.0"}]}.
 
 {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)",
@@ -43,7 +44,7 @@
 
 %% Profiles
 {profiles, [{test, [
-                   {deps, [{meck, "0.8.7"}]},
+                   {deps, [{meck, "0.8.12"}]},
                    {erl_opts, [debug_info, nowarn_export_all]}
                    ]
             },

rebar.lock

@@ -6,6 +6,7 @@
  {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.2.0">>},0},
  {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0},
  {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0},
+ {<<"hex_core">>,{pkg,<<"hex_core">>,<<"0.2.0">>},0},
  {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},0},
  {<<"providers">>,{pkg,<<"providers">>,<<"1.7.0">>},0},
  {<<"relx">>,{pkg,<<"relx">>,<<"3.26.0">>},0},
@@ -19,6 +20,7 @@
  {<<"erlware_commons">>, <<"2BAB99CF88941145767A502F1209886F1F0D31695EEF21978A30F15E645721E0">>},
  {<<"eunit_formatters">>, <<"6A9133943D36A465D804C1C5B6E6839030434B8879C5600D7DDB5B3BAD4CCB59">>},
  {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>},
+ {<<"hex_core">>, <<"3A7EACCFB8ADD3FF05D950C10ED5BDB5D0C48C988EBBC5D7AE2A55498F0EFF1B">>},
  {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>},
  {<<"providers">>, <<"BBF730563914328EC2511D205E6477A94831DB7297DE313B3872A2B26C562EAB">>},
  {<<"relx">>, <<"DD645ECAA1AB1647DB80D3E9BCAE0B39ED0A536EF37245F6A74B114C6D0F4E87">>},

src/rebar.app.src

@@ -30,6 +30,7 @@
                   relx,
                   cf,
                   inets,
+                  hex_core,
                   eunit_formatters]},
   {env, [
         %% Default log level
@@ -67,6 +68,7 @@
                      rebar_prv_release,
                      rebar_prv_relup,
                      rebar_prv_report,
+                     rebar_prv_repos,
                      rebar_prv_shell,
                      rebar_prv_state,
                      rebar_prv_tar,

src/rebar.hrl

@@ -25,14 +25,35 @@
 -define(CONFIG_VERSION, "1.1.0").
 -define(DEFAULT_CDN, "https://repo.hex.pm/").
 -define(REMOTE_PACKAGE_DIR, "tarballs").
--define(REMOTE_REGISTRY_FILE, "registry.ets.gz").
 -define(LOCK_FILE, "rebar.lock").
 -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative).
+-define(PACKAGE_INDEX_VERSION, 4).
+-define(PACKAGE_TABLE, package_index_v4).
+-define(INDEX_FILE, "packages-v4.idx").
+-define(HEX_AUTH_FILE, "hex.config").
+-define(PUBLIC_HEX_REPO, <<"hexpm">>).
 
--define(PACKAGE_INDEX_VERSION, 3).
--define(PACKAGE_TABLE, package_index).
--define(INDEX_FILE, "packages.idx").
--define(REGISTRY_FILE, "registry").
+%% ignore this function in all modules
+%% not every module that exports it and relies on it being called implements provider
+-ignore_xref([{format_error, 1}]).
+
+%% the package record is used in a select match spec which upsets dialyzer
+%% this is the suggested workaround from Tobias
+%% http://erlang.org/pipermail/erlang-questions/2009-February/041445.html
+-type ms_field() :: '$1' | '_'.
+
+%% TODO: change package and requirement keys to be required (:=) after dropping support for OTP-18
+-record(package, {key :: {unicode:unicode_binary() | ms_field(), unicode:unicode_binary() | ms_field(),
+                          unicode:unicode_binary() | ms_field()},
+                  checksum :: binary() | ms_field(),
+                  retired :: boolean() | ms_field(),
+                  dependencies :: [#{package => unicode:unicode_binary(),
+                                     requirement => unicode:unicode_binary()}] | ms_field()}).
+
+-record(resource, {type :: atom(),
+                   module :: module(),
+                   state :: term(),
+                   implementation :: rebar_resource | rebar_resource_v2}).
 
 -ifdef(namespaced_types).
 -type rebar_dict() :: dict:dict().

src/rebar3.erl

@@ -103,7 +103,7 @@ run(RawArgs) ->
     case erlang:system_info(version) of
         "6.1" ->
             ?WARN("Due to a filelib bug in Erlang 17.1 it is recommended"
-                 "you update to a newer release.", []);
+                  "you update to a newer release.", []);
         _ ->
             ok
     end,
@@ -139,8 +139,14 @@ run_aux(State, RawArgs) ->
                      rebar_state:set(State1, rebar_packages_cdn, CDN)
              end,
 
+    %% TODO: this means use of REBAR_PROFILE=profile will replace the repos with
+    %% the repos defined in the profile. But it will not work with `as profile`.
+    %% Maybe it shouldn't work with either to be consistent?
+    Resources = application:get_env(rebar, resources, []),
+    State2_ = rebar_state:create_resources(Resources, State2),
+
     %% bootstrap test profile
-    State3 = rebar_state:add_to_profile(State2, test, test_state(State1)),
+    State3 = rebar_state:add_to_profile(State2_, test, test_state(State1)),
 
     %% Process each command, resetting any state between each one
     BaseDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR),
@@ -375,7 +381,11 @@ state_from_global_config(Config, GlobalConfigFile) ->
 
     %% We don't want to worry about global plugin install state effecting later
     %% usage. So we throw away the global profile state used for plugin install.
-    GlobalConfigThrowAway = rebar_state:current_profiles(GlobalConfig, [global]),
+    GlobalConfigThrowAway0 = rebar_state:current_profiles(GlobalConfig, [global]),
+
+    Resources = application:get_env(rebar, resources, []),
+    GlobalConfigThrowAway = rebar_state:create_resources(Resources, GlobalConfigThrowAway0),
+
     GlobalState = case rebar_state:get(GlobalConfigThrowAway, plugins, []) of
                       [] ->
                           GlobalConfigThrowAway;
@@ -386,7 +396,8 @@ state_from_global_config(Config, GlobalConfigFile) ->
                   end,
     GlobalPlugins = rebar_state:providers(GlobalState),
     GlobalConfig2 = rebar_state:set(GlobalConfig, plugins, []),
-    GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global}, rebar_state:get(GlobalConfigThrowAway, plugins, [])),
+    GlobalConfig3 = rebar_state:set(GlobalConfig2, {plugins, global},
+                                    rebar_state:get(GlobalConfigThrowAway, plugins, [])),
     rebar_state:providers(rebar_state:new(GlobalConfig3, Config), GlobalPlugins).
 
 -spec test_state(rebar_state:t()) -> [{'extra_src_dirs',[string()]} | {'erl_opts',[any()]}].

src/rebar_api.erl

@@ -88,4 +88,4 @@ processing_base_dir(State) ->
 %% its configuration, including for validation of certs.
 -spec ssl_opts(string()) -> [term()].
 ssl_opts(Url) ->
-    rebar_pkg_resource:ssl_opts(Url).
+    rebar_utils:ssl_opts(Url).

src/rebar_app_discover.erl

@@ -9,8 +9,7 @@
          find_apps/2,
          find_apps/3,
          find_app/2,
-         find_app/3,
-         find_app/4]).
+         find_app/3]).
 
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
@@ -95,7 +94,7 @@ format_error({missing_module, Module}) ->
 merge_deps(AppInfo, State) ->
     %% These steps make sure that hooks and artifacts are run in the context of
     %% the application they are defined at. If an umbrella structure is used and
-    %% they are deifned at the top level they will instead run in the context of
+    %% they are defined at the top level they will instead run in the context of
     %% the State and at the top level, not as part of an application.
     CurrentProfiles = rebar_state:current_profiles(State),
     Default = reset_hooks(rebar_state:default(State), CurrentProfiles),
@@ -205,7 +204,7 @@ reset_hooks(Opts, CurrentProfiles) ->
 -spec all_app_dirs([file:name()]) -> [{file:name(), [file:name()]}].
 all_app_dirs(LibDirs) ->
     lists:flatmap(fun(LibDir) ->
-                        SrcDirs = find_config_src(LibDir, ["src"]),
+                        {_, SrcDirs} = find_config_src(LibDir, ["src"]),
                         app_dirs(LibDir, SrcDirs)
                   end, LibDirs).
 
@@ -278,8 +277,9 @@ find_apps(LibDirs, SrcDirs, Validate) ->
 %% app info record.
 -spec find_app(file:filename_all(), valid | invalid | all) -> {true, rebar_app_info:t()} | false.
 find_app(AppDir, Validate) ->
-    SrcDirs = find_config_src(AppDir, ["src"]),
-    find_app(rebar_app_info:new(), AppDir, SrcDirs, Validate).
+    {Config, SrcDirs} = find_config_src(AppDir, ["src"]),
+    AppInfo = rebar_app_info:update_opts(rebar_app_info:new(), dict:new(), Config),
+    find_app_(AppInfo, AppDir, SrcDirs, Validate).
 
 %% @doc check that a given app in a directory is there, and whether it's
 %% valid or not based on the second argument. Returns the related
@@ -291,7 +291,7 @@ find_app(AppInfo, AppDir, Validate) ->
     %% of src/
     AppOpts = rebar_app_info:opts(AppInfo),
     SrcDirs = rebar_dir:src_dirs(AppOpts, ["src"]),
-    find_app(AppInfo, AppDir, SrcDirs, Validate).
+    find_app_(AppInfo, AppDir, SrcDirs, Validate).
 
 %% @doc check that a given app in a directory is there, and whether it's
 %% valid or not based on the second argument. The third argument includes
@@ -301,6 +301,14 @@ find_app(AppInfo, AppDir, Validate) ->
                [file:filename_all()], valid | invalid | all) ->
     {true, rebar_app_info:t()} | false.
 find_app(AppInfo, AppDir, SrcDirs, Validate) ->
+    Config = rebar_config:consult(AppDir),
+    AppInfo1 = rebar_app_info:update_opts(AppInfo, rebar_app_info:opts(AppInfo), Config),
+    find_app_(AppInfo1, AppDir, SrcDirs, Validate).
+
+-spec find_app_(rebar_app_info:t(), file:filename_all(),
+               [file:filename_all()], valid | invalid | all) ->
+    {true, rebar_app_info:t()} | false.
+find_app_(AppInfo, AppDir, SrcDirs, Validate) ->
     AppFile = filelib:wildcard(filename:join([AppDir, "ebin", "*.app"])),
     AppSrcFile = lists:append(
         [filelib:wildcard(filename:join([AppDir, SrcDir, "*.app.src"]))
@@ -331,17 +339,14 @@ create_app_info(AppInfo, AppDir, AppFile) ->
     AppInfo2 = rebar_app_info:applications(
                  rebar_app_info:app_details(AppInfo1, AppDetails),
                  IncludedApplications++Applications),
-    C = rebar_config:consult(AppDir),
-    AppInfo3 = rebar_app_info:update_opts(AppInfo2,
-                                          rebar_app_info:opts(AppInfo2), C),
-    Valid = case rebar_app_utils:validate_application_info(AppInfo3) =:= true
-                andalso rebar_app_info:has_all_artifacts(AppInfo3) =:= true of
+    Valid = case rebar_app_utils:validate_application_info(AppInfo2) =:= true
+                andalso rebar_app_info:has_all_artifacts(AppInfo2) =:= true of
                 true ->
                     true;
                 _ ->
                     false
             end,
-    rebar_app_info:dir(rebar_app_info:valid(AppInfo3, Valid), AppDir).
+    rebar_app_info:dir(rebar_app_info:valid(AppInfo2, Valid), AppDir).
 
 %% @doc Read in and parse the .app file if it is availabe. Do the same for
 %% the .app.src file if it exists.
@@ -403,12 +408,20 @@ try_handle_app_file(_AppInfo, Other, _AppDir, _AppSrcFile, _, _Validate) ->
       AppFile :: file:filename(),
       AppDir :: file:filename(),
       AppSrcFile :: file:filename().
-try_handle_app_src_file(_AppInfo, _, _AppDir, [], _Validate) ->
-    false;
+try_handle_app_src_file(AppInfo, _, _AppDir, [], _Validate) ->
+    %% if .app and .app.src are not found check for a mix config file
+    %% it is assumed a plugin will build the application, including
+    %% a .app after this step
+    case filelib:is_file(filename:join(rebar_app_info:dir(AppInfo), "mix.exs")) of
+        true ->
+            {true, AppInfo};
+        false ->
+            false
+    end;
 try_handle_app_src_file(_AppInfo, _, _AppDir, _AppSrcFile, valid) ->
     false;
 try_handle_app_src_file(AppInfo, _, AppDir, [File], Validate) when Validate =:= invalid
-                                                                 ; Validate =:= all ->
+                                                                   ; Validate =:= all ->
     AppInfo1 = rebar_app_info:app_file(AppInfo, undefined),
     AppInfo2 = create_app_info(AppInfo1, AppDir, File),
     case filename:extension(File) of
@@ -437,8 +450,8 @@ to_atom(Bin) ->
 find_config_src(AppDir, Default) ->
     case rebar_config:consult(AppDir) of
         [] ->
-            Default;
+            {[], Default};
         Terms ->
             %% TODO: handle profiles I guess, but we don't have that info
-            proplists:get_value(src_dirs, Terms, Default)
+            {Terms, proplists:get_value(src_dirs, Terms, Default)}
     end.