diff --git a/rebar.lock b/rebar.lock index 40488af3c..3da606bae 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,4 +1,4 @@ -{"1.1.0", +{"1.2.0", [{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.8.0">>},0}, {<<"certifi">>,{pkg,<<"certifi">>,<<"2.5.1">>},0}, {<<"cf">>,{pkg,<<"cf">>,<<"0.2.2">>},0}, @@ -22,5 +22,17 @@ {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>}, {<<"providers">>, <<"70B4197869514344A8A60E2B2A4EF41CA03DEF43CFB1712ECF076A0F3C62F083">>}, {<<"relx">>, <<"AFC019320BB69881718576B3E4E1EB548C1FA3270717BA66A78004C98A77CD17">>}, - {<<"ssl_verify_fun">>, <<"6EAF7AD16CB568BB01753DBBD7A95FF8B91C7979482B95F38443FE2C8852A79B">>}]} + {<<"ssl_verify_fun">>, <<"6EAF7AD16CB568BB01753DBBD7A95FF8B91C7979482B95F38443FE2C8852A79B">>}]}, +{pkg_hash_ext,[ + {<<"bbmustache">>, <<"190EA2206128BDFABF5D9200B8DF97F6511D9C62953655828E28C2BC79161252">>}, + {<<"certifi">>, <<"805ABD97539CAF89EC6D4732C91E62BA9DA0CDA51AC462380BBD28EE697A8C42">>}, + {<<"cf">>, <<"48283B3019BC7FAD56E7B23028A5DA4D3E6CD598A553AB2A99A2153BF5F19B21">>}, + {<<"cth_readable">>, <<"8F799D7BFD444ABFE2B4C07CCB31C56E80043335E1ED3933A2DFBA35F8D97523">>}, + {<<"erlware_commons">>, <<"7AADA93F368D0A0430122E39931B7FB4AC9E94DBF043CDC980AD4330FD9CD166">>}, + {<<"eunit_formatters">>, <<"D6C8BA213424944E6E05BBC097C32001CDD0ABE3925D02454F229B20D68763C9">>}, + {<<"getopt">>, <<"53E1AB83B9CEB65C9672D3E7A35B8092E9BDC9B3EE80721471A161C10C59959C">>}, + {<<"parse_trans">>, <<"17EF63ABDE837AD30680EA7F857DD9E7CED9476CDD7B0394432AF4BFC241B960">>}, + {<<"providers">>, <<"E45745ADE9C476A9A469EA0840E418AB19360DC44F01A233304E118A44486BA0">>}, + {<<"relx">>, <<"6E0456139FC70BADE0C45FF8A8197C5E879A57FD792F771FC632B94C5AEC1EAC">>}, + {<<"ssl_verify_fun">>, <<"13104D7897E38ED7F044C4DE953A6C28597D1C952075EB2E328BC6D6F2BFC496">>}]} ]. diff --git a/src/r3_hex_api.erl b/src/r3_hex_api.erl index a19472f1e..ff1167e2c 100644 --- a/src/r3_hex_api.erl +++ b/src/r3_hex_api.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually %% @hidden @@ -16,6 +16,11 @@ ]). -define(ERL_CONTENT_TYPE, <<"application/vnd.hex+erlang">>). +-export_type([body/0, response/0]). + +-type response() :: {ok, {r3_hex_http:status(), r3_hex_http:headers(), body() | nil}} | {error, term()}. +-type body() :: [body()] | #{binary() => body() | binary()}. + get(Config, Path) -> request(Config, get, Path, undefined). @@ -28,26 +33,20 @@ put(Config, Path, Body) -> delete(Config, Path) -> request(Config, delete, Path, undefined). --ifdef (OTP_RELEASE). - -if(?OTP_RELEASE >= 23). - -compile({nowarn_deprecated_function, [{http_uri, encode, 1}]}). - -endif. --endif. - %% @private encode_query_string(List) -> - QueryString = - join("&", - lists:map(fun - ({K, V}) when is_atom(V) -> - atom_to_list(K) ++ "=" ++ atom_to_list(V); - ({K, V}) when is_binary(V) -> - atom_to_list(K) ++ "=" ++ binary_to_list(V); - ({K, V}) when is_integer(V) -> - atom_to_list(K) ++ "=" ++ integer_to_list(V) - end, List)), - Encoded = http_uri:encode(QueryString), - list_to_binary(Encoded). + Pairs = lists:map(fun ({K, V}) -> {to_list(K), to_list(V)} end, List), + list_to_binary(compose_query(Pairs)). + +%% OTP 21+ +-ifdef (OTP_RELEASE). +compose_query(Pairs) -> + uri_string:compose_query(Pairs). +-else. +compose_query(Pairs) -> + String = join("&", lists:map(fun ({K, V}) -> K ++ "=" ++ V end, Pairs)), + http_uri:encode(String). +-endif. %% @private build_repository_path(#{api_repository := Repo}, Path) when is_binary(Repo) -> @@ -63,7 +62,24 @@ build_organization_path(#{api_organization := undefined}, Path) -> %% @private join_path_segments(Segments) -> - erlang:iolist_to_binary(join(<<"/">>, lists:map(fun encode/1, Segments))). + iolist_to_binary(recompose(Segments)). + +%% OTP 21+ +-ifdef (OTP_RELEASE). +recompose(Segments) -> + Concatenated = join(<<"/">>, Segments), + %% uri_string:recompose/1 accepts path segments as a list, + %% both strings and binaries + uri_string:recompose(#{path => Concatenated}). +-else. +recompose(Segments) -> + join(<<"/">>, lists:map(fun encode_segment/1, Segments)). + +encode_segment(Binary) when is_binary(Binary) -> + encode_segment(binary_to_list(Binary)); +encode_segment(String) when is_list(String) -> + http_uri:encode(String). +-endif. %%==================================================================== %% Internal functions @@ -78,25 +94,20 @@ request(Config, Method, Path, Body) when is_binary(Path) and is_map(Config) -> ReqHeaders2 = put_new(<<"accept">>, ?ERL_CONTENT_TYPE, ReqHeaders), case r3_hex_http:request(Config, Method, build_url(Path, Config), ReqHeaders2, Body) of - {ok, {Status, RespHeaders, RespBody}} = Response -> + {ok, {Status, RespHeaders, RespBody}} -> ContentType = maps:get(<<"content-type">>, RespHeaders, <<"">>), case binary:match(ContentType, ?ERL_CONTENT_TYPE) of {_, _} -> {ok, {Status, RespHeaders, binary_to_term(RespBody)}}; nomatch -> - Response + {ok, {Status, RespHeaders, nil}} end; Other -> Other end. -encode(Binary) when is_binary(Binary) -> - encode(binary_to_list(Binary)); -encode(String) when is_list(String) -> - http_uri:encode(String). - build_url(Path, #{api_url := URI}) -> <>. @@ -124,3 +135,8 @@ join(Sep, [H|T]) -> [H|join_prepend(Sep, T)]. join_prepend(_Sep, []) -> []; join_prepend(Sep, [H|T]) -> [Sep,H|join_prepend(Sep,T)]. + +to_list(A) when is_atom(A) -> atom_to_list(A); +to_list(B) when is_binary(B) -> unicode:characters_to_list(B); +to_list(I) when is_integer(I) -> integer_to_list(I); +to_list(Str) -> unicode:characters_to_list(Str). diff --git a/src/r3_hex_api_key.erl b/src/r3_hex_api_key.erl index ff4e62be7..3fb04c637 100644 --- a/src/r3_hex_api_key.erl +++ b/src/r3_hex_api_key.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually -module(r3_hex_api_key). -export([ @@ -9,23 +9,151 @@ delete_all/1 ]). +-export_type([permission/0]). + +-type permission() :: api_permission() | repo_permission() | repos_permission(). +-ifdef(OTP_19). +-type api_permission() :: #{domain := api, resource => read | write}. +-type repo_permission() :: #{domain := repository, resource := binary()}. +-type repos_permission() :: #{domain := repositories}. +-else. +-type api_permission() :: #{domain => api, resource => read | write}. +-type repo_permission() :: #{domain => repository, resource => binary()}. +-type repos_permission() :: #{domain => repositories}. +-endif. + +%% @doc +%% Lists the user's or organization's API and repository keys. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_key:list(r3_hex_core:default_config()). +%% {ok, {200, ..., [#{ +%% <<"authing_key">> => true, +%% <<"inserted_at">> => <<"2019-02-27T11:15:32Z">>, +%% <<"last_use">> => +%% #{<<"ip">> => <<"1.2.3.4">>, +%% <<"used_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"user_agent">> => <<"hex_core/0.5.0 (httpc) (OTP/21) (erts/10.2)">>}, +%% <<"name">> => <<"hex_core">>, +%% <<"permissions">> => [#{<<"domain">> => <<"api">>,<<"resource">> => <<"read">>}], +%% <<"revoked_at">> => nil, +%% <<"updated_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"url">> => <<"https://hex.pm/api/keys/test">>}, +%% }]}} +%% ''' +%% @end +-spec list(r3_hex_core:config()) -> r3_hex_api:response(). list(Config) when is_map(Config) -> Path = r3_hex_api:build_organization_path(Config, ["keys"]), r3_hex_api:get(Config, Path). -get(Config, Name) when is_map(Config) -> +%% @doc +%% Gets an API or repository key by name. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_key:get(r3_hex_core:default_config(), <<"test">>). +%% {ok, {200, ..., #{ +%% <<"authing_key">> => true, +%% <<"inserted_at">> => <<"2019-02-27T11:15:32Z">>, +%% <<"last_use">> => +%% #{<<"ip">> => <<"1.2.3.4">>, +%% <<"used_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"user_agent">> => <<"hex_core/0.5.0 (httpc) (OTP/21) (erts/10.2)">>}, +%% <<"name">> => <<"hex_core">>, +%% <<"permissions">> => [#{<<"domain">> => <<"api">>,<<"resource">> => <<"read">>}], +%% <<"revoked_at">> => nil, +%% <<"updated_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"url">> => <<"https://hex.pm/api/keys/test">>}, +%% }}} +%% ''' +%% @end +-spec get(r3_hex_core:config(), binary()) -> r3_hex_api:response(). +get(Config, Name) when is_map(Config) and is_binary(Name) -> Path = r3_hex_api:build_organization_path(Config, ["keys", Name]), r3_hex_api:get(Config, Path). -add(Config, Name, Permissions) when is_map(Config) -> +%% @doc +%% Adds a new API or repository key. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_key:add(r3_hex_core:default_config(), <<"test">>, [...]). +%% {ok, {200, ..., #{ +%% <<"authing_key">> => true, +%% <<"inserted_at">> => <<"2019-02-27T11:15:32Z">>, +%% <<"last_use">> => +%% #{<<"ip">> => <<"1.2.3.4">>, +%% <<"used_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"user_agent">> => <<"hex_core/0.5.0 (httpc) (OTP/21) (erts/10.2)">>}, +%% <<"name">> => <<"hex_core">>, +%% <<"permissions">> => [#{<<"domain">> => <<"api">>,<<"resource">> => <<"read">>}], +%% <<"revoked_at">> => nil, +%% <<"updated_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"url">> => <<"https://hex.pm/api/keys/test">>}, +%% }}} +%% ''' +%% @end +-spec add(r3_hex_core:config(), binary(), [permission()]) -> r3_hex_api:response(). +add(Config, Name, Permissions) when is_map(Config) and is_binary(Name) and is_list(Permissions) -> Path = r3_hex_api:build_organization_path(Config, ["keys"]), Params = #{<<"name">> => Name, <<"permissions">> => Permissions}, r3_hex_api:post(Config, Path, Params). -delete(Config, Name) when is_map(Config) -> +%% @doc +%% Deletes an API or repository key. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_key:delete(r3_hex_core:default_config(), <<"test">>). +%% {ok, {200, ..., #{ +%% <<"authing_key">> => true, +%% <<"inserted_at">> => <<"2019-02-27T11:15:32Z">>, +%% <<"last_use">> => +%% #{<<"ip">> => <<"1.2.3.4">>, +%% <<"used_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"user_agent">> => <<"hex_core/0.5.0 (httpc) (OTP/21) (erts/10.2)">>}, +%% <<"name">> => <<"hex_core">>, +%% <<"permissions">> => [#{<<"domain">> => <<"api">>,<<"resource">> => <<"read">>}], +%% <<"revoked_at">> => nil, +%% <<"updated_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"url">> => <<"https://hex.pm/api/keys/test">>}, +%% }}} +%% ''' +%% @end +-spec delete(r3_hex_core:config(), binary()) -> r3_hex_api:response(). +delete(Config, Name) when is_map(Config) and is_binary(Name) -> Path = r3_hex_api:build_organization_path(Config, ["keys", Name]), r3_hex_api:delete(Config, Path). +%% @doc +%% Deletes all API and repository keys associated with the account. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_key:delete_all(r3_hex_core:default_config()). +%% {ok, {200, ..., [#{ +%% <<"authing_key">> => true, +%% <<"inserted_at">> => <<"2019-02-27T11:15:32Z">>, +%% <<"last_use">> => +%% #{<<"ip">> => <<"1.2.3.4">>, +%% <<"used_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"user_agent">> => <<"hex_core/0.5.0 (httpc) (OTP/21) (erts/10.2)">>}, +%% <<"name">> => <<"hex_core">>, +%% <<"permissions">> => [#{<<"domain">> => <<"api">>,<<"resource">> => <<"read">>}], +%% <<"revoked_at">> => nil, +%% <<"updated_at">> => <<"2019-02-27T14:38:54Z">>, +%% <<"url">> => <<"https://hex.pm/api/keys/test">>}, +%% }]}} +%% ''' +%% @end +-spec delete_all(r3_hex_core:config()) -> r3_hex_api:response(). delete_all(Config) when is_map(Config) -> Path = r3_hex_api:build_organization_path(Config, ["keys"]), r3_hex_api:delete(Config, Path). diff --git a/src/r3_hex_api_package.erl b/src/r3_hex_api_package.erl index 0a4503517..97c41a067 100644 --- a/src/r3_hex_api_package.erl +++ b/src/r3_hex_api_package.erl @@ -1,10 +1,10 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually -module(r3_hex_api_package). -export([get/2, search/3]). %% @doc -%% Gets package. +%% Gets a package. %% %% Examples: %% @@ -26,7 +26,8 @@ %% ]}}} %% ''' %% @end -get(Config, Name) when is_binary(Name) and is_map(Config) -> +-spec get(r3_hex_core:config(), binary()) -> r3_hex_api:response(). +get(Config, Name) when is_map(Config) and is_binary(Name)-> Path = r3_hex_api:build_repository_path(Config, ["packages", Name]), r3_hex_api:get(Config, Path). @@ -42,7 +43,8 @@ get(Config, Name) when is_binary(Name) and is_map(Config) -> %% ... %% ]}} %% ''' -search(Config, Query, SearchParams) when is_binary(Query) and is_list(SearchParams) and is_map(Config) -> +-spec search(r3_hex_core:config(), binary(), list(binary())) -> r3_hex_api:response(). +search(Config, Query, SearchParams) when is_map(Config) and is_binary(Query) and is_list(SearchParams) -> QueryString = r3_hex_api:encode_query_string([{search, Query} | SearchParams]), Path = r3_hex_api:join_path_segments(r3_hex_api:build_repository_path(Config, ["packages"])), PathQuery = <>, diff --git a/src/r3_hex_api_package_owner.erl b/src/r3_hex_api_package_owner.erl index f74364ee9..b85ed45ef 100644 --- a/src/r3_hex_api_package_owner.erl +++ b/src/r3_hex_api_package_owner.erl @@ -1,34 +1,98 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually -module(r3_hex_api_package_owner). -export([ - add/3, + add/5, delete/3, get/3, list/2 ]). +%% @doc +%% Lists the packages owners. +%% %% Examples: %% %% ``` -%% > r3_hex_api_owner:list(r3_hex_core:default_config(), <<"package">>). -%% {ok, {200, ..., [ -%% #{<<"username">> => <<"alice">>, ...}, -%% ... -%% ]}} +%% > r3_hex_api_package_owner:list(r3_hex_core:default_config(), <<"package">>). +%% {ok, {200, ..., [#{ +%% <<"email">> => <<"user@example.com">>, +%% <<"full_name">> => <<"John Doe">>, +%% <<"handles">> => #{...}, +%% <<"inserted_at">> => <<"2014-04-21T17:20:12Z">>, +%% <<"level">> => <<"full">>, +%% <<"updated_at">> => <<"2019-08-04T19:28:05Z">>, +%% <<"url">> => <<"https://hex.pm/api/users/user">>, +%% <<"username">> => <<"user">> +%% }]}} %% ''' -list(Config, PackageName) when is_binary(PackageName) and is_map(Config) -> +%% @end +-spec list(r3_hex_core:config(), binary()) -> r3_hex_api:response(). +list(Config, PackageName) when is_binary(PackageName) -> Path = r3_hex_api:build_repository_path(Config, ["packages", PackageName, "owners"]), r3_hex_api:get(Config, Path). -get(Config, PackageName, UsernameOrEmail) when is_binary(PackageName) and is_map(Config) -> +%% @doc +%% Gets a packages owner. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_package_owner:get(r3_hex_core:default_config(), <<"package">>, <<"user">>). +%% {ok, {200, ..., #{ +%% <<"email">> => <<"user@example.com">>, +%% <<"full_name">> => <<"John Doe">>, +%% <<"handles">> => #{...}, +%% <<"inserted_at">> => <<"2014-04-21T17:20:12Z">>, +%% <<"level">> => <<"full">>, +%% <<"updated_at">> => <<"2019-08-04T19:28:05Z">>, +%% <<"url">> => <<"https://hex.pm/api/users/user">>, +%% <<"username">> => <<"user">> +%% }}} +%% ''' +%% @end +-spec get(r3_hex_core:config(), binary(), binary()) -> r3_hex_api:response(). +get(Config, PackageName, UsernameOrEmail) when is_map(Config) and is_binary(PackageName) and is_binary(UsernameOrEmail) -> Path = r3_hex_api:build_repository_path(Config, ["packages", PackageName, "owners", UsernameOrEmail]), r3_hex_api:get(Config, Path). -add(Config, PackageName, UsernameOrEmail) when is_binary(PackageName) and is_map(Config) -> +%% @doc +%% Adds a packages owner. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_package_owner:add(r3_hex_core:default_config(), <<"package">>, <<"user">>, <<"full">>, false). +%% {ok, {200, ..., #{ +%% <<"email">> => <<"user@example.com">>, +%% <<"full_name">> => <<"John Doe">>, +%% <<"handles">> => #{...}, +%% <<"inserted_at">> => <<"2014-04-21T17:20:12Z">>, +%% <<"level">> => <<"full">>, +%% <<"updated_at">> => <<"2019-08-04T19:28:05Z">>, +%% <<"url">> => <<"https://hex.pm/api/users/user">>, +%% <<"username">> => <<"user">> +%% }}} +%% ''' +%% @end +-spec add(r3_hex_core:config(), binary(), binary(), binary(), boolean()) -> r3_hex_api:response(). +add(Config, PackageName, UsernameOrEmail, Level, Transfer) +when is_binary(PackageName) and is_binary(UsernameOrEmail) and is_map(Config) and is_binary(Level) and is_boolean(Transfer) -> Path = r3_hex_api:build_repository_path(Config, ["packages", PackageName, "owners", UsernameOrEmail]), - r3_hex_api:put(Config, Path, #{}). + r3_hex_api:put(Config, Path, #{<<"level">> => Level, <<"transfer">> => Transfer}). -delete(Config, PackageName, UsernameOrEmail) when is_binary(PackageName) and is_map(Config) -> + +%% @doc +%% Deletes a packages owner. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_package_owner:delete(r3_hex_core:default_config(), <<"package">>, <<"user">>). +%% {ok, {204, ..., nil}} +%% ''' +%% @end +-spec delete(r3_hex_core:config(), binary(), binary()) -> r3_hex_api:response(). +delete(Config, PackageName, UsernameOrEmail) when is_map(Config) and is_binary(PackageName) and is_binary(UsernameOrEmail) -> Path = r3_hex_api:build_repository_path(Config, ["packages", PackageName, "owners", UsernameOrEmail]), r3_hex_api:delete(Config, Path). diff --git a/src/r3_hex_api_release.erl b/src/r3_hex_api_release.erl index 0be62b7ec..21cf8b9fe 100644 --- a/src/r3_hex_api_release.erl +++ b/src/r3_hex_api_release.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually -module(r3_hex_api_release). -export([ @@ -9,44 +9,120 @@ unretire/3 ]). +-export_type([retirement_params/0, retirement_reason/0]). + +-type retirement_reason() :: other | invalid | security | deprecated | renamed. + +-ifdef(OTP_19). +-type retirement_params() :: #{reason := retirement_reason(), message => binary()}. +-else. +-type retirement_params() :: #{reason => retirement_reason(), message => binary()}. +-endif. %% @doc -%% Gets package release. +%% Gets a package release. %% %% Examples: %% %% ``` -%% > r3_hex_api:get_release(<<"package">>, <<"1.0.0">>, r3_hex_core:default_config()). +%% > r3_hex_api_release:get(r3_hex_core:default_config(), <<"package">>, <<"1.0.0">>). %% {ok, {200, ..., #{ -%% <<"version">> => <<"1.0.0">>, -%% <<"meta">> => #{ -%% <<"description">> => ..., -%% <<"licenses">> => ..., -%% <<"links">> => ..., -%% <<"maintainers">> => ... -%% }, -%% ...}}} +%% <<"checksum">> => <<"540d210d81f56f17f64309a4896430e727972499b37bd59342dc08d61dff74d8">>, +%% <<"docs_html_url">> => <<"https://hexdocs.pm/package/1.0.0/">>, +%% <<"downloads">> => 740,<<"has_docs">> => true, +%% <<"html_url">> => <<"https://hex.pm/packages/package/1.0.0">>, +%% <<"inserted_at">> => <<"2014-12-09T18:32:03Z">>, +%% <<"meta">> => +%% #{<<"app">> => <<"package">>, +%% <<"build_tools">> => [<<"mix">>]}, +%% <<"package_url">> => <<"https://hex.pm/api/packages/package">>, +%% <<"publisher">> => nil,<<"requirements">> => #{}, +%% <<"retirement">> => nil, +%% <<"updated_at">> => <<"2019-07-28T21:12:11Z">>, +%% <<"url">> => <<"https://hex.pm/api/packages/package/releases/1.0.0">>, +%% <<"version">> => <<"1.0.0">> +%% }}} %% ''' %% @end -get(Config, Name, Version) when is_binary(Name) and is_binary(Version) and is_map(Config) -> +-spec get(r3_hex_core:config(), binary(), binary()) -> r3_hex_api:response(). +get(Config, Name, Version) when is_map(Config) and is_binary(Name) and is_binary(Version) -> Path = r3_hex_api:build_repository_path(Config, ["packages", Name, "releases", Version]), r3_hex_api:get(Config, Path). -publish(Config, Tarball) when is_binary(Tarball) and is_map(Config) -> +%% @doc +%% Publishes a new package release. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_release:publish(r3_hex_core:default_config(), Tarball). +%% {ok, {200, ..., #{ +%% <<"checksum">> => <<"540d210d81f56f17f64309a4896430e727972499b37bd59342dc08d61dff74d8">>, +%% <<"docs_html_url">> => <<"https://hexdocs.pm/package/1.0.0/">>, +%% <<"downloads">> => 740,<<"has_docs">> => true, +%% <<"html_url">> => <<"https://hex.pm/packages/package/1.0.0">>, +%% <<"inserted_at">> => <<"2014-12-09T18:32:03Z">>, +%% <<"meta">> => +%% #{<<"app">> => <<"package">>, +%% <<"build_tools">> => [<<"mix">>]}, +%% <<"package_url">> => <<"https://hex.pm/api/packages/package">>, +%% <<"publisher">> => nil,<<"requirements">> => #{}, +%% <<"retirement">> => nil, +%% <<"updated_at">> => <<"2019-07-28T21:12:11Z">>, +%% <<"url">> => <<"https://hex.pm/api/packages/package/releases/1.0.0">>, +%% <<"version">> => <<"1.0.0">> +%% }}} +%% ''' +%% @end +-spec publish(r3_hex_core:config(), binary()) -> r3_hex_api:response(). +publish(Config, Tarball) when is_map(Config) and is_binary(Tarball) -> Path = r3_hex_api:build_repository_path(Config, ["publish"]), TarballContentType = "application/octet-stream", Config2 = put_header(<<"content-length">>, integer_to_binary(byte_size(Tarball)), Config), Body = {TarballContentType, Tarball}, r3_hex_api:post(Config2, Path, Body). -delete(Config, Name, Version) when is_binary(Name) and is_binary(Version) and is_map(Config) -> +%% @doc +%% Deletes a package release. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_release:delete(r3_hex_core:default_config(), <<"package">>, <<"1.0.0">>). +%% {ok, {204, ..., nil}} +%% ''' +%% @end +-spec delete(r3_hex_core:config(), binary(), binary()) -> r3_hex_api:response(). +delete(Config, Name, Version) when is_map(Config) and is_binary(Name) and is_binary(Version) -> Path = r3_hex_api:build_repository_path(Config, ["packages", Name, "releases", Version]), r3_hex_api:delete(Config, Path). -retire(Config, Name, Version, Params) when is_binary(Name) and is_binary(Version) and is_map(Config) -> +%% @doc +%% Retires a package release. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_release:retire(r3_hex_core:default_config(), <<"package">>, <<"1.0.0">>, Params). +%% {ok, {204, ..., nil}} +%% ''' +%% @end +-spec retire(r3_hex_core:config(), binary(), binary(), retirement_params()) -> r3_hex_api:response(). +retire(Config, Name, Version, Params) when is_map(Config) and is_binary(Name) and is_binary(Version) -> Path = r3_hex_api:build_repository_path(Config, ["packages", Name, "releases", Version, "retire"]), r3_hex_api:post(Config, Path, Params). -unretire(Config, Name, Version) when is_binary(Name) and is_binary(Version) and is_map(Config) -> +%% @doc +%% Unretires a package release. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_release:unretire(r3_hex_core:default_config(), <<"package">>, <<"1.0.0">>). +%% {ok, {204, ..., nil}} +%% ''' +%% @end +-spec unretire(r3_hex_core:config(), binary(), binary()) -> r3_hex_api:response(). +unretire(Config, Name, Version) when is_map(Config) and is_binary(Name) and is_binary(Version) -> Path = r3_hex_api:build_repository_path(Config, ["packages", Name, "releases", Version, "retire"]), r3_hex_api:delete(Config, Path). diff --git a/src/r3_hex_api_user.erl b/src/r3_hex_api_user.erl index 97d225f33..7d6d0471f 100644 --- a/src/r3_hex_api_user.erl +++ b/src/r3_hex_api_user.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually -module(r3_hex_api_user). -export([ @@ -8,10 +8,51 @@ reset_password/2 ]). +%% @doc +%% Gets the authenticated user. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_user:me(r3_hex_core:default_config()). +%% {ok, {200, ..., #{ +%% <<"email">> => <<"user@example.com">>, +%% <<"full_name">> => <<"John Doe">>, +%% <<"handles">> => #{...}, +%% <<"inserted_at">> => <<"2014-04-21T17:20:12Z">>, +%% <<"level">> => <<"full">>, +%% <<"updated_at">> => <<"2019-08-04T19:28:05Z">>, +%% <<"url">> => <<"https://hex.pm/api/users/user">>, +%% <<"username">> => <<"user">> +%% }}} +%% ''' +%% @end +-spec me(r3_hex_core:config()) -> r3_hex_api:response(). me(Config) when is_map(Config) -> r3_hex_api:get(Config, ["users", "me"]). -create(Config, Username, Password, Email) -> +%% @doc +%% Creates a new user account. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_user:create(r3_hex_core:default_config(), <<"user">>, <<"hunter42">>, <<"user@example.com">>). +%% {ok, {201, ..., #{ +%% <<"email">> => <<"user@example.com">>, +%% <<"full_name">> => <<"John Doe">>, +%% <<"handles">> => #{...}, +%% <<"inserted_at">> => <<"2014-04-21T17:20:12Z">>, +%% <<"level">> => <<"full">>, +%% <<"updated_at">> => <<"2019-08-04T19:28:05Z">>, +%% <<"url">> => <<"https://hex.pm/api/users/user">>, +%% <<"username">> => <<"user">> +%% }}} +%% ''' +%% @end +-spec create(r3_hex_core:config(), binary(), binary(), binary()) -> r3_hex_api:response(). +create(Config, Username, Password, Email) +when is_map(Config) and is_binary(Username) and is_binary(Password) and is_binary(Email) -> Params = #{ <<"username">> => Username, <<"password">> => Password, @@ -19,28 +60,39 @@ create(Config, Username, Password, Email) -> }, r3_hex_api:post(Config, ["users"], Params). -reset_password(Username, Config) when is_binary(Username) and is_map(Config) -> +%% @doc +%% Resets the user's password. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_api_user:reset_password(r3_hex_core:default_config(), <<"user">>). +%% {ok, {204, ..., nil}} +%% ''' +%% @end +-spec reset_password(r3_hex_core:config(), binary()) -> r3_hex_api:response(). +reset_password(Config, Username) when is_map(Config) and is_binary(Username) -> r3_hex_api:post(Config, ["users", Username, "reset"], #{}). %% @doc -%% Gets user. +%% Gets a user. %% %% Examples: %% %% ``` -%% > r3_hex_api_user:get(<<"user">>, r3_hex_core:default_config()). +%% > r3_hex_api_user:get(r3_hex_core:default_config()). %% {ok, {200, ..., #{ -%% <<"username">> => <<"user">>, -%% <<"packages">> => [ -%% #{ -%% <<"name">> => ..., -%% <<"url">> => ..., -%% ... -%% }, -%% ... -%% ], -%% ...}}} +%% <<"email">> => <<"user@example.com">>, +%% <<"full_name">> => <<"John Doe">>, +%% <<"handles">> => #{...}, +%% <<"inserted_at">> => <<"2014-04-21T17:20:12Z">>, +%% <<"level">> => <<"full">>, +%% <<"updated_at">> => <<"2019-08-04T19:28:05Z">>, +%% <<"url">> => <<"https://hex.pm/api/users/user">>, +%% <<"username">> => <<"user">> +%% }}} %% ''' %% @end -get(Config, Username) when is_binary(Username) and is_map(Config) -> +-spec get(r3_hex_core:config(), binary()) -> r3_hex_api:response(). +get(Config, Username) when is_map(Config) and is_binary(Username) -> r3_hex_api:get(Config, ["users", Username]). diff --git a/src/r3_hex_core.erl b/src/r3_hex_core.erl index 8489ac3c5..1f524b59a 100644 --- a/src/r3_hex_core.erl +++ b/src/r3_hex_core.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually %% @doc %% hex_core entrypoint module. @@ -61,6 +61,7 @@ J1i2xWFndWa6nfFnRxZmCStCOZWYYPlaxr+FZceFbpMwzTNs4g3d4tLNUcbKAIH4 http_adapter => module(), http_etag => binary() | undefined, http_adapter_config => map(), + http_headers => map(), http_user_agent_fragment => binary(), repo_key => binary() | undefined, repo_name => binary(), @@ -88,5 +89,6 @@ default_config() -> repo_url => <<"https://repo.hex.pm">>, repo_organization => undefined, repo_verify => true, - repo_verify_origin => true + repo_verify_origin => true, + http_headers => #{} }. diff --git a/src/r3_hex_core.hrl b/src/r3_hex_core.hrl index 7d19c84f5..721fca860 100644 --- a/src/r3_hex_core.hrl +++ b/src/r3_hex_core.hrl @@ -1,3 +1,3 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually --define(HEX_CORE_VERSION, "0.5.1"). +-define(HEX_CORE_VERSION, "0.6.8"). diff --git a/src/r3_hex_erl_tar.erl b/src/r3_hex_erl_tar.erl index 2cc36c6e5..a2da27838 100644 --- a/src/r3_hex_erl_tar.erl +++ b/src/r3_hex_erl_tar.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually %% @private %% Copied from https://github.com/erlang/otp/blob/OTP-20.0.1/lib/stdlib/src/erl_tar.erl @@ -8,6 +8,9 @@ %% - Preserve modes when building tarball %% - Do not crash if failing to write tar %% - Allow setting file_info opts on :r3_hex_erl_tar.add +%% - Add safe_relative_path_links/2 to check directory traversal vulnerability when extracting files, +%% it differs from OTP's current fix (2020-02-04) in that it checks regular files instead of +%% symlink targets. This allows creating symlinks with relative path targets such as `../tmp/log` %% %% %CopyrightBegin% @@ -1649,13 +1652,50 @@ write_extracted_element(#tar_header{name=Name0}=Header, Bin, Opts) -> make_safe_path([$/|Path], Opts) -> make_safe_path(Path, Opts); make_safe_path(Path, #read_opts{cwd=Cwd}) -> - case r3_hex_filename:safe_relative_path(Path) of + case safe_relative_path_links(Path, Cwd) of unsafe -> throw({error,{Path,unsafe_path}}); SafePath -> filename:absname(SafePath, Cwd) end. +safe_relative_path_links(Path, Cwd) -> + case filename:pathtype(Path) of + relative -> safe_relative_path_links(filename:split(Path), Cwd, [], ""); + _ -> unsafe + end. + +safe_relative_path_links([], _Cwd, _PrevLinks, Acc) -> + Acc; + +safe_relative_path_links([Segment | Segments], Cwd, PrevLinks, Acc) -> + AccSegment = join(Acc, Segment), + + case r3_hex_filename:safe_relative_path(AccSegment) of + unsafe -> + unsafe; + + SafeAccSegment -> + case file:read_link(join(Cwd, SafeAccSegment)) of + {ok, LinkPath} -> + case lists:member(LinkPath, PrevLinks) of + true -> + unsafe; + false -> + case safe_relative_path_links(filename:split(LinkPath), Cwd, [LinkPath | PrevLinks], Acc) of + unsafe -> unsafe; + NewAcc -> safe_relative_path_links(Segments, Cwd, [], NewAcc) + end + end; + + {error, _} -> + safe_relative_path_links(Segments, Cwd, PrevLinks, SafeAccSegment) + end + end. + +join([], Path) -> Path; +join(Left, Right) -> filename:join(Left, Right). + create_regular(Name, NameInArchive, Bin, Opts) -> case write_extracted_file(Name, Bin, Opts) of not_written -> diff --git a/src/r3_hex_erl_tar.hrl b/src/r3_hex_erl_tar.hrl index 489ce81be..647729697 100644 --- a/src/r3_hex_erl_tar.hrl +++ b/src/r3_hex_erl_tar.hrl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually % Copied from https://github.com/erlang/otp/blob/OTP-20.0.1/lib/stdlib/src/erl_tar.hrl @@ -24,13 +24,13 @@ %% Options used when adding files to a tar archive. -record(add_opts, { read_info, %% Fun to use for read file/link info. - chunk_size = 0, %% For file reading when sending to sftp. 0=do not chunk - verbose = false, %% Verbose on/off. - atime = undefined, - mtime = undefined, - ctime = undefined, - uid = 0, - gid = 0}). + chunk_size = 0 :: integer(), %% For file reading when sending to sftp. 0=do not chunk + verbose = false :: boolean(), %% Verbose on/off. + atime = undefined :: undefined | integer(), + mtime = undefined :: undefined | integer(), + ctime = undefined :: undefined | integer(), + uid = 0 :: integer(), + gid = 0 :: integer()}). -type add_opts() :: #add_opts{}. %% Options used when reading a tar archive. @@ -43,9 +43,15 @@ verbose = false :: boolean()}). %% Verbose on/off. -type read_opts() :: #read_opts{}. --type add_opt() :: dereference | - verbose | - {chunks, pos_integer()}. +-type add_opt() :: dereference + | verbose + | {chunks, pos_integer()} + | {atime, integer()} + | {mtime, integer()} + | {ctime, integer()} + | {uid, integer()} + | {gid, integer()}. + -type extract_opt() :: {cwd, string()} | {files, [string()]} | diff --git a/src/r3_hex_filename.erl b/src/r3_hex_filename.erl index da2c0cdb8..fe27a6c17 100644 --- a/src/r3_hex_filename.erl +++ b/src/r3_hex_filename.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually % @private % Excerpt from https://github.com/erlang/otp/blob/OTP-20.0.1/lib/stdlib/src/filename.erl#L761-L788 diff --git a/src/r3_hex_http.erl b/src/r3_hex_http.erl index f1b7a3f07..d42584145 100644 --- a/src/r3_hex_http.erl +++ b/src/r3_hex_http.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually -module(r3_hex_http). -export([request/5]). diff --git a/src/r3_hex_http_httpc.erl b/src/r3_hex_http_httpc.erl index 6de822ab3..c4b5d6f2b 100644 --- a/src/r3_hex_http_httpc.erl +++ b/src/r3_hex_http_httpc.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually %% @hidden @@ -13,10 +13,12 @@ request(Method, URI, ReqHeaders, Body, AdapterConfig) -> Profile = maps:get(profile, AdapterConfig, default), Request = build_request(URI, ReqHeaders, Body), - {ok, {{_, StatusCode, _}, RespHeaders, RespBody}} = - httpc:request(Method, Request, [], [{body_format, binary}], Profile), - RespHeaders2 = load_headers(RespHeaders), - {ok, {StatusCode, RespHeaders2, RespBody}}. + case httpc:request(Method, Request, [], [{body_format, binary}], Profile) of + {ok, {{_, StatusCode, _}, RespHeaders, RespBody}} -> + RespHeaders2 = load_headers(RespHeaders), + {ok, {StatusCode, RespHeaders2, RespBody}}; + {error, Reason} -> {error, Reason} + end. %%==================================================================== %% Internal functions diff --git a/src/r3_hex_pb_names.erl b/src/r3_hex_pb_names.erl index 8cfe973f1..fa0f87f4e 100644 --- a/src/r3_hex_pb_names.erl +++ b/src/r3_hex_pb_names.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually %% -*- coding: utf-8 -*- %% Automatically generated, do not edit diff --git a/src/r3_hex_pb_package.erl b/src/r3_hex_pb_package.erl index 7da1b8e6a..820df11fd 100644 --- a/src/r3_hex_pb_package.erl +++ b/src/r3_hex_pb_package.erl @@ -1,8 +1,8 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually %% -*- coding: utf-8 -*- %% Automatically generated, do not edit -%% Generated by gpb_compile version 4.3.1 +%% Generated by gpb_compile version 4.2.1 -module(r3_hex_pb_package). -export([encode_msg/2, encode_msg/3]). @@ -36,19 +36,17 @@ name => iodata(), % = 2 repository => iodata() % = 3 }. - -type 'Release'() :: #{version => iodata(), % = 1 - checksum => iodata(), % = 2 + inner_checksum => iodata(), % = 2 dependencies => ['Dependency'()] % = 3 %% retired => 'RetirementStatus'() % = 4 + %% outer_checksum => iodata() % = 5 }. - -type 'RetirementStatus'() :: #{reason => 'RETIRED_OTHER' | 'RETIRED_INVALID' | 'RETIRED_SECURITY' | 'RETIRED_DEPRECATED' | 'RETIRED_RENAMED' | integer() % = 1, enum RetirementReason %% message => iodata() % = 2 }. - -type 'Dependency'() :: #{package => iodata(), % = 1 requirement => iodata() % = 2 @@ -56,29 +54,27 @@ %% app => iodata() % = 4 %% repository => iodata() % = 5 }. - -export_type(['Package'/0, 'Release'/0, 'RetirementStatus'/0, 'Dependency'/0]). --spec encode_msg('Package'() | 'Release'() | 'RetirementStatus'() | 'Dependency'(), atom()) -> binary(). -encode_msg(Msg, MsgName) when is_atom(MsgName) -> +-spec encode_msg('Package'() | 'Release'() | 'RetirementStatus'() | 'Dependency'(),'Package' | 'Release' | 'RetirementStatus' | 'Dependency') -> binary(). +encode_msg(Msg, MsgName) -> encode_msg(Msg, MsgName, []). --spec encode_msg('Package'() | 'Release'() | 'RetirementStatus'() | 'Dependency'(), atom(), list()) -> binary(). + +-spec encode_msg('Package'() | 'Release'() | 'RetirementStatus'() | 'Dependency'(),'Package' | 'Release' | 'RetirementStatus' | 'Dependency', list()) -> binary(). encode_msg(Msg, MsgName, Opts) -> verify_msg(Msg, MsgName, Opts), TrUserData = proplists:get_value(user_data, Opts), case MsgName of - 'Package' -> - e_msg_Package(id(Msg, TrUserData), TrUserData); - 'Release' -> - e_msg_Release(id(Msg, TrUserData), TrUserData); + 'Package' -> e_msg_Package(Msg, TrUserData); + 'Release' -> e_msg_Release(Msg, TrUserData); 'RetirementStatus' -> - e_msg_RetirementStatus(id(Msg, TrUserData), TrUserData); - 'Dependency' -> - e_msg_Dependency(id(Msg, TrUserData), TrUserData) + e_msg_RetirementStatus(Msg, TrUserData); + 'Dependency' -> e_msg_Dependency(Msg, TrUserData) end. + e_msg_Package(Msg, TrUserData) -> e_msg_Package(Msg, <<>>, TrUserData). @@ -95,26 +91,27 @@ e_msg_Package(#{name := F2, repository := F3} = M, Bin, end, B2 = begin TrF2 = id(F2, TrUserData), - e_type_string(TrF2, <>, TrUserData) + e_type_string(TrF2, <>) end, begin TrF3 = id(F3, TrUserData), - e_type_string(TrF3, <>, TrUserData) + e_type_string(TrF3, <>) end. e_msg_Release(Msg, TrUserData) -> e_msg_Release(Msg, <<>>, TrUserData). -e_msg_Release(#{version := F1, checksum := F2} = M, Bin, - TrUserData) -> +e_msg_Release(#{version := F1, inner_checksum := F2} = + M, + Bin, TrUserData) -> B1 = begin TrF1 = id(F1, TrUserData), - e_type_string(TrF1, <>, TrUserData) + e_type_string(TrF1, <>) end, B2 = begin TrF2 = id(F2, TrUserData), - e_type_bytes(TrF2, <>, TrUserData) + e_type_bytes(TrF2, <>) end, B3 = case M of #{dependencies := F3} -> @@ -125,14 +122,22 @@ e_msg_Release(#{version := F1, checksum := F2} = M, Bin, end; _ -> B2 end, + B4 = case M of + #{retired := F4} -> + begin + TrF4 = id(F4, TrUserData), + e_mfield_Release_retired(TrF4, <>, + TrUserData) + end; + _ -> B3 + end, case M of - #{retired := F4} -> + #{outer_checksum := F5} -> begin - TrF4 = id(F4, TrUserData), - e_mfield_Release_retired(TrF4, <>, - TrUserData) + TrF5 = id(F5, TrUserData), + e_type_bytes(TrF5, <>) end; - _ -> B3 + _ -> B4 end. e_msg_RetirementStatus(Msg, TrUserData) -> @@ -143,14 +148,13 @@ e_msg_RetirementStatus(#{reason := F1} = M, Bin, TrUserData) -> B1 = begin TrF1 = id(F1, TrUserData), - e_enum_RetirementReason(TrF1, <>, - TrUserData) + e_enum_RetirementReason(TrF1, <>) end, case M of #{message := F2} -> begin TrF2 = id(F2, TrUserData), - e_type_string(TrF2, <>, TrUserData) + e_type_string(TrF2, <>) end; _ -> B1 end. @@ -164,17 +168,17 @@ e_msg_Dependency(#{package := F1, requirement := F2} = Bin, TrUserData) -> B1 = begin TrF1 = id(F1, TrUserData), - e_type_string(TrF1, <>, TrUserData) + e_type_string(TrF1, <>) end, B2 = begin TrF2 = id(F2, TrUserData), - e_type_string(TrF2, <>, TrUserData) + e_type_string(TrF2, <>) end, B3 = case M of #{optional := F3} -> begin TrF3 = id(F3, TrUserData), - e_type_bool(TrF3, <>, TrUserData) + e_type_bool(TrF3, <>) end; _ -> B2 end, @@ -182,7 +186,7 @@ e_msg_Dependency(#{package := F1, requirement := F2} = #{app := F4} -> begin TrF4 = id(F4, TrUserData), - e_type_string(TrF4, <>, TrUserData) + e_type_string(TrF4, <>) end; _ -> B3 end, @@ -190,7 +194,7 @@ e_msg_Dependency(#{package := F1, requirement := F2} = #{repository := F5} -> begin TrF5 = id(F5, TrUserData), - e_type_string(TrF5, <>, TrUserData) + e_type_string(TrF5, <>) end; _ -> B4 end. @@ -228,111 +232,36 @@ e_mfield_Release_retired(Msg, Bin, TrUserData) -> Bin2 = e_varint(byte_size(SubBin), Bin), <>. -e_enum_RetirementReason('RETIRED_OTHER', Bin, - _TrUserData) -> +e_enum_RetirementReason('RETIRED_OTHER', Bin) -> <>; -e_enum_RetirementReason('RETIRED_INVALID', Bin, - _TrUserData) -> +e_enum_RetirementReason('RETIRED_INVALID', Bin) -> <>; -e_enum_RetirementReason('RETIRED_SECURITY', Bin, - _TrUserData) -> +e_enum_RetirementReason('RETIRED_SECURITY', Bin) -> <>; -e_enum_RetirementReason('RETIRED_DEPRECATED', Bin, - _TrUserData) -> +e_enum_RetirementReason('RETIRED_DEPRECATED', Bin) -> <>; -e_enum_RetirementReason('RETIRED_RENAMED', Bin, - _TrUserData) -> +e_enum_RetirementReason('RETIRED_RENAMED', Bin) -> <>; -e_enum_RetirementReason(V, Bin, _TrUserData) -> - e_varint(V, Bin). - --compile({nowarn_unused_function,e_type_sint/3}). -e_type_sint(Value, Bin, _TrUserData) when Value >= 0 -> - e_varint(Value * 2, Bin); -e_type_sint(Value, Bin, _TrUserData) -> - e_varint(Value * -2 - 1, Bin). - --compile({nowarn_unused_function,e_type_int32/3}). -e_type_int32(Value, Bin, _TrUserData) - when 0 =< Value, Value =< 127 -> - <>; -e_type_int32(Value, Bin, _TrUserData) -> - <> = <>, - e_varint(N, Bin). - --compile({nowarn_unused_function,e_type_int64/3}). -e_type_int64(Value, Bin, _TrUserData) - when 0 =< Value, Value =< 127 -> - <>; -e_type_int64(Value, Bin, _TrUserData) -> - <> = <>, - e_varint(N, Bin). - --compile({nowarn_unused_function,e_type_bool/3}). -e_type_bool(true, Bin, _TrUserData) -> - <>; -e_type_bool(false, Bin, _TrUserData) -> - <>; -e_type_bool(1, Bin, _TrUserData) -> <>; -e_type_bool(0, Bin, _TrUserData) -> <>. +e_enum_RetirementReason(V, Bin) -> e_varint(V, Bin). --compile({nowarn_unused_function,e_type_string/3}). -e_type_string(S, Bin, _TrUserData) -> +e_type_bool(true, Bin) -> <>; +e_type_bool(false, Bin) -> <>; +e_type_bool(1, Bin) -> <>; +e_type_bool(0, Bin) -> <>. + +e_type_string(S, Bin) -> Utf8 = unicode:characters_to_binary(S), Bin2 = e_varint(byte_size(Utf8), Bin), <>. --compile({nowarn_unused_function,e_type_bytes/3}). -e_type_bytes(Bytes, Bin, _TrUserData) - when is_binary(Bytes) -> +e_type_bytes(Bytes, Bin) when is_binary(Bytes) -> Bin2 = e_varint(byte_size(Bytes), Bin), <>; -e_type_bytes(Bytes, Bin, _TrUserData) - when is_list(Bytes) -> +e_type_bytes(Bytes, Bin) when is_list(Bytes) -> BytesBin = iolist_to_binary(Bytes), Bin2 = e_varint(byte_size(BytesBin), Bin), <>. --compile({nowarn_unused_function,e_type_fixed32/3}). -e_type_fixed32(Value, Bin, _TrUserData) -> - <>. - --compile({nowarn_unused_function,e_type_sfixed32/3}). -e_type_sfixed32(Value, Bin, _TrUserData) -> - <>. - --compile({nowarn_unused_function,e_type_fixed64/3}). -e_type_fixed64(Value, Bin, _TrUserData) -> - <>. - --compile({nowarn_unused_function,e_type_sfixed64/3}). -e_type_sfixed64(Value, Bin, _TrUserData) -> - <>. - --compile({nowarn_unused_function,e_type_float/3}). -e_type_float(V, Bin, _) when is_number(V) -> - <>; -e_type_float(infinity, Bin, _) -> - <>; -e_type_float('-infinity', Bin, _) -> - <>; -e_type_float(nan, Bin, _) -> - <>. - --compile({nowarn_unused_function,e_type_double/3}). -e_type_double(V, Bin, _) when is_number(V) -> - <>; -e_type_double(infinity, Bin, _) -> - <>; -e_type_double('-infinity', Bin, _) -> - <>; -e_type_double(nan, Bin, _) -> - <>. - --compile({nowarn_unused_function,e_varint/3}). -e_varint(N, Bin, _TrUserData) -> e_varint(N, Bin). - --compile({nowarn_unused_function,e_varint/2}). e_varint(N, Bin) when N =< 127 -> <>; e_varint(N, Bin) -> Bin2 = <>, @@ -369,14 +298,14 @@ decode_msg_1_catch(Bin, MsgName, TrUserData) -> -endif. decode_msg_2_doit('Package', Bin, TrUserData) -> - id(d_msg_Package(Bin, TrUserData), TrUserData); + d_msg_Package(Bin, TrUserData); decode_msg_2_doit('Release', Bin, TrUserData) -> - id(d_msg_Release(Bin, TrUserData), TrUserData); + d_msg_Release(Bin, TrUserData); decode_msg_2_doit('RetirementStatus', Bin, TrUserData) -> - id(d_msg_RetirementStatus(Bin, TrUserData), TrUserData); + d_msg_RetirementStatus(Bin, TrUserData); decode_msg_2_doit('Dependency', Bin, TrUserData) -> - id(d_msg_Dependency(Bin, TrUserData), TrUserData). + d_msg_Dependency(Bin, TrUserData). @@ -399,10 +328,8 @@ dfp_read_field_def_Package(<<26, Rest/binary>>, Z1, Z2, F@_3, TrUserData); dfp_read_field_def_Package(<<>>, 0, 0, R1, F@_2, F@_3, TrUserData) -> - S1 = #{name => F@_2, repository => F@_3}, - if R1 == '$undef' -> S1; - true -> S1#{releases => lists_reverse(R1, TrUserData)} - end; + #{releases => lists_reverse(R1, TrUserData), + name => F@_2, repository => F@_3}; dfp_read_field_def_Package(Other, Z1, Z2, F@_1, F@_2, F@_3, TrUserData) -> dg_read_field_def_Package(Other, Z1, Z2, F@_1, F@_2, @@ -447,10 +374,8 @@ dg_read_field_def_Package(<<0:1, X:7, Rest/binary>>, N, end; dg_read_field_def_Package(<<>>, 0, 0, R1, F@_2, F@_3, TrUserData) -> - S1 = #{name => F@_2, repository => F@_3}, - if R1 == '$undef' -> S1; - true -> S1#{releases => lists_reverse(R1, TrUserData)} - end. + #{releases => lists_reverse(R1, TrUserData), + name => F@_2, repository => F@_3}. d_field_Package_releases(<<1:1, X:7, Rest/binary>>, N, Acc, F@_1, F@_2, F@_3, TrUserData) @@ -479,7 +404,7 @@ d_field_Package_name(<<0:1, X:7, Rest/binary>>, N, Acc, {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, - {id(binary:copy(Bytes), TrUserData), Rest2} + {binary:copy(Bytes), Rest2} end, dfp_read_field_def_Package(RestF, 0, 0, F@_1, NewFValue, F@_3, TrUserData). @@ -494,7 +419,7 @@ d_field_Package_repository(<<0:1, X:7, Rest/binary>>, N, {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, - {id(binary:copy(Bytes), TrUserData), Rest2} + {binary:copy(Bytes), Rest2} end, dfp_read_field_def_Package(RestF, 0, 0, F@_1, F@_2, NewFValue, TrUserData). @@ -540,127 +465,139 @@ d_msg_Release(Bin, TrUserData) -> dfp_read_field_def_Release(Bin, 0, 0, id('$undef', TrUserData), id('$undef', TrUserData), id([], TrUserData), + id('$undef', TrUserData), id('$undef', TrUserData), TrUserData). dfp_read_field_def_Release(<<10, Rest/binary>>, Z1, Z2, - F@_1, F@_2, F@_3, F@_4, TrUserData) -> + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) -> d_field_Release_version(Rest, Z1, Z2, F@_1, F@_2, F@_3, - F@_4, TrUserData); + F@_4, F@_5, TrUserData); dfp_read_field_def_Release(<<18, Rest/binary>>, Z1, Z2, - F@_1, F@_2, F@_3, F@_4, TrUserData) -> - d_field_Release_checksum(Rest, Z1, Z2, F@_1, F@_2, F@_3, - F@_4, TrUserData); + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) -> + d_field_Release_inner_checksum(Rest, Z1, Z2, F@_1, F@_2, + F@_3, F@_4, F@_5, TrUserData); dfp_read_field_def_Release(<<26, Rest/binary>>, Z1, Z2, - F@_1, F@_2, F@_3, F@_4, TrUserData) -> + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) -> d_field_Release_dependencies(Rest, Z1, Z2, F@_1, F@_2, - F@_3, F@_4, TrUserData); + F@_3, F@_4, F@_5, TrUserData); dfp_read_field_def_Release(<<34, Rest/binary>>, Z1, Z2, - F@_1, F@_2, F@_3, F@_4, TrUserData) -> + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) -> d_field_Release_retired(Rest, Z1, Z2, F@_1, F@_2, F@_3, - F@_4, TrUserData); + F@_4, F@_5, TrUserData); +dfp_read_field_def_Release(<<42, Rest/binary>>, Z1, Z2, + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) -> + d_field_Release_outer_checksum(Rest, Z1, Z2, F@_1, F@_2, + F@_3, F@_4, F@_5, TrUserData); dfp_read_field_def_Release(<<>>, 0, 0, F@_1, F@_2, R1, - F@_4, TrUserData) -> - S1 = #{version => F@_1, checksum => F@_2}, - S2 = if R1 == '$undef' -> S1; - true -> - S1#{dependencies => lists_reverse(R1, TrUserData)} + F@_4, F@_5, TrUserData) -> + S1 = #{version => F@_1, inner_checksum => F@_2, + dependencies => lists_reverse(R1, TrUserData)}, + S2 = if F@_4 == '$undef' -> S1; + true -> S1#{retired => F@_4} end, - if F@_4 == '$undef' -> S2; - true -> S2#{retired => F@_4} + if F@_5 == '$undef' -> S2; + true -> S2#{outer_checksum => F@_5} end; dfp_read_field_def_Release(Other, Z1, Z2, F@_1, F@_2, - F@_3, F@_4, TrUserData) -> + F@_3, F@_4, F@_5, TrUserData) -> dg_read_field_def_Release(Other, Z1, Z2, F@_1, F@_2, - F@_3, F@_4, TrUserData). + F@_3, F@_4, F@_5, TrUserData). dg_read_field_def_Release(<<1:1, X:7, Rest/binary>>, N, - Acc, F@_1, F@_2, F@_3, F@_4, TrUserData) + Acc, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) when N < 32 - 7 -> dg_read_field_def_Release(Rest, N + 7, X bsl N + Acc, - F@_1, F@_2, F@_3, F@_4, TrUserData); + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData); dg_read_field_def_Release(<<0:1, X:7, Rest/binary>>, N, - Acc, F@_1, F@_2, F@_3, F@_4, TrUserData) -> + Acc, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) -> Key = X bsl N + Acc, case Key of 10 -> d_field_Release_version(Rest, 0, 0, F@_1, F@_2, F@_3, - F@_4, TrUserData); + F@_4, F@_5, TrUserData); 18 -> - d_field_Release_checksum(Rest, 0, 0, F@_1, F@_2, F@_3, - F@_4, TrUserData); + d_field_Release_inner_checksum(Rest, 0, 0, F@_1, F@_2, + F@_3, F@_4, F@_5, TrUserData); 26 -> d_field_Release_dependencies(Rest, 0, 0, F@_1, F@_2, - F@_3, F@_4, TrUserData); + F@_3, F@_4, F@_5, TrUserData); 34 -> d_field_Release_retired(Rest, 0, 0, F@_1, F@_2, F@_3, - F@_4, TrUserData); + F@_4, F@_5, TrUserData); + 42 -> + d_field_Release_outer_checksum(Rest, 0, 0, F@_1, F@_2, + F@_3, F@_4, F@_5, TrUserData); _ -> case Key band 7 of 0 -> skip_varint_Release(Rest, 0, 0, F@_1, F@_2, F@_3, F@_4, - TrUserData); + F@_5, TrUserData); 1 -> skip_64_Release(Rest, 0, 0, F@_1, F@_2, F@_3, F@_4, - TrUserData); + F@_5, TrUserData); 2 -> skip_length_delimited_Release(Rest, 0, 0, F@_1, F@_2, - F@_3, F@_4, TrUserData); + F@_3, F@_4, F@_5, TrUserData); 3 -> skip_group_Release(Rest, Key bsr 3, 0, F@_1, F@_2, F@_3, - F@_4, TrUserData); + F@_4, F@_5, TrUserData); 5 -> skip_32_Release(Rest, 0, 0, F@_1, F@_2, F@_3, F@_4, - TrUserData) + F@_5, TrUserData) end end; dg_read_field_def_Release(<<>>, 0, 0, F@_1, F@_2, R1, - F@_4, TrUserData) -> - S1 = #{version => F@_1, checksum => F@_2}, - S2 = if R1 == '$undef' -> S1; - true -> - S1#{dependencies => lists_reverse(R1, TrUserData)} + F@_4, F@_5, TrUserData) -> + S1 = #{version => F@_1, inner_checksum => F@_2, + dependencies => lists_reverse(R1, TrUserData)}, + S2 = if F@_4 == '$undef' -> S1; + true -> S1#{retired => F@_4} end, - if F@_4 == '$undef' -> S2; - true -> S2#{retired => F@_4} + if F@_5 == '$undef' -> S2; + true -> S2#{outer_checksum => F@_5} end. d_field_Release_version(<<1:1, X:7, Rest/binary>>, N, - Acc, F@_1, F@_2, F@_3, F@_4, TrUserData) + Acc, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) when N < 57 -> d_field_Release_version(Rest, N + 7, X bsl N + Acc, - F@_1, F@_2, F@_3, F@_4, TrUserData); + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData); d_field_Release_version(<<0:1, X:7, Rest/binary>>, N, - Acc, _, F@_2, F@_3, F@_4, TrUserData) -> + Acc, _, F@_2, F@_3, F@_4, F@_5, TrUserData) -> {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, - {id(binary:copy(Bytes), TrUserData), Rest2} + {binary:copy(Bytes), Rest2} end, dfp_read_field_def_Release(RestF, 0, 0, NewFValue, F@_2, - F@_3, F@_4, TrUserData). + F@_3, F@_4, F@_5, TrUserData). -d_field_Release_checksum(<<1:1, X:7, Rest/binary>>, N, - Acc, F@_1, F@_2, F@_3, F@_4, TrUserData) +d_field_Release_inner_checksum(<<1:1, X:7, + Rest/binary>>, + N, Acc, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) when N < 57 -> - d_field_Release_checksum(Rest, N + 7, X bsl N + Acc, - F@_1, F@_2, F@_3, F@_4, TrUserData); -d_field_Release_checksum(<<0:1, X:7, Rest/binary>>, N, - Acc, F@_1, _, F@_3, F@_4, TrUserData) -> + d_field_Release_inner_checksum(Rest, N + 7, + X bsl N + Acc, F@_1, F@_2, F@_3, F@_4, F@_5, + TrUserData); +d_field_Release_inner_checksum(<<0:1, X:7, + Rest/binary>>, + N, Acc, F@_1, _, F@_3, F@_4, F@_5, TrUserData) -> {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, - {id(binary:copy(Bytes), TrUserData), Rest2} + {binary:copy(Bytes), Rest2} end, dfp_read_field_def_Release(RestF, 0, 0, F@_1, NewFValue, - F@_3, F@_4, TrUserData). + F@_3, F@_4, F@_5, TrUserData). d_field_Release_dependencies(<<1:1, X:7, Rest/binary>>, - N, Acc, F@_1, F@_2, F@_3, F@_4, TrUserData) + N, Acc, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) when N < 57 -> d_field_Release_dependencies(Rest, N + 7, X bsl N + Acc, - F@_1, F@_2, F@_3, F@_4, TrUserData); + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData); d_field_Release_dependencies(<<0:1, X:7, Rest/binary>>, - N, Acc, F@_1, F@_2, Prev, F@_4, TrUserData) -> + N, Acc, F@_1, F@_2, Prev, F@_4, F@_5, + TrUserData) -> {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, @@ -668,16 +605,16 @@ d_field_Release_dependencies(<<0:1, X:7, Rest/binary>>, Rest2} end, dfp_read_field_def_Release(RestF, 0, 0, F@_1, F@_2, - cons(NewFValue, Prev, TrUserData), F@_4, + cons(NewFValue, Prev, TrUserData), F@_4, F@_5, TrUserData). d_field_Release_retired(<<1:1, X:7, Rest/binary>>, N, - Acc, F@_1, F@_2, F@_3, F@_4, TrUserData) + Acc, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) when N < 57 -> d_field_Release_retired(Rest, N + 7, X bsl N + Acc, - F@_1, F@_2, F@_3, F@_4, TrUserData); + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData); d_field_Release_retired(<<0:1, X:7, Rest/binary>>, N, - Acc, F@_1, F@_2, F@_3, Prev, TrUserData) -> + Acc, F@_1, F@_2, F@_3, Prev, F@_5, TrUserData) -> {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, @@ -693,45 +630,64 @@ d_field_Release_retired(<<0:1, X:7, Rest/binary>>, N, NewFValue, TrUserData) end, - TrUserData). + F@_5, TrUserData). + +d_field_Release_outer_checksum(<<1:1, X:7, + Rest/binary>>, + N, Acc, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) + when N < 57 -> + d_field_Release_outer_checksum(Rest, N + 7, + X bsl N + Acc, F@_1, F@_2, F@_3, F@_4, F@_5, + TrUserData); +d_field_Release_outer_checksum(<<0:1, X:7, + Rest/binary>>, + N, Acc, F@_1, F@_2, F@_3, F@_4, _, TrUserData) -> + {NewFValue, RestF} = begin + Len = X bsl N + Acc, + <> = Rest, + {binary:copy(Bytes), Rest2} + end, + dfp_read_field_def_Release(RestF, 0, 0, F@_1, F@_2, + F@_3, F@_4, NewFValue, TrUserData). skip_varint_Release(<<1:1, _:7, Rest/binary>>, Z1, Z2, - F@_1, F@_2, F@_3, F@_4, TrUserData) -> + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) -> skip_varint_Release(Rest, Z1, Z2, F@_1, F@_2, F@_3, - F@_4, TrUserData); + F@_4, F@_5, TrUserData); skip_varint_Release(<<0:1, _:7, Rest/binary>>, Z1, Z2, - F@_1, F@_2, F@_3, F@_4, TrUserData) -> + F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) -> dfp_read_field_def_Release(Rest, Z1, Z2, F@_1, F@_2, - F@_3, F@_4, TrUserData). + F@_3, F@_4, F@_5, TrUserData). skip_length_delimited_Release(<<1:1, X:7, Rest/binary>>, - N, Acc, F@_1, F@_2, F@_3, F@_4, TrUserData) + N, Acc, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData) when N < 57 -> skip_length_delimited_Release(Rest, N + 7, - X bsl N + Acc, F@_1, F@_2, F@_3, F@_4, + X bsl N + Acc, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData); skip_length_delimited_Release(<<0:1, X:7, Rest/binary>>, - N, Acc, F@_1, F@_2, F@_3, F@_4, TrUserData) -> + N, Acc, F@_1, F@_2, F@_3, F@_4, F@_5, + TrUserData) -> Length = X bsl N + Acc, <<_:Length/binary, Rest2/binary>> = Rest, dfp_read_field_def_Release(Rest2, 0, 0, F@_1, F@_2, - F@_3, F@_4, TrUserData). + F@_3, F@_4, F@_5, TrUserData). skip_group_Release(Bin, FNum, Z2, F@_1, F@_2, F@_3, - F@_4, TrUserData) -> + F@_4, F@_5, TrUserData) -> {_, Rest} = read_group(Bin, FNum), dfp_read_field_def_Release(Rest, 0, Z2, F@_1, F@_2, - F@_3, F@_4, TrUserData). + F@_3, F@_4, F@_5, TrUserData). skip_32_Release(<<_:32, Rest/binary>>, Z1, Z2, F@_1, - F@_2, F@_3, F@_4, TrUserData) -> + F@_2, F@_3, F@_4, F@_5, TrUserData) -> dfp_read_field_def_Release(Rest, Z1, Z2, F@_1, F@_2, - F@_3, F@_4, TrUserData). + F@_3, F@_4, F@_5, TrUserData). skip_64_Release(<<_:64, Rest/binary>>, Z1, Z2, F@_1, - F@_2, F@_3, F@_4, TrUserData) -> + F@_2, F@_3, F@_4, F@_5, TrUserData) -> dfp_read_field_def_Release(Rest, Z1, Z2, F@_1, F@_2, - F@_3, F@_4, TrUserData). + F@_3, F@_4, F@_5, TrUserData). d_msg_RetirementStatus(Bin, TrUserData) -> dfp_read_field_def_RetirementStatus(Bin, 0, 0, @@ -809,13 +765,12 @@ d_field_RetirementStatus_reason(<<1:1, X:7, d_field_RetirementStatus_reason(<<0:1, X:7, Rest/binary>>, N, Acc, _, F@_2, TrUserData) -> - {NewFValue, RestF} = {id(d_enum_RetirementReason(begin - <> = - <<(X bsl N + - Acc):32/unsigned-native>>, - id(Res, TrUserData) - end), - TrUserData), + {NewFValue, RestF} = {d_enum_RetirementReason(begin + <> = + <<(X bsl N + + Acc):32/unsigned-native>>, + Res + end), Rest}, dfp_read_field_def_RetirementStatus(RestF, 0, 0, NewFValue, F@_2, TrUserData). @@ -832,7 +787,7 @@ d_field_RetirementStatus_message(<<0:1, X:7, {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, - {id(binary:copy(Bytes), TrUserData), Rest2} + {binary:copy(Bytes), Rest2} end, dfp_read_field_def_RetirementStatus(RestF, 0, 0, F@_1, NewFValue, TrUserData). @@ -989,7 +944,7 @@ d_field_Dependency_package(<<0:1, X:7, Rest/binary>>, N, {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, - {id(binary:copy(Bytes), TrUserData), Rest2} + {binary:copy(Bytes), Rest2} end, dfp_read_field_def_Dependency(RestF, 0, 0, NewFValue, F@_2, F@_3, F@_4, F@_5, TrUserData). @@ -1007,7 +962,7 @@ d_field_Dependency_requirement(<<0:1, X:7, {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, - {id(binary:copy(Bytes), TrUserData), Rest2} + {binary:copy(Bytes), Rest2} end, dfp_read_field_def_Dependency(RestF, 0, 0, F@_1, NewFValue, F@_3, F@_4, F@_5, TrUserData). @@ -1019,9 +974,7 @@ d_field_Dependency_optional(<<1:1, X:7, Rest/binary>>, F@_1, F@_2, F@_3, F@_4, F@_5, TrUserData); d_field_Dependency_optional(<<0:1, X:7, Rest/binary>>, N, Acc, F@_1, F@_2, _, F@_4, F@_5, TrUserData) -> - {NewFValue, RestF} = {id(X bsl N + Acc =/= 0, - TrUserData), - Rest}, + {NewFValue, RestF} = {X bsl N + Acc =/= 0, Rest}, dfp_read_field_def_Dependency(RestF, 0, 0, F@_1, F@_2, NewFValue, F@_4, F@_5, TrUserData). @@ -1035,7 +988,7 @@ d_field_Dependency_app(<<0:1, X:7, Rest/binary>>, N, {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, - {id(binary:copy(Bytes), TrUserData), Rest2} + {binary:copy(Bytes), Rest2} end, dfp_read_field_def_Dependency(RestF, 0, 0, F@_1, F@_2, F@_3, NewFValue, F@_5, TrUserData). @@ -1051,7 +1004,7 @@ d_field_Dependency_repository(<<0:1, X:7, Rest/binary>>, {NewFValue, RestF} = begin Len = X bsl N + Acc, <> = Rest, - {id(binary:copy(Bytes), TrUserData), Rest2} + {binary:copy(Bytes), Rest2} end, dfp_read_field_def_Dependency(RestF, 0, 0, F@_1, F@_2, F@_3, F@_4, NewFValue, TrUserData). @@ -1163,7 +1116,7 @@ read_gr_ld(<<0:1, X:7, Tl/binary>>, N, Acc, NumBytes, FieldNum) -> <<_:Len/binary, Tl2/binary>> = Tl, read_gr_b(Tl2, 0, 0, NumBytes1 + Len, 0, FieldNum). -merge_msgs(Prev, New, MsgName) when is_atom(MsgName) -> +merge_msgs(Prev, New, MsgName) -> merge_msgs(Prev, New, MsgName, []). merge_msgs(Prev, New, MsgName, Opts) -> @@ -1177,7 +1130,6 @@ merge_msgs(Prev, New, MsgName, Opts) -> merge_msg_Dependency(Prev, New, TrUserData) end. --compile({nowarn_unused_function,merge_msg_Package/3}). merge_msg_Package(#{} = PMsg, #{name := NFname, repository := NFrepository} = NMsg, TrUserData) -> @@ -1194,11 +1146,13 @@ merge_msg_Package(#{} = PMsg, {_, _} -> S1 end. --compile({nowarn_unused_function,merge_msg_Release/3}). merge_msg_Release(#{} = PMsg, - #{version := NFversion, checksum := NFchecksum} = NMsg, + #{version := NFversion, + inner_checksum := NFinner_checksum} = + NMsg, TrUserData) -> - S1 = #{version => NFversion, checksum => NFchecksum}, + S1 = #{version => NFversion, + inner_checksum => NFinner_checksum}, S2 = case {PMsg, NMsg} of {#{dependencies := PFdependencies}, #{dependencies := NFdependencies}} -> @@ -1211,19 +1165,25 @@ merge_msg_Release(#{} = PMsg, S1#{dependencies => PFdependencies}; {_, _} -> S1 end, + S3 = case {PMsg, NMsg} of + {#{retired := PFretired}, #{retired := NFretired}} -> + S2#{retired => + merge_msg_RetirementStatus(PFretired, NFretired, + TrUserData)}; + {_, #{retired := NFretired}} -> + S2#{retired => NFretired}; + {#{retired := PFretired}, _} -> + S2#{retired => PFretired}; + {_, _} -> S2 + end, case {PMsg, NMsg} of - {#{retired := PFretired}, #{retired := NFretired}} -> - S2#{retired => - merge_msg_RetirementStatus(PFretired, NFretired, - TrUserData)}; - {_, #{retired := NFretired}} -> - S2#{retired => NFretired}; - {#{retired := PFretired}, _} -> - S2#{retired => PFretired}; - {_, _} -> S2 + {_, #{outer_checksum := NFouter_checksum}} -> + S3#{outer_checksum => NFouter_checksum}; + {#{outer_checksum := PFouter_checksum}, _} -> + S3#{outer_checksum => PFouter_checksum}; + _ -> S3 end. --compile({nowarn_unused_function,merge_msg_RetirementStatus/3}). merge_msg_RetirementStatus(#{} = PMsg, #{reason := NFreason} = NMsg, _) -> S1 = #{reason => NFreason}, @@ -1235,7 +1195,6 @@ merge_msg_RetirementStatus(#{} = PMsg, _ -> S1 end. --compile({nowarn_unused_function,merge_msg_Dependency/3}). merge_msg_Dependency(#{} = PMsg, #{package := NFpackage, requirement := NFrequirement} = NMsg, @@ -1263,23 +1222,25 @@ merge_msg_Dependency(#{} = PMsg, end. -verify_msg(Msg, MsgName) when is_atom(MsgName) -> +verify_msg(Msg, MsgName) -> verify_msg(Msg, MsgName, []). verify_msg(Msg, MsgName, Opts) -> TrUserData = proplists:get_value(user_data, Opts), case MsgName of - 'Package' -> v_msg_Package(Msg, [MsgName], TrUserData); - 'Release' -> v_msg_Release(Msg, [MsgName], TrUserData); + 'Package' -> + v_msg_Package(Msg, ['Package'], TrUserData); + 'Release' -> + v_msg_Release(Msg, ['Release'], TrUserData); 'RetirementStatus' -> - v_msg_RetirementStatus(Msg, [MsgName], TrUserData); + v_msg_RetirementStatus(Msg, ['RetirementStatus'], + TrUserData); 'Dependency' -> - v_msg_Dependency(Msg, [MsgName], TrUserData); + v_msg_Dependency(Msg, ['Dependency'], TrUserData); _ -> mk_type_error(not_a_known_message, Msg, []) end. --compile({nowarn_unused_function,v_msg_Package/3}). v_msg_Package(#{name := F2, repository := F3} = M, Path, TrUserData) -> case M of @@ -1294,8 +1255,8 @@ v_msg_Package(#{name := F2, repository := F3} = M, Path, end; _ -> ok end, - v_type_string(F2, [name | Path], TrUserData), - v_type_string(F3, [repository | Path], TrUserData), + v_type_string(F2, [name | Path]), + v_type_string(F3, [repository | Path]), lists:foreach(fun (releases) -> ok; (name) -> ok; (repository) -> ok; @@ -1311,11 +1272,11 @@ v_msg_Package(M, Path, _TrUserData) when is_map(M) -> v_msg_Package(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'Package'}, X, Path). --compile({nowarn_unused_function,v_msg_Release/3}). -v_msg_Release(#{version := F1, checksum := F2} = M, +v_msg_Release(#{version := F1, inner_checksum := F2} = + M, Path, TrUserData) -> - v_type_string(F1, [version | Path], TrUserData), - v_type_bytes(F2, [checksum | Path], TrUserData), + v_type_string(F1, [version | Path]), + v_type_bytes(F2, [inner_checksum | Path]), case M of #{dependencies := F3} -> if is_list(F3) -> @@ -1335,10 +1296,16 @@ v_msg_Release(#{version := F1, checksum := F2} = M, TrUserData); _ -> ok end, + case M of + #{outer_checksum := F5} -> + v_type_bytes(F5, [outer_checksum | Path]); + _ -> ok + end, lists:foreach(fun (version) -> ok; - (checksum) -> ok; + (inner_checksum) -> ok; (dependencies) -> ok; (retired) -> ok; + (outer_checksum) -> ok; (OtherKey) -> mk_type_error({extraneous_key, OtherKey}, M, Path) end, @@ -1346,19 +1313,15 @@ v_msg_Release(#{version := F1, checksum := F2} = M, ok; v_msg_Release(M, Path, _TrUserData) when is_map(M) -> mk_type_error({missing_fields, - [version, checksum] -- maps:keys(M), 'Release'}, + [version, inner_checksum] -- maps:keys(M), 'Release'}, M, Path); v_msg_Release(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'Release'}, X, Path). --compile({nowarn_unused_function,v_msg_RetirementStatus/3}). -v_msg_RetirementStatus(#{reason := F1} = M, Path, - TrUserData) -> - v_enum_RetirementReason(F1, [reason | Path], - TrUserData), +v_msg_RetirementStatus(#{reason := F1} = M, Path, _) -> + v_enum_RetirementReason(F1, [reason | Path]), case M of - #{message := F2} -> - v_type_string(F2, [message | Path], TrUserData); + #{message := F2} -> v_type_string(F2, [message | Path]); _ -> ok end, lists:foreach(fun (reason) -> ok; @@ -1377,25 +1340,22 @@ v_msg_RetirementStatus(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'RetirementStatus'}, X, Path). --compile({nowarn_unused_function,v_msg_Dependency/3}). v_msg_Dependency(#{package := F1, requirement := F2} = M, - Path, TrUserData) -> - v_type_string(F1, [package | Path], TrUserData), - v_type_string(F2, [requirement | Path], TrUserData), + Path, _) -> + v_type_string(F1, [package | Path]), + v_type_string(F2, [requirement | Path]), case M of - #{optional := F3} -> - v_type_bool(F3, [optional | Path], TrUserData); + #{optional := F3} -> v_type_bool(F3, [optional | Path]); _ -> ok end, case M of - #{app := F4} -> - v_type_string(F4, [app | Path], TrUserData); + #{app := F4} -> v_type_string(F4, [app | Path]); _ -> ok end, case M of #{repository := F5} -> - v_type_string(F5, [repository | Path], TrUserData); + v_type_string(F5, [repository | Path]); _ -> ok end, lists:foreach(fun (package) -> ok; @@ -1415,52 +1375,37 @@ v_msg_Dependency(M, Path, _TrUserData) when is_map(M) -> v_msg_Dependency(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'Dependency'}, X, Path). --compile({nowarn_unused_function,v_enum_RetirementReason/3}). -v_enum_RetirementReason('RETIRED_OTHER', _Path, - _TrUserData) -> - ok; -v_enum_RetirementReason('RETIRED_INVALID', _Path, - _TrUserData) -> - ok; -v_enum_RetirementReason('RETIRED_SECURITY', _Path, - _TrUserData) -> - ok; -v_enum_RetirementReason('RETIRED_DEPRECATED', _Path, - _TrUserData) -> +v_enum_RetirementReason('RETIRED_OTHER', _Path) -> ok; +v_enum_RetirementReason('RETIRED_INVALID', _Path) -> ok; +v_enum_RetirementReason('RETIRED_SECURITY', _Path) -> ok; -v_enum_RetirementReason('RETIRED_RENAMED', _Path, - _TrUserData) -> +v_enum_RetirementReason('RETIRED_DEPRECATED', _Path) -> ok; -v_enum_RetirementReason(V, Path, TrUserData) - when is_integer(V) -> - v_type_sint32(V, Path, TrUserData); -v_enum_RetirementReason(X, Path, _TrUserData) -> +v_enum_RetirementReason('RETIRED_RENAMED', _Path) -> ok; +v_enum_RetirementReason(V, Path) when is_integer(V) -> + v_type_sint32(V, Path); +v_enum_RetirementReason(X, Path) -> mk_type_error({invalid_enum, 'RetirementReason'}, X, Path). --compile({nowarn_unused_function,v_type_sint32/3}). -v_type_sint32(N, _Path, _TrUserData) +v_type_sint32(N, _Path) when -2147483648 =< N, N =< 2147483647 -> ok; -v_type_sint32(N, Path, _TrUserData) - when is_integer(N) -> +v_type_sint32(N, Path) when is_integer(N) -> mk_type_error({value_out_of_range, sint32, signed, 32}, N, Path); -v_type_sint32(X, Path, _TrUserData) -> +v_type_sint32(X, Path) -> mk_type_error({bad_integer, sint32, signed, 32}, X, Path). --compile({nowarn_unused_function,v_type_bool/3}). -v_type_bool(false, _Path, _TrUserData) -> ok; -v_type_bool(true, _Path, _TrUserData) -> ok; -v_type_bool(0, _Path, _TrUserData) -> ok; -v_type_bool(1, _Path, _TrUserData) -> ok; -v_type_bool(X, Path, _TrUserData) -> +v_type_bool(false, _Path) -> ok; +v_type_bool(true, _Path) -> ok; +v_type_bool(0, _Path) -> ok; +v_type_bool(1, _Path) -> ok; +v_type_bool(X, Path) -> mk_type_error(bad_boolean_value, X, Path). --compile({nowarn_unused_function,v_type_string/3}). -v_type_string(S, Path, _TrUserData) - when is_list(S); is_binary(S) -> +v_type_string(S, Path) when is_list(S); is_binary(S) -> try unicode:characters_to_binary(S) of B when is_binary(B) -> ok; {error, _, _} -> @@ -1469,18 +1414,14 @@ v_type_string(S, Path, _TrUserData) error:badarg -> mk_type_error(bad_unicode_string, S, Path) end; -v_type_string(X, Path, _TrUserData) -> +v_type_string(X, Path) -> mk_type_error(bad_unicode_string, X, Path). --compile({nowarn_unused_function,v_type_bytes/3}). -v_type_bytes(B, _Path, _TrUserData) when is_binary(B) -> - ok; -v_type_bytes(B, _Path, _TrUserData) when is_list(B) -> - ok; -v_type_bytes(X, Path, _TrUserData) -> +v_type_bytes(B, _Path) when is_binary(B) -> ok; +v_type_bytes(B, _Path) when is_list(B) -> ok; +v_type_bytes(X, Path) -> mk_type_error(bad_binary_value, X, Path). --compile({nowarn_unused_function,mk_type_error/3}). -spec mk_type_error(_, _, list()) -> no_return(). mk_type_error(Error, ValueSeen, Path) -> Path2 = prettify_path(Path), @@ -1488,7 +1429,6 @@ mk_type_error(Error, ValueSeen, Path) -> {Error, [{value, ValueSeen}, {path, Path2}]}}). --compile({nowarn_unused_function,prettify_path/1}). prettify_path([]) -> top_level; prettify_path(PathR) -> list_to_atom(string:join(lists:map(fun atom_to_list/1, @@ -1496,26 +1436,14 @@ prettify_path(PathR) -> ".")). --compile({nowarn_unused_function,id/2}). -compile({inline,id/2}). id(X, _TrUserData) -> X. --compile({nowarn_unused_function,v_ok/3}). --compile({inline,v_ok/3}). -v_ok(_Value, _Path, _TrUserData) -> ok. - --compile({nowarn_unused_function,m_overwrite/3}). --compile({inline,m_overwrite/3}). -m_overwrite(_Prev, New, _TrUserData) -> New. - --compile({nowarn_unused_function,cons/3}). -compile({inline,cons/3}). cons(Elem, Acc, _TrUserData) -> [Elem | Acc]. --compile({nowarn_unused_function,lists_reverse/2}). -compile({inline,lists_reverse/2}). 'lists_reverse'(L, _TrUserData) -> lists:reverse(L). --compile({nowarn_unused_function,'erlang_++'/3}). -compile({inline,'erlang_++'/3}). 'erlang_++'(A, B, _TrUserData) -> A ++ B. @@ -1535,14 +1463,16 @@ get_msg_defs() -> {{msg, 'Release'}, [#{name => version, fnum => 1, rnum => 2, type => string, occurrence => required, opts => []}, - #{name => checksum, fnum => 2, rnum => 3, type => bytes, - occurrence => required, opts => []}, + #{name => inner_checksum, fnum => 2, rnum => 3, + type => bytes, occurrence => required, opts => []}, #{name => dependencies, fnum => 3, rnum => 4, type => {msg, 'Dependency'}, occurrence => repeated, opts => []}, #{name => retired, fnum => 4, rnum => 5, type => {msg, 'RetirementStatus'}, - occurrence => optional, opts => []}]}, + occurrence => optional, opts => []}, + #{name => outer_checksum, fnum => 5, rnum => 6, + type => bytes, occurrence => optional, opts => []}]}, {{msg, 'RetirementStatus'}, [#{name => reason, fnum => 1, rnum => 2, type => {enum, 'RetirementReason'}, @@ -1603,14 +1533,16 @@ find_msg_def('Package') -> find_msg_def('Release') -> [#{name => version, fnum => 1, rnum => 2, type => string, occurrence => required, opts => []}, - #{name => checksum, fnum => 2, rnum => 3, type => bytes, - occurrence => required, opts => []}, + #{name => inner_checksum, fnum => 2, rnum => 3, + type => bytes, occurrence => required, opts => []}, #{name => dependencies, fnum => 3, rnum => 4, type => {msg, 'Dependency'}, occurrence => repeated, opts => []}, #{name => retired, fnum => 4, rnum => 5, type => {msg, 'RetirementStatus'}, - occurrence => optional, opts => []}]; + occurrence => optional, opts => []}, + #{name => outer_checksum, fnum => 5, rnum => 6, + type => bytes, occurrence => optional, opts => []}]; find_msg_def('RetirementStatus') -> [#{name => reason, fnum => 1, rnum => 2, type => {enum, 'RetirementReason'}, @@ -1693,7 +1625,7 @@ get_package_name() -> undefined. gpb_version_as_string() -> - "4.3.1". + "4.2.1". gpb_version_as_list() -> - [4,3,1]. + [4,2,1]. diff --git a/src/r3_hex_pb_signed.erl b/src/r3_hex_pb_signed.erl index 9e2e791ba..22bc58e42 100644 --- a/src/r3_hex_pb_signed.erl +++ b/src/r3_hex_pb_signed.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually %% -*- coding: utf-8 -*- %% Automatically generated, do not edit diff --git a/src/r3_hex_pb_versions.erl b/src/r3_hex_pb_versions.erl index b3c310f9e..3cbd4a039 100644 --- a/src/r3_hex_pb_versions.erl +++ b/src/r3_hex_pb_versions.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually %% -*- coding: utf-8 -*- %% Automatically generated, do not edit diff --git a/src/r3_hex_registry.erl b/src/r3_hex_registry.erl index 43b71346e..6c8da4de4 100644 --- a/src/r3_hex_registry.erl +++ b/src/r3_hex_registry.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually -module(r3_hex_registry). -export([ @@ -93,7 +93,7 @@ decode_signed(Signed) -> %% @doc %% Decode message created with sign_protobuf/2 and verify it against public key. --spec decode_and_verify_signed(map(), public_key()) -> {ok, binary()} | {error, term()}. +-spec decode_and_verify_signed(binary(), public_key()) -> {ok, binary()} | {error, term()}. decode_and_verify_signed(Signed, PublicKey) -> #{payload := Payload, signature := Signature} = decode_signed(Signed), case verify(Payload, Signature, PublicKey) of diff --git a/src/r3_hex_repo.erl b/src/r3_hex_repo.erl index 0075e7f14..986b9bd86 100644 --- a/src/r3_hex_repo.erl +++ b/src/r3_hex_repo.erl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually -module(r3_hex_repo). -export([ diff --git a/src/r3_hex_tarball.erl b/src/r3_hex_tarball.erl index 63b61f8b2..9bc7a69c5 100644 --- a/src/r3_hex_tarball.erl +++ b/src/r3_hex_tarball.erl @@ -1,7 +1,7 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually -module(r3_hex_tarball). --export([create/2, create_docs/1, unpack/2, format_checksum/1, format_error/1]). +-export([create/2, create_docs/1, unpack/2, unpack_docs/2, format_checksum/1, format_error/1]). -ifdef(TEST). -export([do_decode_metadata/1, gzip/1, normalize_requirements/1]). -endif. @@ -20,7 +20,7 @@ -type checksum() :: binary(). -type contents() :: #{filename() => binary()}. -type filename() :: string(). --type files() :: [filename() | {filename(), filename()}] | contents(). +-type files() :: [{filename(), filename() | binary()}]. -type metadata() :: map(). -type tarball() :: binary(). @@ -31,34 +31,37 @@ %% @doc %% Creates a package tarball. %% +%% Returns the binary of the tarball the "inner checksum" and "outer checksum". +%% The inner checksum is deprecated in favor of the inner checksum. +%% %% Examples: %% %% ``` %% > Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, %% > Files = [{"src/foo.erl", <<"-module(foo).">>}], -%% > {ok, {Tarball, Checksum}} = r3_hex_tarball:create(Metadata, Files). -%% > Tarball. -%% <<86,69,...>> -%% > Checksum. -%% <<40,32,...>> +%% > r3_hex_tarball:create(Metadata, Files). +%% {ok, #{tarball => <<86,69,...>>, +%% outer_checksum => <<40,32,...>>, +%% inner_checksum => <<178,12,...>>}} %% ''' %% @end --spec create(metadata(), files()) -> {ok, {tarball(), checksum()}}. +-spec create(metadata(), files()) -> {ok, {tarball(), checksum()}} | {error, term()}. create(Metadata, Files) -> MetadataBinary = encode_metadata(Metadata), ContentsTarball = create_memory_tarball(Files), ContentsTarballCompressed = gzip(ContentsTarball), - Checksum = checksum(?VERSION, MetadataBinary, ContentsTarballCompressed), - ChecksumBase16 = encode_base16(Checksum), + InnerChecksum = inner_checksum(?VERSION, MetadataBinary, ContentsTarballCompressed), + InnerChecksumBase16 = encode_base16(InnerChecksum), OuterFiles = [ {"VERSION", ?VERSION}, - {"CHECKSUM", ChecksumBase16}, + {"CHECKSUM", InnerChecksumBase16}, {"metadata.config", MetadataBinary}, {"contents.tar.gz", ContentsTarballCompressed} ], Tarball = create_memory_tarball(OuterFiles), + OuterChecksum = checksum(Tarball), UncompressedSize = byte_size(ContentsTarball), @@ -67,7 +70,7 @@ create(Metadata, Files) -> {error, {tarball, too_big}}; false -> - {ok, {Tarball, Checksum}} + {ok, #{tarball => Tarball, outer_checksum => OuterChecksum, inner_checksum => InnerChecksum}} end. %% @doc @@ -77,19 +80,15 @@ create(Metadata, Files) -> %% %% ``` %% > Files = [{"doc/index.html", <<"Docs">>}], -%% > {ok, {Tarball, Checksum}} = r3_hex_tarball:create_docs(Files). -%% > Tarball. -%% %%=> <<86,69,...>> -%% > Checksum. -%% %%=> <<40,32,...>> +%% > r3_hex_tarball:create_docs(Files). +%% {ok, <<86,69,...>>} %% ''' %% @end --spec create_docs(files()) -> {ok, {tarball(), checksum()}}. +-spec create_docs(files()) -> {ok, tarball()}. create_docs(Files) -> UncompressedTarball = create_memory_tarball(Files), UncompressedSize = byte_size(UncompressedTarball), Tarball = gzip(UncompressedTarball), - Checksum = checksum(Tarball), Size = byte_size(Tarball), case(Size > ?TARBALL_MAX_SIZE) or (UncompressedSize > ?TARBALL_MAX_UNCOMPRESSED_SIZE) of @@ -97,22 +96,25 @@ create_docs(Files) -> {error, {tarball, too_big}}; false -> - {ok, {Tarball, Checksum}} + {ok, Tarball} end. %% @doc %% Unpacks a package tarball. %% +%% Remember to verify the outer tarball checksum against the registry checksum +%% returned from `r3_hex_repo:get_package(Config, Package)`. +%% %% Examples: %% %% ``` %% > r3_hex_tarball:unpack(Tarball, memory). -%% {ok,#{checksum => <<...>>, +%% {ok,#{outer_checksum => <<...>>, %% contents => [{"src/foo.erl",<<"-module(foo).">>}], %% metadata => #{<<"name">> => <<"foo">>, ...}}} %% %% > r3_hex_tarball:unpack(Tarball, "path/to/unpack"). -%% {ok,#{checksum => <<...>>, +%% {ok,#{outer_checksum => <<...>>, %% metadata => #{<<"name">> => <<"foo">>, ...}}} %% ''' -spec unpack(tarball(), memory) -> @@ -130,12 +132,33 @@ unpack(Tarball, Output) -> {error, {tarball, empty}}; {ok, FileList} -> - do_unpack(maps:from_list(FileList), Output); + OuterChecksum = crypto:hash(sha256, Tarball), + do_unpack(maps:from_list(FileList), OuterChecksum, Output); {error, Reason} -> {error, {tarball, Reason}} end. +%% @doc +%% Unpacks a documentation tarball. +%% +%% Examples: +%% +%% ``` +%% > r3_hex_tarball:unpack_docs(Tarball, memory). +%% {ok, [{"index.html", <<"">>}, ...]} +%% +%% > r3_hex_tarball:unpack_docs(Tarball, "path/to/unpack"). +%% ok +%% ''' +-spec unpack_docs(tarball(), memory) -> {ok, contents()} | {error, term()}; + (tarball(), filename()) -> ok | {error, term()}. +unpack_docs(Tarball, _) when byte_size(Tarball) > ?TARBALL_MAX_SIZE -> + {error, {tarball, too_big}}; + +unpack_docs(Tarball, Output) -> + unpack_tarball(Tarball, Output). + %% @doc %% Returns base16-encoded representation of checksum. -spec format_checksum(checksum()) -> binary(). @@ -148,7 +171,6 @@ format_checksum(Checksum) -> format_error({tarball, empty}) -> "empty tarball"; format_error({tarball, too_big}) -> "tarball is too big"; format_error({tarball, {missing_files, Files}}) -> io_lib:format("missing files: ~p", [Files]); -format_error({tarball, {invalid_files, Files}}) -> io_lib:format("invalid files: ~p", [Files]); format_error({tarball, {bad_version, Vsn}}) -> io_lib:format("unsupported version: ~p", [Vsn]); format_error({tarball, invalid_checksum}) -> "invalid tarball checksum"; format_error({tarball, Reason}) -> "tarball error, " ++ r3_hex_erl_tar:format_error(Reason); @@ -168,13 +190,12 @@ format_error({checksum_mismatch, ExpectedChecksum, ActualChecksum}) -> %% Internal functions %%==================================================================== -checksum(Version, MetadataBinary, ContentsBinary) -> +inner_checksum(Version, MetadataBinary, ContentsBinary) -> Blob = <>, crypto:hash(sha256, Blob). -checksum(ContentsBinary) -> - Blob = <>, - crypto:hash(sha256, Blob). +checksum(ContentsBinary) when is_binary(ContentsBinary) -> + crypto:hash(sha256, ContentsBinary). encode_metadata(Meta) -> Data = lists:map( @@ -184,9 +205,10 @@ encode_metadata(Meta) -> end, maps:to_list(Meta)), iolist_to_binary(Data). -do_unpack(Files, Output) -> +do_unpack(Files, OuterChecksum, Output) -> State = #{ - checksum => undefined, + inner_checksum => undefined, + outer_checksum => OuterChecksum, contents => undefined, files => Files, metadata => undefined, @@ -194,23 +216,23 @@ do_unpack(Files, Output) -> }, State1 = check_files(State), State2 = check_version(State1), - State3 = check_checksum(State2), + State3 = check_inner_checksum(State2), State4 = decode_metadata(State3), finish_unpack(State4). finish_unpack({error, _} = Error) -> Error; -finish_unpack(#{metadata := Metadata, files := Files, output := Output}) -> - true = maps:is_key("VERSION", Files), - Checksum = decode_base16(maps:get("CHECKSUM", Files)), +finish_unpack(#{metadata := Metadata, files := Files, inner_checksum := InnerChecksum, outer_checksum := OuterChecksum, output := Output}) -> + _Version = maps:get("VERSION", Files), ContentsBinary = maps:get("contents.tar.gz", Files), + filelib:ensure_dir(filename:join(Output, "*")), case unpack_tarball(ContentsBinary, Output) of ok -> copy_metadata_config(Output, maps:get("metadata.config", Files)), - {ok, #{checksum => Checksum, metadata => Metadata}}; + {ok, #{inner_checksum => InnerChecksum, outer_checksum => OuterChecksum, metadata => Metadata}}; {ok, Contents} -> - {ok, #{checksum => Checksum, metadata => Metadata, contents => Contents}}; + {ok, #{inner_checksum => InnerChecksum, outer_checksum => OuterChecksum, metadata => Metadata, contents => Contents}}; {error, Reason} -> {error, {inner_tarball, Reason}} @@ -226,10 +248,7 @@ check_files(#{files := Files} = State) -> State; {error, {missing_keys, Keys}} -> - {error, {tarball, {missing_files, Keys}}}; - - {error, {unknown_keys, Keys}} -> - {error, {tarball, {invalid_files, Keys}}} + {error, {tarball, {missing_files, Keys}}} end. check_version({error, _} = Error) -> @@ -243,26 +262,27 @@ check_version(#{files := Files} = State) -> {error, {tarball, {bad_version, Version}}} end. -check_checksum({error, _} = Error) -> +% Note: This checksum is deprecated +check_inner_checksum({error, _} = Error) -> Error; -check_checksum(#{files := Files} = State) -> +check_inner_checksum(#{files := Files} = State) -> ChecksumBase16 = maps:get("CHECKSUM", Files), ExpectedChecksum = decode_base16(ChecksumBase16), Version = maps:get("VERSION", Files), MetadataBinary = maps:get("metadata.config", Files), ContentsBinary = maps:get("contents.tar.gz", Files), - ActualChecksum = checksum(Version, MetadataBinary, ContentsBinary), + ActualChecksum = inner_checksum(Version, MetadataBinary, ContentsBinary), if byte_size(ExpectedChecksum) /= 32 -> - {error, {tarball, invalid_checksum}}; + {error, {tarball, invalid_inner_checksum}}; ExpectedChecksum == ActualChecksum -> - maps:put(checksum, ExpectedChecksum, State); + maps:put(inner_checksum, ExpectedChecksum, State); true -> - {error, {tarball, {checksum_mismatch, ExpectedChecksum, ActualChecksum}}} + {error, {tarball, {inner_checksum_mismatch, ExpectedChecksum, ActualChecksum}}} end. decode_metadata({error, _} = Error) -> @@ -343,6 +363,7 @@ guess_build_tools(Metadata) -> unpack_tarball(ContentsBinary, memory) -> r3_hex_erl_tar:extract({binary, ContentsBinary}, [memory, compressed]); unpack_tarball(ContentsBinary, Output) -> + filelib:ensure_dir(filename:join(Output, "*")), case r3_hex_erl_tar:extract({binary, ContentsBinary}, [{cwd, Output}, compressed]) of ok -> [try_updating_mtime(filename:join(Output, Path)) || Path <- filelib:wildcard("**", Output)], @@ -466,8 +487,9 @@ diff_keys(Map, RequiredKeys, OptionalKeys) -> {[], []} -> ok; - {_, [_ | _]} -> - {error, {unknown_keys, UnknownKeys}}; + % Server should validate this but clients should not + % {_, [_ | _]} -> + % {error, {unknown_keys, UnknownKeys}}; _ -> {error, {missing_keys, MissingKeys}} diff --git a/src/r3_safe_erl_term.xrl b/src/r3_safe_erl_term.xrl index 6ac1dae29..e938905a2 100644 --- a/src/r3_safe_erl_term.xrl +++ b/src/r3_safe_erl_term.xrl @@ -1,4 +1,4 @@ -%% Vendored from hex_core v0.5.1, do not edit manually +%% Vendored from hex_core v0.6.8, do not edit manually %%% Author : Robert Virding %%% Purpose : Token definitions for Erlang. @@ -9,7 +9,7 @@ D = [0-9] U = [A-Z] L = [a-z] A = ({U}|{L}|{D}|_|@) -WS = ([\000-\s]|%.*) +WS = ([\000-\s]) Rules. diff --git a/src/rebar.hrl b/src/rebar.hrl index fedf0d0ac..7d8b6a4cd 100644 --- a/src/rebar.hrl +++ b/src/rebar.hrl @@ -22,12 +22,12 @@ -define(DEFAULT_PLUGINS_DIR, "plugins"). -define(DEFAULT_TEST_DEPS_DIR, "test/lib"). -define(DEFAULT_RELEASE_DIR, "rel"). --define(CONFIG_VERSION, "1.1.0"). +-define(CONFIG_VERSION, "1.2.0"). -define(DEFAULT_CDN, "https://repo.hex.pm/"). -define(REMOTE_PACKAGE_DIR, "tarballs"). -define(LOCK_FILE, "rebar.lock"). -define(DEFAULT_COMPILER_SOURCE_FORMAT, relative). --define(PACKAGE_INDEX_VERSION, 5). +-define(PACKAGE_INDEX_VERSION, 6). -define(PACKAGE_TABLE, package_index). -define(INDEX_FILE, "packages.idx"). -define(HEX_AUTH_FILE, "hex.config"). @@ -45,7 +45,8 @@ %% TODO: change package and requirement keys to be required (:=) after dropping support for OTP-18 -record(package, {key :: {unicode:unicode_binary() | ms_field(), unicode:unicode_binary() | ms_field(), unicode:unicode_binary() | ms_field()}, - checksum :: binary() | ms_field(), + inner_checksum :: binary() | ms_field(), + outer_checksum :: binary() | ms_field(), retired :: boolean() | ms_field(), dependencies :: [#{package => unicode:unicode_binary(), requirement => unicode:unicode_binary()}] | ms_field()}). diff --git a/src/rebar_app_utils.erl b/src/rebar_app_utils.erl index 626f6aa4d..0b16ae056 100644 --- a/src/rebar_app_utils.erl +++ b/src/rebar_app_utils.erl @@ -207,7 +207,7 @@ parse_dep(Dep, Parent, DepsDir, State, Locks, Level) -> parse_dep(Parent, {Name, Vsn, {pkg, PkgName}}, DepsDir, IsLock, State) -> {PkgName1, PkgVsn} = {rebar_utils:to_binary(PkgName), rebar_utils:to_binary(Vsn)}, - dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn, undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, Name, PkgVsn, {pkg, PkgName1, PkgVsn, undefined ,undefined}, IsLock, State); parse_dep(Parent, {Name, {pkg, PkgName}}, DepsDir, IsLock, State) -> %% Package dependency with different package name from app name dep_to_app(Parent, DepsDir, Name, undefined, {pkg, rebar_utils:to_binary(PkgName), undefined, undefined}, IsLock, State); @@ -215,7 +215,7 @@ parse_dep(Parent, {Name, Vsn}, DepsDir, IsLock, State) when is_list(Vsn); is_bin %% Versioned Package dependency {PkgName, PkgVsn} = {rebar_utils:to_binary(Name), rebar_utils:to_binary(Vsn)}, - dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn, undefined}, IsLock, State); + dep_to_app(Parent, DepsDir, PkgName, PkgVsn, {pkg, PkgName, PkgVsn, undefined, undefined}, IsLock, State); parse_dep(Parent, Name, DepsDir, IsLock, State) when is_atom(Name); is_binary(Name) -> %% Unversioned package dependency dep_to_app(Parent, DepsDir, rebar_utils:to_binary(Name), undefined, {pkg, rebar_utils:to_binary(Name), undefined, undefined}, IsLock, State); @@ -232,9 +232,9 @@ parse_dep(Parent, {Name, Source, Opts}, DepsDir, IsLock, State) when is_tuple(So ?WARN("Dependency option list ~p in ~p is not supported and will be ignored", [Opts, Name]), dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); parse_dep(Parent, {Name, {pkg, PkgName, Vsn}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> - dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn, undefined}, IsLock, State); -parse_dep(Parent, {Name, {pkg, PkgName, Vsn, Hash}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> - dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn, Hash}, IsLock, State); + dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn, undefined, undefined}, IsLock, State); +parse_dep(Parent, {Name, {pkg, PkgName, Vsn, OldHash, Hash}, Level}, DepsDir, IsLock, State) when is_integer(Level) -> + dep_to_app(Parent, DepsDir, Name, Vsn, {pkg, PkgName, Vsn, OldHash, Hash}, IsLock, State); parse_dep(Parent, {Name, Source, Level}, DepsDir, IsLock, State) when is_tuple(Source) , is_integer(Level) -> dep_to_app(Parent, DepsDir, Name, [], Source, IsLock, State); @@ -287,16 +287,19 @@ expand_deps_sources(Dep, State) -> rebar_app_info:t() when Source :: rebar_resource_v2:source(). update_source(AppInfo, {pkg, PkgName, PkgVsn, Hash}, State) -> - case rebar_packages:resolve_version(PkgName, PkgVsn, Hash, + update_source(AppInfo, {pkg, PkgName, PkgVsn, undefined, Hash}, State); +update_source(AppInfo, {pkg, PkgName, PkgVsn, OldHash, Hash}, State) -> + case rebar_packages:resolve_version(PkgName, PkgVsn, OldHash, Hash, ?PACKAGE_TABLE, State) of {ok, Package, RepoConfig} -> #package{key={_, PkgVsn1, _}, - checksum=Hash1, + inner_checksum=OldHash1, + outer_checksum=Hash1, dependencies=Deps, retired=Retired} = Package, maybe_warn_retired(PkgName, PkgVsn1, Hash, Retired), PkgVsn2 = list_to_binary(lists:flatten(ec_semver:format(PkgVsn1))), - AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, Hash1, RepoConfig}), + AppInfo1 = rebar_app_info:source(AppInfo, {pkg, PkgName, PkgVsn2, OldHash1, Hash1, RepoConfig}), rebar_app_info:update_opts_deps(AppInfo1, Deps); not_found -> throw(?PRV_ERROR({missing_package, PkgName, PkgVsn})); diff --git a/src/rebar_config.erl b/src/rebar_config.erl index 2651ca10b..aa5c0ca9d 100644 --- a/src/rebar_config.erl +++ b/src/rebar_config.erl @@ -34,7 +34,6 @@ ,write_lock_file/2 ,verify_config_format/1 ,format_error/1 - ,merge_locks/2]). -include("rebar.hrl"). @@ -105,7 +104,6 @@ warn_vsn_once() -> "upgrade Rebar3.", []) end. - %% @doc Converts the internal format for locks into the multi-version %% compatible one used within rebar3 lock files. %% @end @@ -132,6 +130,9 @@ write_lock_file(LockFile, Locks) -> format_attrs([]) -> []; format_attrs([{pkg_hash, Vals}|T]) -> [io_lib:format("{pkg_hash,[~n",[]), format_hashes(Vals), "]}", + maybe_comma(T) | format_attrs(T)]; +format_attrs([{pkg_hash_ext, Vals}|T]) -> + [io_lib:format("{pkg_hash_ext,[~n",[]), format_hashes(Vals), "]}", maybe_comma(T) | format_attrs(T)]. %% @private format hashing in order to disturb source diffing as little @@ -156,29 +157,31 @@ maybe_comma([_|_]) -> io_lib:format(",~n", []). read_attrs(_Vsn, Locks, Attrs) -> %% Beta copy does not know how to expand attributes, but %% is ready to support it. - expand_locks(Locks, extract_pkg_hashes(Attrs)). + {OldHashes, NewHashes} = extract_pkg_hashes(Attrs), + expand_locks(Locks, OldHashes, NewHashes). %% @private extract the package hashes from lockfile attributes, if any. --spec extract_pkg_hashes(list()) -> [binary()]. +-spec extract_pkg_hashes(list()) -> {[binary()], [binary()]}. extract_pkg_hashes(Attrs) -> Props = case Attrs of [First|_] -> First; [] -> [] end, - proplists:get_value(pkg_hash, Props, []). + { proplists:get_value(pkg_hash, Props, []), proplists:get_value(pkg_hash_ext, Props, [])}. %% @private extract attributes from the lock file and integrate them %% into the full-blow internal lock format %% @end %% TODO: refine typings for lock() --spec expand_locks(list(), list()) -> list(). -expand_locks([], _Hashes) -> +-spec expand_locks(list(), list(), list()) -> list(). +expand_locks([], _OldHashes, _NewHashes) -> []; -expand_locks([{Name, {pkg,PkgName,Vsn}, Lvl} | Locks], Hashes) -> - Hash = proplists:get_value(Name, Hashes), - [{Name, {pkg,PkgName,Vsn,Hash}, Lvl} | expand_locks(Locks, Hashes)]; -expand_locks([Lock|Locks], Hashes) -> - [Lock | expand_locks(Locks, Hashes)]. +expand_locks([{Name, {pkg,PkgName,Vsn}, Lvl} | Locks], OldHashes, NewHashes) -> + OldHash = proplists:get_value(Name, OldHashes), + NewHash = proplists:get_value(Name, NewHashes), + [{Name, {pkg,PkgName,Vsn,OldHash, NewHash}, Lvl} | expand_locks(Locks, OldHashes, NewHashes)]; +expand_locks([Lock|Locks], OldHashes, NewHashes) -> + [Lock | expand_locks(Locks, OldHashes, NewHashes)]. %% @private split up extra attributes for locks out of the internal lock %% structure for backwards compatibility reasons @@ -186,23 +189,28 @@ expand_locks([Lock|Locks], Hashes) -> write_attrs(Locks) -> %% No attribute known that needs to be taken out of the structure, %% just return terms as is. - {NewLocks, Hashes} = split_locks(Locks, [], []), - case Hashes of - [] -> {NewLocks, []}; - _ -> {NewLocks, [{pkg_hash, lists:sort(Hashes)}]} + {NewLocks, OldHashes, NewHashes} = split_locks(Locks, [], [], []), + case {OldHashes, NewHashes} of + {[], []} -> {NewLocks, []}; + _ -> + {NewLocks, [{pkg_hash, lists:sort(OldHashes)}, {pkg_hash_ext, lists:sort(NewHashes)}]} end. %% @private split up extra attributes for locks out of the internal lock %% structure for backwards compatibility reasons --spec split_locks(list(), list(), [{_,binary()}]) -> {list(), list()}. -split_locks([], Locks, Hashes) -> - {lists:reverse(Locks), Hashes}; -split_locks([{Name, {pkg,PkgName,Vsn,undefined}, Lvl} | Locks], LAcc, HAcc) -> - split_locks(Locks, [{Name,{pkg,PkgName,Vsn},Lvl}|LAcc], HAcc); -split_locks([{Name, {pkg,PkgName,Vsn,Hash}, Lvl} | Locks], LAcc, HAcc) -> - split_locks(Locks, [{Name,{pkg,PkgName,Vsn},Lvl}|LAcc], [{Name, Hash}|HAcc]); -split_locks([Lock|Locks], LAcc, HAcc) -> - split_locks(Locks, [Lock|LAcc], HAcc). +-spec split_locks(list(), list(), [{_,binary()}], [{_,binary()}]) -> {list(), list(), list()}. +split_locks([], Locks, OldHashes, NewHashes) -> + {lists:reverse(Locks), OldHashes, NewHashes}; +split_locks([{Name, {pkg,PkgName,Vsn,undefined}, Lvl} | Locks], LAcc, OldHAcc, NewHAcc) -> + split_locks(Locks, [{Name,{pkg,PkgName,Vsn},Lvl}|LAcc], OldHAcc, NewHAcc); +split_locks([{Name, {pkg,PkgName,Vsn,undefined, undefined}, Lvl} | Locks], LAcc, OldHAcc, NewHAcc) -> + split_locks(Locks, [{Name,{pkg,PkgName,Vsn},Lvl}|LAcc], OldHAcc, NewHAcc); +split_locks([{Name, {pkg,PkgName,Vsn, OldHash}, Lvl} | Locks], LAcc, OldHAcc, NewHAcc) -> + split_locks(Locks, [{Name,{pkg,PkgName, Vsn},Lvl}|LAcc], [{Name, OldHash}|OldHAcc], NewHAcc); +split_locks([{Name, {pkg,PkgName,Vsn, OldHash, NewHash}, Lvl} | Locks], LAcc, OldHAcc, NewHAcc) -> + split_locks(Locks, [{Name,{pkg,PkgName,Vsn},Lvl}|LAcc], [{Name, OldHash}|OldHAcc], [{Name, NewHash}|NewHAcc]); +split_locks([Lock|Locks], LAcc, OldHAcc, NewHAcc) -> + split_locks(Locks, [Lock|LAcc], OldHAcc, NewHAcc). %% @doc reads a given config file, including the `.script' variations, %% if any can be found, and asserts that the config format is in diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index c56a36235..c3bbeaf17 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -8,7 +8,7 @@ ,verify_table/1 ,format_error/1 ,update_package/3 - ,resolve_version/5]). + ,resolve_version/6]). -ifdef(TEST). -export([new_package_table/0, find_highest_matching_/5, cmp_/4, cmpl_/4, valid_vsn/1]). @@ -77,7 +77,7 @@ get_package(Dep, Vsn, Hash, Repos, Table, State) -> MatchingPackages = ets:select(Table, [{#package{key={Dep, ec_semver:parse(Vsn), Repo}, _='_'}, [], ['$_']} || Repo <- Repos]), PackagesWithProperHash = lists:filter( - fun(#package{key = {_Dep, _Vsn, Repo}, checksum = PkgChecksum}) -> + fun(#package{key = {_Dep, _Vsn, Repo}, outer_checksum = PkgChecksum}) -> if (PkgChecksum =/= Hash) andalso (Hash =/= '_') -> ?WARN("Checksum mismatch for package ~ts-~ts from repo ~ts", [Dep, Vsn, Repo]), false; @@ -230,7 +230,7 @@ verify_table(State) -> ets:info(?PACKAGE_TABLE, named_table) =:= true orelse load_and_verify_version(State). parse_deps(Deps) -> - [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} + [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined, undefined}} || D=#{package := Name, requirement := Constraint} <- Deps]. @@ -280,21 +280,24 @@ unverified_repo_message() -> insert_releases(Name, Releases, Repo, Table) -> [true = ets:insert(Table, #package{key={Name, ec_semver:parse(Version), Repo}, - checksum=parse_checksum(Checksum), + inner_checksum=parse_checksum(InnerChecksum), + outer_checksum=parse_checksum(OuterChecksum), retired=maps:get(retired, Release, false), dependencies=parse_deps(Dependencies)}) - || Release=#{checksum := Checksum, + || Release=#{inner_checksum := InnerChecksum, + outer_checksum := OuterChecksum, version := Version, dependencies := Dependencies} <- Releases]. -spec resolve_version(unicode:unicode_binary(), unicode:unicode_binary() | undefined, + binary() | undefined, binary() | undefined, ets:tab(), rebar_state:t()) -> {error, {invalid_vsn, unicode:unicode_binary()}} | not_found | {ok, #package{}, map()}. %% if checksum is defined search for any matching repo matching pkg-vsn and checksum -resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> +resolve_version(Dep, DepVsn, _OldHash, Hash, HexRegistry, State) when is_binary(Hash) -> Resources = rebar_state:resources(State), #{repos := RepoConfigs} = rebar_resource_v2:find_resource_state(pkg, Resources), RepoNames = [RepoName || #{name := RepoName} <- RepoConfigs], @@ -315,7 +318,7 @@ resolve_version(Dep, DepVsn, Hash, HexRegistry, State) when is_binary(Hash) -> end, handle_missing_no_exception(Fun, Dep, State) end; -resolve_version(Dep, undefined, Hash, HexRegistry, State) -> +resolve_version(Dep, undefined, _OldHash, Hash, HexRegistry, State) -> Fun = fun(Repo) -> case highest_matching(Dep, {0,{[],[]}}, Repo, HexRegistry, State) of none -> @@ -325,7 +328,7 @@ resolve_version(Dep, undefined, Hash, HexRegistry, State) -> end end, handle_missing_no_exception(Fun, Dep, State); -resolve_version(Dep, DepVsn, Hash, HexRegistry, State) -> +resolve_version(Dep, DepVsn, _OldHash, Hash, HexRegistry, State) -> case valid_vsn(DepVsn) of false -> {error, {invalid_vsn, DepVsn}}; diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 5de24374c..54688424b 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -45,8 +45,8 @@ init(Type, State) -> ResourceState :: rebar_resource_v2:resource_state(), Res :: {atom(), string(), any(), binary()}. lock(AppInfo, _) -> - {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), - {pkg, Name, Vsn, Hash}. + {pkg, Name, Vsn, OldHash, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), + {pkg, Name, Vsn, OldHash, Hash}. %%------------------------------------------------------------------------------ %% @doc @@ -59,7 +59,7 @@ lock(AppInfo, _) -> ResourceState :: rebar_resource_v2:resource_state(), Res :: boolean(). needs_update(AppInfo, _) -> - {pkg, _Name, Vsn, _Hash, _} = rebar_app_info:source(AppInfo), + {pkg, _Name, Vsn, _OldHash, _Hash, _} = rebar_app_info:source(AppInfo), case rebar_utils:to_binary(rebar_app_info:original_vsn(AppInfo)) =:= rebar_utils:to_binary(Vsn) of true -> false; @@ -101,7 +101,7 @@ download(TmpDir, AppInfo, State, ResourceState) -> UpdateETag :: boolean(), Res :: ok | {error,_} | {unexpected_hash, string(), integer(), integer()} | {fetch_fail, binary(), binary()}. -download(TmpDir, Pkg={pkg, Name, Vsn, _Hash, Repo}, State, _ResourceState, UpdateETag) -> +download(TmpDir, Pkg={pkg, Name, Vsn, _OldHash, _Hash, Repo}, State, _ResourceState, UpdateETag) -> {ok, PackageDir} = rebar_packages:package_dir(Repo, State), Package = binary_to_list(<>), ETagFile = binary_to_list(<>), @@ -214,7 +214,7 @@ store_etag_in_cache(Path, ETag) -> ETagPath :: file:name(), UpdateETag :: boolean(), Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()}. -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _Hash, RepoConfig}, ETag, +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _OldHash, _Hash, RepoConfig}, ETag, ETagPath, UpdateETag) -> case request(RepoConfig, Name, Vsn, ETag) of {ok, cached} -> @@ -246,19 +246,35 @@ serve_from_cache(TmpDir, CachePath, Pkg) -> Tarball :: binary(), Package :: package(), Res :: ok | {error,_} | {bad_registry_checksum, integer(), integer()}. -serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, Hash, _RepoConfig}) -> +serve_from_memory(TmpDir, Binary, {pkg, _Name, _Vsn, OldHash, Hash, _RepoConfig}) -> RegistryChecksum = list_to_integer(binary_to_list(Hash), 16), + OldRegistryChecksum = maybe_old_registry_checksum(OldHash), case r3_hex_tarball:unpack(Binary, TmpDir) of - {ok, #{checksum := <>}} when RegistryChecksum =/= Checksum -> - ?DEBUG("Expected hash ~64.16.0B does not match checksum of fetched package ~64.16.0B", - [RegistryChecksum, Checksum]), - {bad_registry_checksum, RegistryChecksum, Checksum}; - {ok, #{checksum := <>}} -> + {ok, #{outer_checksum := <>} = Res} when RegistryChecksum =/= Checksum -> + #{inner_checksum := <>} = Res, + %% Not triggerable in tests, but code feels logically wrong without it since inner checksums are not hard + %% deprecated. This logic should be removed when inner checksums do become hard deprecated and/or no longer + %% supported by rebar3. + case OldRegistryChecksum == OldChecksum of + true -> + ?DEBUG("Expected hash ~64.16.0B does not match outer checksum of fetched package ~64.16.0B, but + matches inner checksum ~64.16.0B", + [RegistryChecksum, Checksum, OldChecksum]), + {bad_registry_checksum, RegistryChecksum, OldChecksum}; + false -> + ?DEBUG("Expected hash ~64.16.0B does not match outer checksum or inner checksum of fetched package + ~64.16.0B / ~64.16.0B", [RegistryChecksum, Checksum, OldChecksum]), + {bad_registry_checksum, RegistryChecksum, Checksum} + end; + {ok, #{outer_checksum := <>}} -> ok; {error, Reason} -> {error, {hex_tarball, Reason}} end. +maybe_old_registry_checksum(undefined) -> undefined; +maybe_old_registry_checksum(Hash) -> list_to_integer(binary_to_list(Hash), 16). + -spec serve_from_download(TmpDir, CachePath, Package, Binary) -> Res when TmpDir :: file:name(), CachePath :: file:name(), diff --git a/src/rebar_resource_v2.erl b/src/rebar_resource_v2.erl index 537b5f025..c036a005c 100644 --- a/src/rebar_resource_v2.erl +++ b/src/rebar_resource_v2.erl @@ -70,7 +70,7 @@ find_resource_state(Type, Resources) -> format_source(AppInfo) -> Name = rebar_app_info:name(AppInfo), case rebar_app_info:source(AppInfo) of - {pkg, _Name, Vsn, _Hash, _} -> + {pkg, _Name, Vsn, _OldHash, _Hash, _} -> io_lib:format("~ts v~s", [Name, Vsn]); Source -> io_lib:format("~ts (from ~p)", [Name, Source]) @@ -139,7 +139,7 @@ get_resource_type({Type, Location, _}, Resources) -> get_resource(Type, Location, Resources); get_resource_type({Type, _, _, Location}, Resources) -> get_resource(Type, Location, Resources); -get_resource_type(Location={Type, _, _, _, _}, Resources) -> +get_resource_type(Location={Type, _, _, _, _, _}, Resources) -> get_resource(Type, Location, Resources); get_resource_type(Source, _) -> throw(?PRV_ERROR({no_resource, Source})). diff --git a/test/mock_pkg_resource.erl b/test/mock_pkg_resource.erl index a176e88b9..ec571f3c6 100644 --- a/test/mock_pkg_resource.erl +++ b/test/mock_pkg_resource.erl @@ -24,9 +24,10 @@ mock() -> mock([]). | {not_in_index, [{App, Vsn}]} | {pkgdeps, [{{App,Vsn}, [Dep]}]}, App :: string(), - Dep :: {App, string(), {pkg, App, Vsn, Hash}}, + Dep :: {App, string(), {pkg, App, Vsn, InnerHash, OuterHash}}, Vsn :: string(), - Hash :: string() | undefined. + InnerHash :: string() | undefined, + OuterHash :: string() | undefined. mock(Opts) -> meck:new(?MOD, [no_link, passthrough]), mock_lock(Opts), @@ -47,8 +48,8 @@ unmock() -> %% @doc creates values for a lock file. mock_lock(_) -> meck:expect(?MOD, lock, fun(AppInfo, _) -> - {pkg, Name, Vsn, Hash, _RepoConfig} = rebar_app_info:source(AppInfo), - {pkg, Name, Vsn, Hash} + {pkg, Name, Vsn, InnerHash, OuterHash, _RepoConfig} = rebar_app_info:source(AppInfo), + {pkg, Name, Vsn, InnerHash, OuterHash} end). %% @doc The config passed to the `mock/2' function can specify which apps @@ -58,7 +59,7 @@ mock_update(Opts) -> meck:expect( ?MOD, needs_update, fun(AppInfo, _) -> - {pkg, App, _Vsn, _Hash, _} = rebar_app_info:source(AppInfo), + {pkg, App, _Vsn, _InnerHash, _OuterHash, _} = rebar_app_info:source(AppInfo), lists:member(binary_to_list(App), ToUpdate) end). @@ -84,7 +85,7 @@ mock_download(Opts) -> meck:expect( ?MOD, download, fun (Dir, AppInfo, _, _) -> - {pkg, AppBin, Vsn, _, _} = rebar_app_info:source(AppInfo), + {pkg, AppBin, Vsn, _, _, _} = rebar_app_info:source(AppInfo), App = rebar_utils:to_list(AppBin), filelib:ensure_dir(Dir), AppDeps = proplists:get_value({App,Vsn}, Deps, []), @@ -100,7 +101,7 @@ mock_download(Opts) -> <<"version">> => Vsn}, Files = all_files(rebar_app_info:dir(AppInfo1)), - {ok, {Tarball, _Checksum}} = r3_hex_tarball:create(Metadata, archive_names(Dir, Files)), + {ok, #{tarball := Tarball}} = r3_hex_tarball:create(Metadata, archive_names(Dir, Files)), Archive = filename:join([Dir, TarApp]), file:write_file(Archive, Tarball), @@ -153,8 +154,9 @@ find_parts([{AppName, Deps}|Rest], Skip, Acc) -> Acc), find_parts(Rest, Skip, AccNew) end. + parse_deps(Deps) -> - [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined}} || D=#{package := Name, + [{maps:get(app, D, Name), {pkg, Name, Constraint, undefined, undefined}} || D=#{package := Name, requirement := Constraint} <- Deps]. to_index(AllDeps, Dict, Repos) -> @@ -166,15 +168,17 @@ to_index(AllDeps, Dict, Repos) -> DepsList = [#{package => DKB, app => DKB, requirement => DVB, - source => {pkg, DKB, DVB, undefined}} + source => {pkg, DKB, DVB, undefined, undefined}} || {DK, DV} <- Deps, DKB <- [ec_cnv:to_binary(DK)], DVB <- [ec_cnv:to_binary(DV)]], Repo = rebar_test_utils:random_element(Repos), + ets:insert(?PACKAGE_TABLE, #package{key={N, ec_semver:parse(V), Repo}, dependencies=parse_deps(DepsList), retired=false, - checksum = <<"checksum">>}) + inner_checksum = <<"inner_checksum">>, + outer_checksum = <<"checksum">>}) end, ok, Dict), lists:foreach(fun({{Name, Vsn}, _}) -> @@ -186,7 +190,8 @@ to_index(AllDeps, Dict, Repos) -> ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(Name), ec_semver:parse(Vsn), Repo}, dependencies=[], retired=false, - checksum = <<"checksum">>}); + inner_checksum = <<"inner_checksum">>, + outer_checksum = <<"checksum">>}); true -> ok end diff --git a/test/rebar_lock_SUITE.erl b/test/rebar_lock_SUITE.erl index f1ab3b527..58b6051d1 100644 --- a/test/rebar_lock_SUITE.erl +++ b/test/rebar_lock_SUITE.erl @@ -24,9 +24,9 @@ current_version(Config) -> {<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2}, {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0}, {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1}, - {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>,undefined},3}, - {<<"pkg2">>,{pkg,<<"name1">>,<<"1.1.6">>,undefined},2}, - {<<"pkg3">>,{pkg,<<"name2">>,<<"3.0.6">>,undefined},1} + {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>,undefined, undefined},3}, + {<<"pkg2">>,{pkg,<<"name1">>,<<"1.1.6">>,undefined, undefined},2}, + {<<"pkg3">>,{pkg,<<"name2">>,<<"3.0.6">>,undefined, undefined},1} ], %% Simulate a beta lockfile file:write_file(LockFile, io_lib:format("~p.~n", [Locks])), @@ -37,22 +37,24 @@ current_version(Config) -> %% Adding hash data Hashes = [{<<"pkg1">>, <<"tarballhash">>}, {<<"pkg3">>, <<"otherhash">>}], + ExtHashes = [{<<"pkg1">>, <<"outer_tarballhash">>}, + {<<"pkg3">>, <<"outer_otherhash">>}], ExpandedLocks = [ {<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2}, {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0}, {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1}, - {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>,<<"tarballhash">>},3}, - {<<"pkg2">>,{pkg,<<"name1">>,<<"1.1.6">>,undefined},2}, - {<<"pkg3">>,{pkg,<<"name2">>,<<"3.0.6">>,<<"otherhash">>},1} + {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>,<<"tarballhash">>, <<"outer_tarballhash">>},3}, + {<<"pkg2">>,{pkg,<<"name1">>,<<"1.1.6">>,undefined, undefined},2}, + {<<"pkg3">>,{pkg,<<"name2">>,<<"3.0.6">>,<<"otherhash">>, <<"outer_otherhash">>},1} ], file:write_file(LockFile, io_lib:format("~p.~n~p.~n", - [{"1.1.0", Locks}, - [{pkg_hash, Hashes}]])), + [{"1.2.0", Locks}, + [{pkg_hash, Hashes}, {pkg_hash_ext, ExtHashes}]])), ?assertEqual(ExpandedLocks, rebar_config:consult_lock_file(LockFile)), %% Then check that we can reverse that ok = rebar_config:write_lock_file(LockFile, ExpandedLocks), - ?assertEqual({ok, [{"1.1.0", Locks}, [{pkg_hash, Hashes}]]}, + ?assertEqual({ok, [{"1.2.0", Locks}, [{pkg_hash, Hashes}, {pkg_hash_ext, ExtHashes}]]}, file:consult(LockFile)). beta_version(Config) -> @@ -66,7 +68,7 @@ beta_version(Config) -> {<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2}, {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0}, {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1}, - {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>,undefined},3} + {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>,undefined, undefined},3} ], file:write_file(LockFile, io_lib:format("~p.~n", [Locks])), ?assertEqual(ExpandedLocks, rebar_config:consult_lock_file(LockFile)). @@ -83,7 +85,7 @@ future_versions_no_attrs(Config) -> ExpandedLocks = [{<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2}, {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0}, {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1}, - {<<"pkg1">>, {pkg,<<"name">>,<<"0.1.6">>,undefined},3}], + {<<"pkg1">>, {pkg,<<"name">>,<<"0.1.6">>,undefined, undefined},3}], LockData = {"3.5.2", Locks}, file:write_file(LockFile, io_lib:format("~p.~n", [LockData])), ?assertEqual(ExpandedLocks, rebar_config:consult_lock_file(LockFile)). @@ -100,13 +102,14 @@ future_versions_attrs(Config) -> ExpandedLocks = [{<<"app1">>, {git,"some_url", {ref,"some_ref"}}, 2}, {<<"app2">>, {git,"some_url", {ref,"some_ref"}}, 0}, {<<"app3">>, {hg,"some_url", {ref,"some_ref"}}, 1}, - {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>, <<"tarballhash">>},3}], + {<<"pkg1">>,{pkg,<<"name">>,<<"0.1.6">>, <<"tarballhash">>, <<"outer_tarballhash">>},3}], Hashes = [{<<"pkg1">>, <<"tarballhash">>}], + ExtHashes = [{<<"pkg1">>, <<"outer_tarballhash">>}], LockData = {"3.5.2", Locks}, file:write_file(LockFile, io_lib:format("~p.~n~p.~ngarbage.~n", [LockData, [{a, x}, - {pkg_hash, Hashes}, + {pkg_hash, Hashes},{pkg_hash_ext, ExtHashes}, {b, y}]])), ?assertEqual(ExpandedLocks, rebar_config:consult_lock_file(LockFile)). diff --git a/test/rebar_pkg_SUITE.erl b/test/rebar_pkg_SUITE.erl index 383f9ef17..9c870cfca 100644 --- a/test/rebar_pkg_SUITE.erl +++ b/test/rebar_pkg_SUITE.erl @@ -11,7 +11,7 @@ -define(badpkg_checksum, <<"A14E3718B33F8124E98004433193509EC6660F6CA03302657CAB8785751D77A0">>). -define(badindex_checksum, <<"7B2CBED315C89F3126B5BF553DD7FF0FB5FE94B064888DD1B095CE8BF4B6A16A">>). -define(bad_checksum, <<"D576B442A68C7B92BACDE1EFE9C6E54D8D6C74BDB71D8175B9D3C6EC8C7B62A7">>). --define(good_checksum, <<"12726BDE1F65583A0817A7E8AADCA73F03FD8CB06F01E6CD29117C4A0DA0AFCF">>). +-define(good_checksum, <<"ABA3B638A653A2414BF9DFAF76D90C937C53D1BE5B5D51A990C6FCC3A775C6F">>). -define(BADPKG_ETAG, <<"BADETAG">>). all() -> [good_uncached, good_cached, badpkg, badhash_nocache, @@ -116,7 +116,7 @@ good_uncached(Config) -> {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), ?assertEqual(ok, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)), Cache = ?config(cache_dir, Config), ?assert(filelib:is_regular(filename:join(Cache, <>))). @@ -129,7 +129,7 @@ good_cached(Config) -> ?assert(filelib:is_regular(CachedFile)), {ok, Content} = file:read_file(CachedFile), ?assertEqual(ok, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)), {ok, Content} = file:read_file(CachedFile). @@ -138,7 +138,7 @@ badindexchk(Config) -> {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, ?bad_checksum, #{}}, State, #{}, true)), %% The cached file is there for forensic purposes Cache = ?config(cache_dir, Config), ?assert(filelib:is_regular(filename:join(Cache, <>))). @@ -151,8 +151,8 @@ badpkg(Config) -> CachePath = filename:join(Cache, <>), ETagPath = filename:join(Cache, <>), rebar_pkg_resource:store_etag_in_cache(ETagPath, ?BADPKG_ETAG), - ?assertMatch({error, {hex_tarball, {tarball, {checksum_mismatch, _, _}}}}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?badpkg_checksum, #{}}, State, #{}, false)), + ?assertMatch({error, {hex_tarball, {tarball, {inner_checksum_mismatch, _, _}}}}, + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?badpkg_checksum, ?badpkg_checksum, #{}}, State, #{}, false)), %% The cached/etag files are there for forensic purposes ?assert(filelib:is_regular(ETagPath)), ?assert(filelib:is_regular(CachePath)). @@ -162,7 +162,7 @@ badhash_nocache(Config) -> {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, ?bad_checksum, #{}}, State, #{}, true)), %% The cached file is there for forensic purposes Cache = ?config(cache_dir, Config), ?assert(filelib:is_regular(filename:join(Cache, <>))). @@ -176,7 +176,7 @@ badhash_cache(Config) -> ?assert(filelib:is_regular(CachedFile)), {ok, Content} = file:read_file(CachedFile), ?assertMatch({error, {rebar_pkg_resource, {bad_registry_checksum, _, _, _, _}}}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, #{}}, State, #{}, true)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?bad_checksum, ?bad_checksum, #{}}, State, #{}, true)), %% The cached file is there still, unchanged. ?assert(filelib:is_regular(CachedFile)), ?assertEqual({ok, Content}, file:read_file(CachedFile)). @@ -190,7 +190,7 @@ bad_to_good(Config) -> ?assert(filelib:is_regular(CachedFile)), {ok, Contents} = file:read_file(CachedFile), ?assertEqual(ok, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)), %% Cache has refreshed ?assert({ok, Contents} =/= file:read_file(CachedFile)). @@ -205,7 +205,7 @@ good_disconnect(Config) -> {ok, Content} = file:read_file(CachedFile), rebar_pkg_resource:store_etag_in_cache(ETagFile, ?BADPKG_ETAG), ?assertEqual(ok, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)), + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)), {ok, Content} = file:read_file(CachedFile). bad_disconnect(Config) -> @@ -213,7 +213,7 @@ bad_disconnect(Config) -> {Pkg,Vsn} = ?config(pkg, Config), State = ?config(state, Config), ?assertEqual({fetch_fail, Pkg, Vsn}, - rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, #{}}, State, #{}, true)). + rebar_pkg_resource:download(Tmp, {pkg, Pkg, Vsn, ?good_checksum, ?good_checksum, #{}}, State, #{}, true)). pkgs_provider(Config) -> Config1 = rebar_test_utils:init_rebar_state(Config), @@ -253,13 +253,13 @@ mock_config(Name, Config) -> TmpDir = filename:join([Priv, "tmp", atom_to_list(Name)]), Tid = ets:new(registry_table, [public]), AllDeps = [ - {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, [<<"rebar3">>]]}, - {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}, - {{<<"goodpkg">>,<<"1.0.1">>}, [[], ?good_checksum, [<<"rebar3">>]]}, - {{<<"goodpkg">>,<<"1.1.1">>}, [[], ?good_checksum, [<<"rebar3">>]]}, - {{<<"goodpkg">>,<<"2.0.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}, - {{<<"goodpkg">>,<<"3.0.0-rc.0">>}, [[], ?good_checksum, [<<"rebar3">>]]}, - {{<<"badpkg">>,<<"1.0.0">>}, [[], ?badpkg_checksum, [<<"rebar3">>]]} + {{<<"badindexchk">>,<<"1.0.0">>}, [[], ?bad_checksum, ?bad_checksum, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"1.0.0">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"1.0.1">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"1.1.1">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"2.0.0">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, + {{<<"goodpkg">>,<<"3.0.0-rc.0">>}, [[], ?good_checksum, ?good_checksum, [<<"rebar3">>]]}, + {{<<"badpkg">>,<<"1.0.0">>}, [[], ?badpkg_checksum, ?badpkg_checksum, [<<"rebar3">>]]} ], ets:insert_new(Tid, AllDeps), CacheDir = filename:join([CacheRoot, "hex", "com", "test", "packages"]), @@ -268,13 +268,14 @@ mock_config(Name, Config) -> catch ets:delete(?PACKAGE_TABLE), rebar_packages:new_package_table(), - lists:foreach(fun({{N, Vsn}, [Deps, Checksum, _]}) -> + lists:foreach(fun({{N, Vsn}, [Deps, InnerChecksum, OuterChecksum, _]}) -> case ets:member(?PACKAGE_TABLE, {ec_cnv:to_binary(N), Vsn, <<"hexpm">>}) of false -> ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>}, dependencies=Deps, retired=false, - checksum=Checksum}); + inner_checksum=InnerChecksum, + outer_checksum=OuterChecksum}); true -> ok end @@ -286,10 +287,11 @@ mock_config(Name, Config) -> fun(_Config, PkgName) -> Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}), Releases = - [#{checksum => Checksum, + [#{outer_checksum => OuterChecksum, + inner_checksum => InnerChecksum, version => Vsn, dependencies => Deps} || - {{_, Vsn}, [Deps, Checksum, _]} <- Matches], + {{_, Vsn}, [Deps, InnerChecksum, OuterChecksum, _]} <- Matches], {ok, {200, #{}, Releases}} end), diff --git a/test/rebar_pkg_alias_SUITE.erl b/test/rebar_pkg_alias_SUITE.erl index bfa5058c5..a819252e0 100644 --- a/test/rebar_pkg_alias_SUITE.erl +++ b/test/rebar_pkg_alias_SUITE.erl @@ -232,7 +232,7 @@ mock_config(Name, Config) -> ets:insert(?PACKAGE_TABLE, #package{key={ec_cnv:to_binary(N), ec_semver:parse(Vsn), <<"hexpm">>}, dependencies=[{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps], retired=false, - checksum=Checksum}); + outer_checksum=Checksum}); true -> ok end; @@ -246,7 +246,7 @@ mock_config(Name, Config) -> fun(_Config, PkgName) -> Matches = ets:match_object(Tid, {{PkgName,'_'}, '_'}), Releases = - [#{checksum => Checksum, + [#{outer_checksum => Checksum, version => Vsn, dependencies => [{DAppName, {pkg, DN, DV, undefined}} || {DN, DV, _, DAppName} <- Deps]} || diff --git a/test/rebar_pkg_repos_SUITE.erl b/test/rebar_pkg_repos_SUITE.erl index 8446bbf5d..bb9d9d117 100644 --- a/test/rebar_pkg_repos_SUITE.erl +++ b/test/rebar_pkg_repos_SUITE.erl @@ -22,8 +22,8 @@ init_per_group(resolve_version, Config) -> Hexpm = <<"hexpm">>, Repos = [Repo1, Repo2, Repo3, Hexpm], - Deps = [{"A", "0.1.1", <<"good checksum">>, Repo1, false}, - {"A", "0.1.1", <<"good checksum">>, Repo2, false}, + Deps = [{"A", "0.1.1", <<"inner checksum">>, <<"good outer checksum">>, Repo1, false}, + {"A", "0.1.1", <<"inner checksum">>, <<"good outer checksum">>, Repo2, false}, {"B", "1.0.0", Repo1, false}, {"B", "2.0.0", Repo2, false}, {"B", "1.4.0", Repo3, false}, @@ -31,8 +31,8 @@ init_per_group(resolve_version, Config) -> {"B", "1.4.6", Hexpm, #{reason => 'RETIRED_INVALID'}}, {"B", "1.5.0", Hexpm, false}, {"B", "1.5.6-rc.0", Hexpm, true}, - {"C", "1.3.1", <<"bad checksum">>, Repo1, false}, - {"C", "1.3.1", <<"good checksum">>, Repo2, false}], + {"C", "1.3.1", <<"inner checksum">>, <<"bad outer checksum">>, Repo1, false}, + {"C", "1.3.1", <<"inner checksum">>, <<"good outer checksum">>, Repo2, false}], [{deps, Deps}, {repos, Repos} | Config]; init_per_group(_, Config) -> Config. @@ -317,17 +317,17 @@ use_first_repo_match(Config) -> State = ?config(state, Config), ?assertMatch({ok,{package,{<<"B">>, {{2,0,0}, {[],[]}}, Repo2}, - <<"some checksum">>, false, []}, + <<"inner checksum">>,<<"outer checksum">>, false, []}, #{name := Repo2, http_adapter_config := #{profile := rebar}}}, - rebar_packages:resolve_version(<<"B">>, <<"> 1.4.0">>, undefined, + rebar_packages:resolve_version(<<"B">>, <<"> 1.4.0">>, undefined, undefined, ?PACKAGE_TABLE, State)), ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3}, - <<"some checksum">>, false, []}, + <<"inner checksum">>,<<"outer checksum">>, false, []}, #{name := Repo3, http_adapter_config := #{profile := rebar}}}, - rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, undefined, ?PACKAGE_TABLE, State)). %% tests that even though an easier repo has C-1.3.1 it doesn't use it since its hash is different @@ -335,20 +335,20 @@ use_exact_with_hash(Config) -> State = ?config(state, Config), ?assertMatch({ok,{package,{<<"C">>, {{1,3,1}, {[],[]}}, Repo2}, - <<"good checksum">>, false, []}, + <<"inner checksum">>, <<"good outer checksum">>, false, []}, #{name := Repo2, http_adapter_config := #{profile := rebar}}}, - rebar_packages:resolve_version(<<"C">>, <<"1.3.1">>, <<"good checksum">>, + rebar_packages:resolve_version(<<"C">>, <<"1.3.1">>, <<"inner checksum">>, <<"good outer checksum">>, ?PACKAGE_TABLE, State)). fail_repo_update(Config) -> State = ?config(state, Config), ?assertMatch({ok,{package,{<<"B">>, {{1,4,0}, {[],[]}}, Repo3}, - <<"some checksum">>, false, []}, + <<"inner checksum">>,<<"outer checksum">>, false, []}, #{name := Repo3, http_adapter_config := #{profile := rebar}}}, - rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, undefined, ?PACKAGE_TABLE, State)). ignore_match_in_excluded_repo(Config) -> @@ -356,44 +356,44 @@ ignore_match_in_excluded_repo(Config) -> Repos = ?config(repos, Config), ?assertMatch({ok,{package,{<<"B">>, {{1,4,6}, {[],[]}}, Hexpm}, - <<"some checksum">>, #{reason := 'RETIRED_INVALID'}, []}, + <<"inner checksum">>,<<"outer checksum">>, #{reason := 'RETIRED_INVALID'}, []}, #{name := Hexpm, http_adapter_config := #{profile := rebar}}}, - rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.4.0">>, undefined, undefined, ?PACKAGE_TABLE, State)), [_, Repo2 | _] = Repos, ?assertMatch({ok,{package,{<<"A">>, {{0,1,1}, {[],[]}}, Repo2}, - <<"good checksum">>, false, []}, + <<"inner checksum">>, <<"good outer checksum">>, false, []}, #{name := Repo2, http_adapter_config := #{profile := rebar}}}, - rebar_packages:resolve_version(<<"A">>, <<"0.1.1">>, <<"good checksum">>, + rebar_packages:resolve_version(<<"A">>, <<"0.1.1">>, <<"inner checksum">>, <<"good outer checksum">>, ?PACKAGE_TABLE, State)). optional_prereleases(Config) -> State = ?config(state, Config), ?assertMatch({ok,{package,{<<"B">>, {{1,5,0}, {[],[]}}, Hexpm}, - <<"some checksum">>, false, []}, + <<"inner checksum">>,<<"outer checksum">>, false, []}, #{name := Hexpm, http_adapter_config := #{profile := rebar}}}, - rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, undefined, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, undefined, undefined, ?PACKAGE_TABLE, State)), ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm}, - <<"some checksum">>, true, []}, + <<"inner checksum">>,<<"outer checksum">>, true, []}, #{name := Hexpm, http_adapter_config := #{profile := rebar}}}, - rebar_packages:resolve_version(<<"B">>, <<"1.5.6-rc.0">>, <<"some checksum">>, + rebar_packages:resolve_version(<<"B">>, <<"1.5.6-rc.0">>, <<"inner checksum">>, <<"outer checksum">>, ?PACKAGE_TABLE, State)), %% allow prerelease through configuration State1 = rebar_state:set(State, deps_allow_prerelease, true), ?assertMatch({ok,{package,{<<"B">>, {{1,5,6}, {[<<"rc">>,0],[]}}, Hexpm}, - <<"some checksum">>, true, []}, + <<"inner checksum">>,<<"outer checksum">>, true, []}, #{name := Hexpm, http_adapter_config := #{profile := rebar}}}, - rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, <<"some checksum">>, + rebar_packages:resolve_version(<<"B">>, <<"~> 1.5.0">>, <<"inner checksum">>, <<"outer checksum">>, ?PACKAGE_TABLE, State1)). %% @@ -413,12 +413,14 @@ insert_deps(Deps) -> rebar_utils:to_binary(Repo)}, dependencies=[], retired=Retired, - checksum = <<"some checksum">>}); - ({Name, Version, Checksum, Repo, Retired}) -> + inner_checksum = <<"inner checksum">>, + outer_checksum = <<"outer checksum">>}); + ({Name, Version, InnerChecksum, OuterChecksum, Repo, Retired}) -> ets:insert(?PACKAGE_TABLE, #package{key={rebar_utils:to_binary(Name), ec_semver:parse(Version), rebar_utils:to_binary(Repo)}, dependencies=[], retired=Retired, - checksum = Checksum}) + inner_checksum = InnerChecksum, + outer_checksum = OuterChecksum}) end, Deps). diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index b4f0043a5..bc5d35d74 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -190,21 +190,21 @@ expand_deps(git, [{Name, Vsn, Deps} | Rest]) -> Dep = {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}}, [{Dep, expand_deps(git, Deps)} | expand_deps(git, Rest)]; expand_deps(pkg, [{Name, Deps} | Rest]) -> - Dep = {pkg, Name, "0.0.0", undefined}, + Dep = {pkg, Name, "0.0.0", undefined, undefined}, [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]; expand_deps(pkg, [{Name, Vsn, Deps} | Rest]) -> - Dep = {pkg, Name, Vsn, undefined}, + Dep = {pkg, Name, Vsn, undefined, undefined}, [{Dep, expand_deps(pkg, Deps)} | expand_deps(pkg, Rest)]; expand_deps(mixed, [{Name, Deps} | Rest]) -> Dep = if hd(Name) >= $a, hd(Name) =< $z -> - {pkg, rebar_string:uppercase(Name), "0.0.0", undefined} + {pkg, rebar_string:uppercase(Name), "0.0.0", undefined, undefined} ; hd(Name) >= $A, hd(Name) =< $Z -> {Name, ".*", {git, "https://example.org/user/"++Name++".git", "master"}} end, [{Dep, expand_deps(mixed, Deps)} | expand_deps(mixed, Rest)]; expand_deps(mixed, [{Name, Vsn, Deps} | Rest]) -> Dep = if hd(Name) >= $a, hd(Name) =< $z -> - {pkg, rebar_string:uppercase(Name), Vsn, undefined} + {pkg, rebar_string:uppercase(Name), Vsn, undefined, undefined} ; hd(Name) >= $A, hd(Name) =< $Z -> {Name, Vsn, {git, "https://example.org/user/"++Name++".git", {tag, Vsn}}} end, @@ -218,7 +218,7 @@ expand_deps(mixed, [{Name, Vsn, Deps} | Rest]) -> flat_deps(Deps) -> flat_deps(Deps, [], []). flat_deps([], Src, Pkg) -> {Src, Pkg}; -flat_deps([{{pkg, Name, Vsn, undefined}, PkgDeps} | Rest], Src, Pkg) -> +flat_deps([{{pkg, Name, Vsn, undefined, undefined}, PkgDeps} | Rest], Src, Pkg) -> Current = {{iolist_to_binary(Name), iolist_to_binary(Vsn)}, top_level_deps(PkgDeps)}, {[], FlatPkgDeps} = flat_deps(PkgDeps), @@ -236,7 +236,7 @@ vsn_from_ref({git, _, {_, Vsn}}) -> Vsn; vsn_from_ref({git, _, Vsn}) -> Vsn. top_level_deps([]) -> []; -top_level_deps([{{pkg, Name, Vsn, undefined}, _} | Deps]) -> +top_level_deps([{{pkg, Name, Vsn, undefined, undefined}, _} | Deps]) -> [{list_to_atom(Name), Vsn} | top_level_deps(Deps)]; top_level_deps([{{Name, Vsn, Ref}, _} | Deps]) -> [{list_to_atom(Name), Vsn, Ref} | top_level_deps(Deps)]. @@ -355,10 +355,10 @@ check_results(AppDir, Expected, ProfileRun) -> case lists:keyfind(iolist_to_binary(Name), 1, Locks) of false -> error({lock_not_found, Name}); - {_LockName, {pkg, _, LockVsn, Hash}, _} -> + {_LockName, {pkg, _, LockVsn, _InnerHash, OuterHash}, _} -> ?assertEqual(iolist_to_binary(Vsn), iolist_to_binary(LockVsn)), - ?assertNotEqual(undefined, Hash); + ?assertNotEqual(undefined, OuterHash); {_LockName, {_, _, {ref, LockVsn}}, _} -> ?assertEqual(iolist_to_binary(Vsn), iolist_to_binary(LockVsn)) @@ -368,10 +368,10 @@ check_results(AppDir, Expected, ProfileRun) -> case lists:keyfind(iolist_to_binary(Name), 1, Locks) of false -> error({lock_not_found, Name}); - {_LockName, {pkg, _, LockVsn, Hash}, _} -> + {_LockName, {pkg, _, LockVsn, _InnerHash, OuterHash}, _} -> ?assertEqual(iolist_to_binary(Vsn), iolist_to_binary(LockVsn)), - ?assertNotEqual(undefined, Hash); + ?assertNotEqual(undefined, OuterHash); {_LockName, {_, _, {ref, LockVsn}}, _} -> error({source_lock, {Name, LockVsn}}) end @@ -511,7 +511,7 @@ package_app(AppDir, DestDir, PkgName, PkgVsn) -> Files = lists:zip([filename:join("src", F) || F <- Fs], [filename:join(AppSrc,F) || F <- Fs]), Metadata = #{<<"app">> => list_to_binary(PkgName), <<"version">> => list_to_binary(PkgVsn)}, - {ok, {Tarball, <>}} = r3_hex_tarball:create(Metadata, Files), + {ok, #{tarball := Tarball, outer_checksum := <>}} = r3_hex_tarball:create(Metadata, Files), Name = PkgName++"-"++PkgVsn++".tar", Archive = filename:join(DestDir, Name),