fix #412 add webhooks for instant version updates

http-update.md

@@ -0,0 +1,24 @@
+# Triggering version updates over HTTP
+
+To queue an update of your package you can use the `POST /api/packages/:packageName/update` endpoint.
+
+## `POST /api/packages/:packageName/update`
+
+Queues an update for the specified package.
+
+Query params:
+`secret`: string (optional) provide the secret as query
+`header`: string (optional) provide which header is used to check the secret (must start with X-)
+
+Body params: (application/x-www-form-urlencoded, multipart/form-data or application/json)
+`secret`: string (optional) provide the secret as body param
+
+## `POST /api/packages/:packageName/update/github`
+
+Queues an update for the specified package. Compatible with GitHub webhooks and only triggers on `create` events. Must pass secret as query param and not in GitHub webhook settings.
+
+## `POST /api/packages/:packageName/update/gitlab`
+
+Queues an update for the specified package. Compatible with GitLab webhooks and only triggers on `tag_push` events. The secret is specified in the GitLab control panel.
+
+Calls `POST /api/packages/:packageName/update` with `header=X-Gitlab-Token` query param after hook parsing.

source/dubregistry/api.d

@@ -96,6 +96,18 @@ interface IPackages {
 	Json getInfo(string _name, string _version, bool minimize = false);
 
 	Json[string] getInfos(string[] packages, bool include_dependencies = false, bool minimize = false);
+
+	@path(":name/update")
+	string postUpdate(string _name, string secret = "");
+
+	@path(":name/update/github")
+	@headerParam("event", "X-GitHub-Event")
+	@queryParam("secret", "secret")
+	string postUpdateGithub(string _name, string secret, string event, Json hook = Json.init);
+
+	@path(":name/update/gitlab")
+	@headerParam("secret", "X-Gitlab-Token")
+	string postUpdateGitlab(string _name, string secret, string object_kind = "");
 }
 
 class LocalDubRegistryAPI : DubRegistryAPI {
@@ -175,6 +187,52 @@ override {
 			.check!(r => r !is null)(HTTPStatus.notFound, "None of the packages were found")
 			.byKeyValue.map!(p => tuple(p.key, p.value.info)).assocArray;
 	}
+
+	@before!updateSecretReader("secret")
+	string postUpdate(string _name, string secret = "")
+	{
+		if (!secret.length)
+			return "No secret sent";
+
+		string expected = m_registry.getPackageSecret(_name);
+		if (!expected.length || secret != expected)
+			return "Secret doesn't match";
+
+		m_registry.triggerPackageUpdate(_name);
+		return "Queued package update";
+	}
+
+	string postUpdateGithub(string _name, string secret, string event, Json hook = Json.init)
+	{
+		if (event == "create") {
+			return postUpdate(_name, secret);
+		} else if (event == "ping") {
+			enforceBadRequest(hook.type == Json.Type.object, "hook is not of type json");
+			auto eventsObj = *enforceBadRequest("events" in hook, "no events object sent in hook object");
+			enforceBadRequest(eventsObj.type == Json.Type.array, "Hook events must be of type array");
+			auto events = eventsObj.get!(Json[]);
+
+			foreach (ev; events)
+				if (ev.type == Json.Type.string && ev.get!string == "create")
+					return "valid";
+
+			string expected = m_registry.getPackageSecret(_name);
+			if (expected.length && secret.length && secret == expected)
+				m_registry.addPackageError(_name,
+					"GitHub hook configuration is invalid. Hook is missing 'create' event. (Tags or branches)");
+
+			return "invalid hook - create event missing";
+		} else
+			return "ignored event " ~ event;
+	}
+
+	string postUpdateGitlab(string _name, string secret, string object_kind)
+	{
+		if (object_kind != "tag_push")
+			return "ignored event " ~ object_kind;
+
+		return postUpdate(_name, secret);
+	}
 }
 
 private:
@@ -186,6 +244,18 @@ private:
 	}
 }
 
+private string updateSecretReader(scope HTTPServerRequest req, scope HTTPServerResponse res)
+{
+	string header = req.query.get("header", "");
+	if (header.length)
+		return req.headers.get(header);
+
+	string ret = req.contentType == "application/json" ? req.json["secret"].get!string : req.form.get("secret", "");
+	if (ret.length)
+		return ret;
+
+	return req.query.get("secret", "");
+}
 
 private	auto ref T check(alias cond, T)(auto ref T t, HTTPStatus status, string msg)
 {

source/dubregistry/dbcontroller.d

@@ -191,6 +191,16 @@ class DbController {
 		m_packages.update(["name": packname], ["$set": ["errors": error]]);
 	}
 
+	void addPackageError(string packname, string error)
+	{
+		m_packages.update(["name": packname], ["$push": ["errors": error]]);
+	}
+
+	void clearPackageErrors(string packname)
+	{
+		m_packages.update(["name": packname], ["$set": ["errors": Bson.emptyArray]]);
+	}
+
 	void setPackageCategories(string packname, string[] categories...)
 	{
 		m_packages.update(["name": packname], ["$set": ["categories": categories]]);
@@ -246,6 +256,21 @@ class DbController {
 		return data.get.data.rawData;
 	}
 
+	string getPackageSecret(string packname)
+	{
+		auto ret = m_packages.findOne(["name": packname]).tryIndex("secret");
+		if (ret.isNull) return null;
+		else return ret.get.get!string;
+	}
+
+	void setPackageSecret(string packname, string secret)
+	{
+		if (secret.length)
+			m_packages.update(["name": packname], ["$set": ["secret": secret]]);
+		else
+			m_packages.update(["name": packname], ["$unset": ["secret": 0]]);
+	}
+
 	void addVersion(string packname, DbPackageVersion ver)
 	{
 		assert(ver.version_.startsWith("~") || ver.version_.isValidVersion());
@@ -482,6 +507,8 @@ struct DbPackage {
 	long updateCounter = 0; // used to implement lockless read-modify-write cycles
 	@optional BsonObjectID logo; // reference to m_files
 	@optional string documentationURL;
+	/// Random secret string used for package management API (webhooks)
+	@optional string secret;
 }
 
 struct DbRepository {

source/dubregistry/registry.d

@@ -239,7 +239,7 @@ class DubRegistry {
 			.map!(pack => getPackageInfo(pack, flags));
 	}
 
-	auto getPackageInfo(DbPackage pack, PackageInfoFlags flags = PackageInfoFlags.none)
+	PackageInfo getPackageInfo(DbPackage pack, PackageInfoFlags flags = PackageInfoFlags.none)
 	{
 		auto rep = getRepository(pack.repository);
 
@@ -261,6 +261,8 @@ class DubRegistry {
 		}
 		if (flags & PackageInfoFlags.includeErrors)
 			nfo["errors"] = serializeToJson(pack.errors);
+		if (flags & PackageInfoFlags.includeAdmin)
+			ret.secret = pack.secret;
 
 		ret.info = nfo;
 
@@ -429,6 +431,38 @@ class DubRegistry {
 		return m_db.getPackageLogo(pack_name, rev);
 	}
 
+	void addPackageError(string pack_name, string error)
+	{
+		m_db.addPackageError(pack_name, error);
+	}
+
+	void clearPackageErrors(string pack_name)
+	{
+		m_db.clearPackageErrors(pack_name);
+	}
+
+	void unsetPackageSecret(string pack_name)
+	{
+		m_db.setPackageSecret(pack_name, null);
+	}
+
+	string getPackageSecret(string pack_name)
+	{
+		return m_db.getPackageSecret(pack_name);
+	}
+
+	void regenPackageSecret(string pack_name)
+	{
+		import std.ascii : lowerHexDigits;
+		import std.random : uniform;
+
+		char[24] token;
+		foreach (ref c; token)
+			c = lowerHexDigits[uniform(0, $)];
+
+		m_db.setPackageSecret(pack_name, token[].idup);
+	}
+
 	void updatePackages()
 	{
 		logDiagnostic("Triggering package update...");
@@ -733,6 +767,7 @@ struct PackageVersionInfo {
 struct PackageInfo {
 	PackageVersionInfo[] versions;
 	BsonObjectID logo;
+	string secret;
 	Json info; /// JSON package information, as reported to the client
 }
 
@@ -743,6 +778,7 @@ enum PackageInfoFlags
 	includeDependencies = 1 << 0, /// include package info of dependencies
 	includeErrors = 1 << 1, /// include package errors
 	minimize = 1 << 2, /// return only minimal information (for dependency resolver)
+	includeAdmin = 1 << 3, /// include package admin information such as secrets
 }
 
 /// Computes a package score from given package stats and global distributions of those stats.

source/dubregistry/web.d

@@ -650,13 +650,13 @@ class DubRegistryFullWebFrontend : DubRegistryWebFrontend {
 	{
 		enforceUserPackage(_user, _packname);
 		auto packageName = _packname;
-		auto nfo = m_registry.getPackageInfo(packageName, PackageInfoFlags.includeErrors);
+		auto nfo = m_registry.getPackageInfo(packageName, PackageInfoFlags.includeErrors | PackageInfoFlags.includeAdmin);
 		if (nfo.info.type == Json.Type.null_) return;
 		auto categories = m_categories;
 		auto registry = m_registry;
 		auto user = _user;
 		auto error = _error;
-		render!("my_packages.package.dt", packageName, categories, user, registry, error);
+		render!("my_packages.package.dt", packageName, categories, user, registry, error, nfo);
 	}
 
 	@auth @path("/my_packages/:packname/update")
@@ -755,6 +755,24 @@ class DubRegistryFullWebFrontend : DubRegistryWebFrontend {
 		redirect("/my_packages/"~_packname);
 	}
 
+	@auth @path("/my_packages/:packname/regen_secret")
+	void postPackageRegenSecret(string _packname, User _user)
+	{
+		enforceUserPackage(_user, _packname);
+		m_registry.regenPackageSecret(_packname);
+
+		redirect("/my_packages/"~_packname~"#repository");
+	}
+
+	@auth @path("/my_packages/:packname/unset_secret")
+	void postPackageUnsetSecret(string _packname, User _user)
+	{
+		enforceUserPackage(_user, _packname);
+		m_registry.unsetPackageSecret(_packname);
+
+		redirect("/my_packages/"~_packname~"#repository");
+	}
+
 	@path("/docs/commandline")
 	void getCommandLineDocs()
 	{

views/my_packages.package.dt

@@ -8,8 +8,7 @@ block body
 	- import dubregistry.registry : PackageInfoFlags;
 	- import dubregistry.web : Category;
 
-	- auto pack = registry.getPackageInfo(packageName, PackageInfoFlags.includeErrors).info;
-
+	- auto pack = nfo.info;
 	h1 Edit package #[a.blind(href="#{req.rootDir}packages/#{packageName}")= packageName]
 		- auto latest = pack["versions"].length ? pack["versions"][pack["versions"].length-1] : Json(null);
 		- if (latest.type == Json.Type.Object)
@@ -82,9 +81,10 @@ block body
 					p Package is scheduled for an automatic update check. Still have to wait for one more package.
 				- else if (update_check_index > 1)
 					p Package is scheduled for an automatic update check. Still have to wait for #{update_check_index} other packages to complete.
-				form.inputForm(method="POST", action="#{req.rootDir}my_packages/#{packageName}/update")
-					p
+				.updates
+					form.inputForm(method="POST", action="#{req.rootDir}my_packages/#{packageName}/update")
 						button(type="submit") Trigger manual update
+					a(href="#repository"): button Setup update hook
 
 	section.pkgconfig#tab-categories
 		a#categories(name="categories")
@@ -116,6 +116,36 @@ block body
 		a#repository(name="repository")
 		h2 Repository
 
+		h3 Update Hooks
+		p You can POST to a special endpoint on the dub registry to trigger a manual package update.
+		p Setup these hooks in GitHub, GitLab or some other service you use to trigger when you push a new tag to make them available on the registry as quickly as possible.
+		- auto secret = nfo.secret;
+		.inputForm
+			- if (secret.length)
+				- auto apiUrl = req.fullURL;
+				- apiUrl.path = typeof(apiUrl.path).init;
+				p
+					label(for="generic") Generic update webhook: (POST)
+					input#generic(type="text", readonly, value="#{apiUrl}/api/packages/#{packageName}/update")
+				p
+					label(for="generic") GitHub webhook:
+					input#generic(type="text", readonly, value="#{apiUrl}/api/packages/#{packageName}/update/github?secret=#{secret}")
+				p
+					label(for="generic") GitLab webhook:
+					input#generic(type="text", readonly, value="#{apiUrl}/api/packages/#{packageName}/update/gitlab")
+				hr
+				p
+					label(for="generic") Secret:
+					input#generic(type="text", readonly, value=secret)
+				hr
+				form(method="POST", action="#{req.rootDir}my_packages/#{packageName}/unset_secret", onsubmit="confirm('Are you sure you want to revoke the secret and disable webhook support? Existing webhooks will not work anymore.')")
+					button(type="submit") Revoke Webhooks
+				form(method="POST", action="#{req.rootDir}my_packages/#{packageName}/regen_secret", onsubmit="confirm('Are you sure you want to regenerate the secret? Existing webhooks will not work anymore.')")
+					button(type="submit") Invalidate Secret
+			- else
+				form(method="POST", action="#{req.rootDir}my_packages/#{packageName}/regen_secret")
+					button(type="submit") Enable Webhooks
+
 		h3 Transfer Repository
 		.inputForm
 			form(method="POST", action="#{req.rootDir}my_packages/#{packageName}/set_repository")

views/my_packages.register.dt

@@ -16,7 +16,7 @@ block body
 		h1 Add new package
 			p.light
 				a.blind(href="#{req.rootDir}publish") Learn more
-				| about publishing.
+				|  about publishing.
 
 		form(method="POST", action="#{req.rootDir}register_package")
 			p