Skip to content

Commit 2d7bf22

Browse files
nfalco79nfalco79
committed
[JENKINS-76184] Enable cache for webhook requests to avoid rate limit for large organisations
Add cache of requests for native webhook implementations
1 parent b6579e0 commit 2d7bf22

File tree

10 files changed

+178
-22
lines changed

10 files changed

+178
-22
lines changed

‎docs/images/screenshot-23.png‎

23.1 KB
Loading

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
<changelist>-SNAPSHOT</changelist>
2929
<gitHubRepo>jenkinsci/bitbucket-branch-source-plugin</gitHubRepo>
3030
<jenkins.baseline>2.504</jenkins.baseline>
31-
<jenkins.version>${jenkins.baseline}.1</jenkins.version>
31+
<jenkins.version>${jenkins.baseline}.3</jenkins.version>
3232
<ban-junit4-imports.skip>false</ban-junit4-imports.skip>
3333
<hpi.compatibleSinceVersion>937.0.0</hpi.compatibleSinceVersion>
3434
<tagNameFormat>@{project.version}</tagNameFormat>

‎src/main/java/com/cloudbees/jenkins/plugins/bitbucket/BitbucketSCMSource.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -752,7 +752,7 @@ protected String getProjectKey() {
752752

753753
private void setPrimaryCloneLinks(List<BitbucketHref> links) {
754754
links.forEach(link -> {
755-
if (Strings.CI.startsWith(link.getName(), "http")) {
755+
if (/*Strings.CI.*/StringUtils.startsWith(link.getName(), "http")) {
756756
// Remove the username from URL because it will be set into the GIT_URL variable
757757
// credentials used to git clone or push/pull operation could be different than this (for example SSH)
758758
// and will run into a failure

‎src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ public class BitbucketCloudApiClient extends AbstractBitbucketApi implements Bit
113113
private static final Cache<String, BitbucketTeam> cachedTeam = new Cache<>(6, HOURS);
114114
private static final Cache<String, List<BitbucketCloudRepository>> cachedRepositories = new Cache<>(3, HOURS);
115115
private static final Cache<String, BitbucketCloudCommit> cachedCommits = new Cache<>(24, HOURS);
116-
private transient BitbucketRepository cachedRepository;
116+
private transient BitbucketRepository localCachedRepository;
117117
private transient String cachedDefaultBranch;
118118

119119
public static List<String> stats() {
@@ -265,14 +265,14 @@ public BitbucketRepository getRepository() throws IOException {
265265
if (repositoryName == null) {
266266
throw new UnsupportedOperationException("Cannot get a repository from an API instance that is not associated with a repository");
267267
}
268-
if (!enableCache || cachedRepository == null) {
268+
if (!enableCache || localCachedRepository == null) {
269269
String url = UriTemplate.fromTemplate(REPO_URL_TEMPLATE)
270270
.set("owner", owner)
271271
.set("repo", repositoryName)
272272
.expand();
273-
cachedRepository = getRequestAs(url, BitbucketCloudRepository.class);
273+
localCachedRepository = getRequestAs(url, BitbucketCloudRepository.class);
274274
}
275-
return cachedRepository;
275+
return localCachedRepository;
276276
}
277277

278278
/**

‎src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import hudson.util.ListBoxModel;
4040
import java.net.MalformedURLException;
4141
import java.net.URL;
42+
import java.util.List;
4243
import jenkins.model.Jenkins;
4344
import org.apache.commons.lang3.StringUtils;
4445
import org.jenkinsci.plugins.plaincredentials.StringCredentials;
@@ -84,6 +85,13 @@ public abstract class AbstractBitbucketWebhookConfiguration implements Bitbucket
8485
*/
8586
private String endpointJenkinsRootURL;
8687

88+
private boolean enableCache = true;
89+
90+
/**
91+
* How long, in minutes, to cache the webhook response.
92+
*/
93+
private Integer webhooksCacheDuration;
94+
8795
protected AbstractBitbucketWebhookConfiguration(boolean manageHooks, @CheckForNull String credentialsId,
8896
boolean enableHookSignature, @CheckForNull String hookSignatureCredentialsId) {
8997
this.manageHooks = manageHooks && StringUtils.isNotBlank(credentialsId);
@@ -144,7 +152,47 @@ public String getDisplayName() {
144152
return Messages.ServerWebhookImplementation_displayName();
145153
}
146154

155+
public boolean isEnableCache() {
156+
return enableCache;
157+
}
158+
159+
@DataBoundSetter
160+
public void setEnableCache(boolean enableCache) {
161+
this.enableCache = enableCache;
162+
}
163+
164+
public Integer getWebhooksCacheDuration() {
165+
return webhooksCacheDuration;
166+
}
167+
168+
@DataBoundSetter
169+
public void setWebhooksCacheDuration(Integer webhooksCacheDuration) {
170+
this.webhooksCacheDuration = webhooksCacheDuration == null ? Integer.valueOf(180) : webhooksCacheDuration;
171+
}
172+
147173
public abstract static class AbstractBitbucketWebhookDescriptorImpl extends BitbucketWebhookDescriptor {
174+
protected abstract void clearCaches();
175+
protected abstract List<String> getStats();
176+
177+
@RequirePOST
178+
public FormValidation doShowStats() {
179+
Jenkins.get().checkPermission(Jenkins.MANAGE);
180+
181+
List<String> stats = getStats();
182+
StringBuilder builder = new StringBuilder();
183+
for (String stat : stats) {
184+
builder.append(stat).append("<br>");
185+
}
186+
return FormValidation.okWithMarkup(builder.toString());
187+
}
188+
189+
@RequirePOST
190+
public FormValidation doClear() {
191+
Jenkins.get().checkPermission(Jenkins.MANAGE);
192+
193+
clearCaches();
194+
return FormValidation.ok("Caches cleared");
195+
}
148196

149197
@RequirePOST
150198
public static FormValidation doCheckEndpointJenkinsRootURL(@QueryParameter String value) {

‎src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookConfiguration.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import edu.umd.cs.findbugs.annotations.NonNull;
3333
import hudson.Extension;
3434
import hudson.util.ListBoxModel;
35+
import java.util.List;
3536
import jenkins.model.Jenkins;
3637
import org.jenkinsci.Symbol;
3738
import org.kohsuke.stapler.DataBoundConstructor;
@@ -69,6 +70,16 @@ public Class<? extends BitbucketWebhookManager> getManager() {
6970
@Extension
7071
public static class DescriptorImpl extends AbstractBitbucketWebhookDescriptorImpl {
7172

73+
@Override
74+
protected void clearCaches() {
75+
CloudWebhookManager.clearCaches();
76+
}
77+
78+
@Override
79+
protected List<String> getStats() {
80+
return CloudWebhookManager.stats();
81+
}
82+
7283
@Override
7384
public String getDisplayName() {
7485
return "Native Cloud";

‎src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookManager.java‎

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,18 @@
2424
package com.cloudbees.jenkins.plugins.bitbucket.impl.webhook.cloud;
2525

2626
import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketAuthenticatedClient;
27+
import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException;
2728
import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketWebHook;
2829
import com.cloudbees.jenkins.plugins.bitbucket.api.endpoint.BitbucketEndpoint;
2930
import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookConfiguration;
3031
import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookManager;
3132
import com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketCloudPage;
33+
import com.cloudbees.jenkins.plugins.bitbucket.client.Cache;
3234
import com.cloudbees.jenkins.plugins.bitbucket.client.repository.BitbucketCloudWebhook;
3335
import com.cloudbees.jenkins.plugins.bitbucket.hooks.HookEventType;
36+
import com.cloudbees.jenkins.plugins.bitbucket.impl.client.ICheckedCallable;
3437
import com.cloudbees.jenkins.plugins.bitbucket.impl.endpoint.BitbucketCloudEndpoint;
38+
import com.cloudbees.jenkins.plugins.bitbucket.impl.util.BitbucketApiUtils;
3539
import com.cloudbees.jenkins.plugins.bitbucket.impl.util.BitbucketCredentialsUtils;
3640
import com.cloudbees.jenkins.plugins.bitbucket.impl.util.JsonParser;
3741
import com.damnhandy.uri.template.UriTemplate;
@@ -48,6 +52,7 @@
4852
import java.util.List;
4953
import java.util.Set;
5054
import java.util.TreeSet;
55+
import java.util.concurrent.ExecutionException;
5156
import java.util.logging.Level;
5257
import java.util.logging.Logger;
5358
import jenkins.model.Jenkins;
@@ -57,10 +62,24 @@
5762
import org.apache.commons.lang3.Strings;
5863
import org.jenkinsci.plugins.plaincredentials.StringCredentials;
5964

65+
import static java.util.concurrent.TimeUnit.HOURS;
66+
import static java.util.concurrent.TimeUnit.MINUTES;
67+
6068
@Extension
6169
public class CloudWebhookManager implements BitbucketWebhookManager {
6270
private static final String WEBHOOK_URL = "/2.0/repositories{/owner,repo}/hooks{/hook}{?page,pagelen}";
6371
private static final Logger logger = Logger.getLogger(CloudWebhookManager.class.getName());
72+
private static final Cache<String, List<BitbucketWebHook>> cachedRepositoryWebhooks = new Cache<>(3, HOURS);
73+
74+
public static void clearCaches() {
75+
cachedRepositoryWebhooks.evictAll();
76+
}
77+
78+
public static List<String> stats() {
79+
List<String> stats = new ArrayList<>();
80+
stats.add("Repositories webhooks: " + cachedRepositoryWebhooks.stats().toString());
81+
return stats;
82+
}
6483

6584
// The list of events available in Bitbucket Cloud.
6685
private static final List<String> CLOUD_EVENTS = Collections.unmodifiableList(Arrays.asList(
@@ -87,6 +106,9 @@ public void apply(SCMSourceTrait configurationTrait) {
87106
@Override
88107
public void apply(BitbucketWebhookConfiguration configuration) {
89108
this.configuration = (CloudWebhookConfiguration) configuration;
109+
if (this.configuration.isEnableCache()) {
110+
cachedRepositoryWebhooks.setExpireDuration(this.configuration.getWebhooksCacheDuration(), MINUTES);
111+
}
90112
}
91113

92114
@Override
@@ -105,21 +127,38 @@ public Collection<BitbucketWebHook> read(@NonNull BitbucketAuthenticatedClient c
105127
.set("pagelen", 100)
106128
.expand();
107129

108-
List<BitbucketWebHook> resources = new ArrayList<>();
130+
ICheckedCallable<List<BitbucketWebHook>, IOException> request = () -> {
131+
List<BitbucketWebHook> resources = new ArrayList<>();
109132

110-
TypeReference<BitbucketCloudPage<BitbucketCloudWebhook>> type = new TypeReference<BitbucketCloudPage<BitbucketCloudWebhook>>(){};
111-
BitbucketCloudPage<BitbucketCloudWebhook> page = JsonParser.toJava(client.get(url), type);
112-
resources.addAll(page.getValues().stream()
113-
.filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL))
114-
.toList());
115-
while (!page.isLastPage()){
116-
String response = client.get(page.getNext());
117-
page = JsonParser.toJava(response, type);
133+
TypeReference<BitbucketCloudPage<BitbucketCloudWebhook>> type = new TypeReference<BitbucketCloudPage<BitbucketCloudWebhook>>(){};
134+
BitbucketCloudPage<BitbucketCloudWebhook> page = JsonParser.toJava(client.get(url), type);
118135
resources.addAll(page.getValues().stream()
119136
.filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL))
120137
.toList());
138+
while (!page.isLastPage()){
139+
String response = client.get(page.getNext());
140+
page = JsonParser.toJava(response, type);
141+
resources.addAll(page.getValues().stream()
142+
.filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL))
143+
.toList());
144+
}
145+
return resources;
146+
};
147+
if (configuration.isEnableCache()) {
148+
try {
149+
// TODO choose an opportune cacheKey maybe owner::repository::secretId ??
150+
return cachedRepositoryWebhooks.get(url, request);
151+
} catch (ExecutionException e) {
152+
BitbucketRequestException bre = BitbucketApiUtils.unwrap(e);
153+
if (bre != null) {
154+
throw bre;
155+
} else {
156+
throw new IOException(e);
157+
}
158+
}
159+
} else {
160+
return request.call();
121161
}
122-
return resources;
123162
}
124163

125164
@NonNull

‎src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookConfiguration.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import edu.umd.cs.findbugs.annotations.NonNull;
3333
import hudson.Extension;
3434
import hudson.util.ListBoxModel;
35+
import java.util.List;
3536
import jenkins.model.Jenkins;
3637
import org.jenkinsci.Symbol;
3738
import org.kohsuke.stapler.DataBoundConstructor;
@@ -72,6 +73,16 @@ public Class<? extends BitbucketWebhookManager> getManager() {
7273
@Extension
7374
public static class DescriptorImpl extends AbstractBitbucketWebhookDescriptorImpl {
7475

76+
@Override
77+
protected void clearCaches() {
78+
ServerWebhookManager.clearCaches();
79+
}
80+
81+
@Override
82+
protected List<String> getStats() {
83+
return ServerWebhookManager.stats();
84+
}
85+
7586
@Override
7687
public String getDisplayName() {
7788
return "Native Data Center";

‎src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/server/ServerWebhookManager.java‎

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,15 @@
2424
package com.cloudbees.jenkins.plugins.bitbucket.impl.webhook.server;
2525

2626
import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketAuthenticatedClient;
27+
import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException;
2728
import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketWebHook;
2829
import com.cloudbees.jenkins.plugins.bitbucket.api.endpoint.BitbucketEndpoint;
2930
import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookConfiguration;
3031
import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookManager;
32+
import com.cloudbees.jenkins.plugins.bitbucket.client.Cache;
3133
import com.cloudbees.jenkins.plugins.bitbucket.hooks.HookEventType;
34+
import com.cloudbees.jenkins.plugins.bitbucket.impl.client.ICheckedCallable;
35+
import com.cloudbees.jenkins.plugins.bitbucket.impl.util.BitbucketApiUtils;
3236
import com.cloudbees.jenkins.plugins.bitbucket.impl.util.BitbucketCredentialsUtils;
3337
import com.cloudbees.jenkins.plugins.bitbucket.impl.util.JsonParser;
3438
import com.cloudbees.jenkins.plugins.bitbucket.server.client.BitbucketServerPage;
@@ -47,6 +51,7 @@
4751
import java.util.List;
4852
import java.util.Set;
4953
import java.util.TreeSet;
54+
import java.util.concurrent.ExecutionException;
5055
import java.util.logging.Level;
5156
import java.util.logging.Logger;
5257
import jenkins.model.Jenkins;
@@ -55,10 +60,24 @@
5560
import org.apache.commons.lang3.ObjectUtils;
5661
import org.jenkinsci.plugins.plaincredentials.StringCredentials;
5762

63+
import static java.util.concurrent.TimeUnit.HOURS;
64+
import static java.util.concurrent.TimeUnit.MINUTES;
65+
5866
@Extension
5967
public class ServerWebhookManager implements BitbucketWebhookManager {
6068
private static final String WEBHOOK_API = "/rest/api/1.0/projects/{owner}/repos/{repo}/webhooks{/id}{?start,limit}";
6169
private static final Logger logger = Logger.getLogger(ServerWebhookManager.class.getName());
70+
private static final Cache<String, List<BitbucketWebHook>> cachedRepositoryWebhooks = new Cache<>(3, HOURS);
71+
72+
public static void clearCaches() {
73+
cachedRepositoryWebhooks.evictAll();
74+
}
75+
76+
public static List<String> stats() {
77+
List<String> stats = new ArrayList<>();
78+
stats.add("Repositories webhooks: " + cachedRepositoryWebhooks.stats().toString());
79+
return stats;
80+
}
6281

6382
// The list of events available in Bitbucket Data Center for the minimum supported version.
6483
private static final List<String> NATIVE_SERVER_EVENTS = Collections.unmodifiableList(Arrays.asList(
@@ -99,6 +118,9 @@ public void setCallbackURL(@NonNull String callbackURL, @NonNull BitbucketEndpoi
99118
@Override
100119
public void apply(BitbucketWebhookConfiguration configuration) {
101120
this.configuration = (ServerWebhookConfiguration) configuration;
121+
if (this.configuration.isEnableCache()) {
122+
cachedRepositoryWebhooks.setExpireDuration(this.configuration.getWebhooksCacheDuration(), MINUTES);
123+
}
102124
}
103125

104126
@Override
@@ -113,12 +135,29 @@ public Collection<BitbucketWebHook> read(@NonNull BitbucketAuthenticatedClient c
113135
.set("limit", 200)
114136
.expand();
115137

116-
TypeReference<BitbucketServerPage<BitbucketServerWebhook>> type = new TypeReference<BitbucketServerPage<BitbucketServerWebhook>>(){};
117-
return JsonParser.toJava(client.get(url), type)
118-
.getValues().stream()
119-
.map(BitbucketWebHook.class::cast)
120-
.filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL))
121-
.toList();
138+
ICheckedCallable<List<BitbucketWebHook>, IOException> request = () -> {
139+
TypeReference<BitbucketServerPage<BitbucketServerWebhook>> type = new TypeReference<BitbucketServerPage<BitbucketServerWebhook>>(){};
140+
return JsonParser.toJava(client.get(url), type)
141+
.getValues().stream()
142+
.map(BitbucketWebHook.class::cast)
143+
.filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL))
144+
.toList();
145+
};
146+
if (configuration.isEnableCache()) {
147+
try {
148+
// TODO choose an opportune cacheKey maybe owner::repository::secretId ??
149+
return cachedRepositoryWebhooks.get(url, request);
150+
} catch (ExecutionException e) {
151+
BitbucketRequestException bre = BitbucketApiUtils.unwrap(e);
152+
if (bre != null) {
153+
throw bre;
154+
} else {
155+
throw new IOException(e);
156+
}
157+
}
158+
} else {
159+
return request.call();
160+
}
122161
}
123162

124163
@NonNull

‎src/main/resources/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration/config.jelly‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,4 +38,12 @@ THE SOFTWARE.
3838
<c:select />
3939
</f:entry>
4040
</f:optionalBlock>
41+
42+
<f:optionalBlock title="${%Enable cache}" field="enableCache" inline="true">
43+
<f:entry title="${%How long to cache webhook requests, in minutes}" field="webhooksCacheDuration">
44+
<f:number default="180" />
45+
</f:entry>
46+
<f:validateButton title="${%Clear caches}" method="clear" />
47+
<f:validateButton title="${%Show statistics}" method="showStats" />
48+
</f:optionalBlock>
4149
</j:jelly>

0 commit comments

Comments
 (0)