Skip to content

Commit 1b92244

Browse files
The-JonseyThe-Jonsey
committed
Add Retries And Backoff To GitLabSCMSource:getMembers
1 parent 6e30226 commit 1b92244

File tree

3 files changed

+163
-19
lines changed

3 files changed

+163
-19
lines changed

‎src/main/java/io/jenkins/plugins/gitlabbranchsource/GitLabSCMSource.java‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import hudson.util.ListBoxModel;
3232
import io.jenkins.plugins.gitlabbranchsource.helpers.GitLabAvatar;
3333
import io.jenkins.plugins.gitlabbranchsource.helpers.GitLabLink;
34+
import io.jenkins.plugins.gitlabbranchsource.helpers.Sleeper;
3435
import io.jenkins.plugins.gitlabserverconfig.credentials.PersonalAccessToken;
3536
import io.jenkins.plugins.gitlabserverconfig.servers.GitLabServer;
3637
import io.jenkins.plugins.gitlabserverconfig.servers.GitLabServers;
@@ -118,6 +119,10 @@ public class GitLabSCMSource extends AbstractGitSCMSource {
118119
private transient Project gitlabProject;
119120
private Long projectId;
120121

122+
private static final Integer MAX_RETRIES = 5;
123+
124+
private static final Integer INITIAL_DELAY_MS = 5000;
125+
121126
/**
122127
* The cache of {@link ObjectMetadataAction} instances for each open MR.
123128
*/
@@ -226,17 +231,41 @@ protected Project getGitlabProject(GitLabApi gitLabApi) {
226231
public HashMap<String, AccessLevel> getMembers() {
227232
HashMap<String, AccessLevel> members = new HashMap<>();
228233
try {
229-
GitLabApi gitLabApi = apiBuilder(this.getOwner(), serverName);
230-
for (Member m : gitLabApi.getProjectApi().getAllMembers(projectPath)) {
234+
for (Member m : getMembersWithRetries()) {
231235
members.put(m.getUsername(), m.getAccessLevel());
232236
}
233237
} catch (GitLabApiException e) {
234238
LOGGER.log(Level.WARNING, "Exception while fetching members" + e, e);
235239
return new HashMap<>();
240+
} catch (InterruptedException e) {
241+
throw new RuntimeException(e);
236242
}
237243
return members;
238244
}
239245

246+
private List<Member> getMembersWithRetries() throws GitLabApiException, InterruptedException {
247+
int delay = INITIAL_DELAY_MS;
248+
int attemptNb = 0;
249+
final Sleeper sleeper = new Sleeper();
250+
GitLabApi gitLabApi = apiBuilder(this.getOwner(), serverName);
251+
while (true) {
252+
try {
253+
return gitLabApi.getProjectApi().getAllMembers(projectPath);
254+
} catch (GitLabApiException e) {
255+
if (e.getHttpStatus() == 429) {
256+
sleeper.sleep(delay);
257+
delay *= 2;
258+
attemptNb++;
259+
if (attemptNb > MAX_RETRIES) {
260+
throw e;
261+
}
262+
} else {
263+
throw e;
264+
}
265+
}
266+
}
267+
}
268+
240269
public Long getProjectId() {
241270
return projectId;
242271
}

‎src/main/java/io/jenkins/plugins/gitlabbranchsource/helpers/Sleeper.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package io.jenkins.plugins.gitlabbranchsource.helpers;
2+
3+
public class Sleeper {
4+
5+
public void sleep(int millis) throws InterruptedException {
6+
Thread.sleep(millis);
7+
}
8+
}

‎src/test/java/io/jenkins/plugins/gitlabbranchsource/GitLabSCMSourceTest.java‎

Lines changed: 124 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,35 @@
88
import hudson.security.AccessControlled;
99
import hudson.util.StreamTaskListener;
1010
import io.jenkins.plugins.gitlabbranchsource.helpers.GitLabHelper;
11+
import io.jenkins.plugins.gitlabbranchsource.helpers.Sleeper;
1112
import io.jenkins.plugins.gitlabserverconfig.servers.GitLabServer;
1213
import io.jenkins.plugins.gitlabserverconfig.servers.GitLabServers;
1314
import java.io.ByteArrayOutputStream;
1415
import java.io.IOException;
1516
import java.nio.charset.StandardCharsets;
1617
import java.util.Arrays;
18+
import java.util.List;
19+
import java.util.Map;
1720
import java.util.Set;
21+
import java.util.concurrent.atomic.AtomicInteger;
1822
import jenkins.branch.BranchSource;
1923
import jenkins.scm.api.SCMHead;
24+
import jenkins.scm.api.SCMSourceOwner;
2025
import org.gitlab4j.api.GitLabApi;
2126
import org.gitlab4j.api.GitLabApiException;
2227
import org.gitlab4j.api.MergeRequestApi;
2328
import org.gitlab4j.api.ProjectApi;
2429
import org.gitlab4j.api.RepositoryApi;
30+
import org.gitlab4j.api.models.AccessLevel;
31+
import org.gitlab4j.api.models.Member;
2532
import org.gitlab4j.api.models.Project;
2633
import org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject;
34+
import org.junit.After;
35+
import org.junit.Before;
2736
import org.junit.ClassRule;
2837
import org.junit.Test;
2938
import org.jvnet.hudson.test.JenkinsRule;
39+
import org.mockito.MockedConstruction;
3040
import org.mockito.MockedStatic;
3141
import org.mockito.Mockito;
3242

@@ -39,6 +49,22 @@ public class GitLabSCMSourceTest {
3949
@ClassRule
4050
public static JenkinsRule j = new JenkinsRule();
4151

52+
private MockedStatic<GitLabHelper> utilities;
53+
54+
private MockedConstruction<Sleeper> sleeperMockedConstruction;
55+
56+
@Before
57+
public void setUp() {
58+
utilities = Mockito.mockStatic(GitLabHelper.class);
59+
sleeperMockedConstruction = Mockito.mockConstruction(Sleeper.class);
60+
}
61+
62+
@After
63+
public void tearDown() {
64+
utilities.close();
65+
sleeperMockedConstruction.close();
66+
}
67+
4268
@Test
4369
public void retrieveMRWithEmptyProjectSettings() throws GitLabApiException, IOException, InterruptedException {
4470
GitLabApi gitLabApi = Mockito.mock(GitLabApi.class);
@@ -49,22 +75,103 @@ public void retrieveMRWithEmptyProjectSettings() throws GitLabApiException, IOEx
4975
Mockito.when(gitLabApi.getMergeRequestApi()).thenReturn(mrApi);
5076
Mockito.when(gitLabApi.getRepositoryApi()).thenReturn(repoApi);
5177
Mockito.when(projectApi.getProject(any())).thenReturn(new Project());
52-
try (MockedStatic<GitLabHelper> utilities = Mockito.mockStatic(GitLabHelper.class)) {
53-
utilities
54-
.when(() -> GitLabHelper.apiBuilder(any(AccessControlled.class), anyString()))
55-
.thenReturn(gitLabApi);
56-
GitLabServers.get().addServer(new GitLabServer("", SERVER, ""));
57-
GitLabSCMSourceBuilder sb =
58-
new GitLabSCMSourceBuilder(SOURCE_ID, SERVER, "creds", "po", "group/project", "project");
59-
WorkflowMultiBranchProject project = j.createProject(WorkflowMultiBranchProject.class, PROJECT_NAME);
60-
BranchSource source = new BranchSource(sb.build());
61-
source.getSource()
62-
.setTraits(Arrays.asList(new BranchDiscoveryTrait(0), new OriginMergeRequestDiscoveryTrait(1)));
63-
project.getSourcesList().add(source);
64-
ByteArrayOutputStream out = new ByteArrayOutputStream();
65-
final TaskListener listener = new StreamTaskListener(out, StandardCharsets.UTF_8);
66-
Set<SCMHead> scmHead = source.getSource().fetch(listener);
67-
assertEquals(0, scmHead.size());
68-
}
78+
utilities
79+
.when(() -> GitLabHelper.apiBuilder(any(AccessControlled.class), anyString()))
80+
.thenReturn(gitLabApi);
81+
GitLabServers.get().addServer(new GitLabServer("", SERVER, ""));
82+
GitLabSCMSourceBuilder sb =
83+
new GitLabSCMSourceBuilder(SOURCE_ID, SERVER, "creds", "po", "group/project", "project");
84+
WorkflowMultiBranchProject project = j.createProject(WorkflowMultiBranchProject.class, PROJECT_NAME);
85+
BranchSource source = new BranchSource(sb.build());
86+
source.getSource()
87+
.setTraits(Arrays.asList(new BranchDiscoveryTrait(0), new OriginMergeRequestDiscoveryTrait(1)));
88+
project.getSourcesList().add(source);
89+
ByteArrayOutputStream out = new ByteArrayOutputStream();
90+
final TaskListener listener = new StreamTaskListener(out, StandardCharsets.UTF_8);
91+
Set<SCMHead> scmHead = source.getSource().fetch(listener);
92+
assertEquals(0, scmHead.size());
93+
}
94+
95+
@Test
96+
public void testGetMembersWithNoRetries() throws GitLabApiException {
97+
GitLabApi gitLabApi = Mockito.mock(GitLabApi.class);
98+
ProjectApi projectApi = Mockito.mock(ProjectApi.class);
99+
Mockito.when(gitLabApi.getProjectApi()).thenReturn(projectApi);
100+
Member mockMember = Mockito.mock(Member.class);
101+
Mockito.when(mockMember.getUsername()).thenReturn("example.user");
102+
Mockito.when(mockMember.getAccessLevel()).thenReturn(AccessLevel.DEVELOPER);
103+
SCMSourceOwner mockOwner = Mockito.mock(SCMSourceOwner.class);
104+
Mockito.when(projectApi.getAllMembers("group/project")).thenReturn(List.of(mockMember));
105+
utilities
106+
.when(() -> GitLabHelper.apiBuilder(any(AccessControlled.class), anyString()))
107+
.thenReturn(gitLabApi);
108+
GitLabServers.get().addServer(new GitLabServer("", SERVER, ""));
109+
GitLabSCMSourceBuilder sb =
110+
new GitLabSCMSourceBuilder(SOURCE_ID, SERVER, "creds", "po", "group/project", "project");
111+
GitLabSCMSource source = sb.build();
112+
source.setOwner(mockOwner);
113+
assertEquals(Map.of("example.user", AccessLevel.DEVELOPER), source.getMembers());
114+
Sleeper sleeper = sleeperMockedConstruction.constructed().get(0);
115+
Mockito.verifyNoInteractions(sleeper);
116+
}
117+
118+
@Test
119+
public void testGetMembersWithAllRetries() throws GitLabApiException, InterruptedException {
120+
GitLabApi gitLabApi = Mockito.mock(GitLabApi.class);
121+
ProjectApi projectApi = Mockito.mock(ProjectApi.class);
122+
Mockito.when(gitLabApi.getProjectApi()).thenReturn(projectApi);
123+
SCMSourceOwner mockOwner = Mockito.mock(SCMSourceOwner.class);
124+
GitLabApiException rateLimitException = new GitLabApiException("Rate limit", 429);
125+
Mockito.when(projectApi.getAllMembers("group/project")).thenThrow(rateLimitException);
126+
utilities
127+
.when(() -> GitLabHelper.apiBuilder(any(AccessControlled.class), anyString()))
128+
.thenReturn(gitLabApi);
129+
GitLabServers.get().addServer(new GitLabServer("", SERVER, ""));
130+
GitLabSCMSourceBuilder sb =
131+
new GitLabSCMSourceBuilder(SOURCE_ID, SERVER, "creds", "po", "group/project", "project");
132+
GitLabSCMSource source = sb.build();
133+
source.setOwner(mockOwner);
134+
assertEquals(Map.of(), source.getMembers());
135+
Sleeper sleeper = sleeperMockedConstruction.constructed().get(0);
136+
Mockito.verify(sleeper, Mockito.times(1)).sleep(5000);
137+
Mockito.verify(sleeper, Mockito.times(1)).sleep(10000);
138+
Mockito.verify(sleeper, Mockito.times(1)).sleep(20000);
139+
Mockito.verify(sleeper, Mockito.times(1)).sleep(40000);
140+
Mockito.verify(sleeper, Mockito.times(1)).sleep(80000);
141+
Mockito.verify(sleeper, Mockito.times(1)).sleep(160000);
142+
Mockito.verifyNoMoreInteractions(sleeper);
143+
}
144+
145+
@Test
146+
public void testGetMembersWithSomeRetries() throws GitLabApiException, InterruptedException {
147+
GitLabApi gitLabApi = Mockito.mock(GitLabApi.class);
148+
ProjectApi projectApi = Mockito.mock(ProjectApi.class);
149+
Mockito.when(gitLabApi.getProjectApi()).thenReturn(projectApi);
150+
GitLabApiException rateLimitException = new GitLabApiException("Rate limit", 429);
151+
Member mockMember = Mockito.mock(Member.class);
152+
Mockito.when(mockMember.getUsername()).thenReturn("example.user");
153+
Mockito.when(mockMember.getAccessLevel()).thenReturn(AccessLevel.DEVELOPER);
154+
SCMSourceOwner mockOwner = Mockito.mock(SCMSourceOwner.class);
155+
AtomicInteger counter = new AtomicInteger();
156+
Mockito.when(projectApi.getAllMembers("group/project")).thenAnswer((input) -> {
157+
if (counter.getAndIncrement() < 3) {
158+
throw rateLimitException;
159+
}
160+
return List.of(mockMember);
161+
});
162+
utilities
163+
.when(() -> GitLabHelper.apiBuilder(any(AccessControlled.class), anyString()))
164+
.thenReturn(gitLabApi);
165+
GitLabServers.get().addServer(new GitLabServer("", SERVER, ""));
166+
GitLabSCMSourceBuilder sb =
167+
new GitLabSCMSourceBuilder(SOURCE_ID, SERVER, "creds", "po", "group/project", "project");
168+
GitLabSCMSource source = sb.build();
169+
source.setOwner(mockOwner);
170+
assertEquals(Map.of("example.user", AccessLevel.DEVELOPER), source.getMembers());
171+
Sleeper sleeper = sleeperMockedConstruction.constructed().get(0);
172+
Mockito.verify(sleeper, Mockito.times(1)).sleep(5000);
173+
Mockito.verify(sleeper, Mockito.times(1)).sleep(10000);
174+
Mockito.verify(sleeper, Mockito.times(1)).sleep(20000);
175+
Mockito.verifyNoMoreInteractions(sleeper);
69176
}
70177
}

0 commit comments

Comments
 (0)