Skip to content

Commit eea7e50

Browse files
wilzbachwilzbach
committed
Add CI PR Recognition: first steps
1 parent 3b46bcf commit eea7e50

File tree

7 files changed

+368
-13
lines changed

7 files changed

+368
-13
lines changed

‎source/dlangbot/app.d‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import dlangbot.bugzilla, dlangbot.github, dlangbot.travis, dlangbot.trello,
44
dlangbot.utils;
55

66
public import dlangbot.bugzilla : bugzillaURL;
7+
public import dlangbot.ci : dTestAPI, circleCiAPI, projectTesterAPI;
78
public import dlangbot.github : githubAPIURL, githubAuth, hookSecret;
89
public import dlangbot.travis : travisAPIURL;
910
public import dlangbot.trello : trelloAPIURL, trelloAuth, trelloSecret;
@@ -20,6 +21,7 @@ import vibe.stream.operations : readAllUTF8;
2021

2122
bool runAsync = true;
2223
bool runTrello = true;
24+
bool runPRReview = true;
2325

2426
Duration timeBetweenFullPRChecks = 5.minutes; // this should never be larger 30 mins on heroku
2527
Throttler!(typeof(&searchForAutoMergePrs)) prThrottler;
@@ -135,7 +137,11 @@ void githubHook(HTTPServerRequest req, HTTPServerResponse res)
135137
string state = json["state"].get!string;
136138
// no need to trigger the checker for failure/pending
137139
if (state == "success")
140+
{
138141
prThrottler(repoSlug);
142+
if (runPRReview)
143+
checkPRForReviewNeed(repoSlug, json);
144+
}
139145

140146
return res.writeBody("handled");
141147
case "pull_request":

‎source/dlangbot/ci.d‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
module dlangbot.ci;
2+
3+
import dlangbot.github;
4+
5+
import vibe.core.log;
6+
import vibe.http.client : HTTPClientRequest, requestHTTP;
7+
8+
import std.conv : to;
9+
import std.format : format;
10+
import std.regex : matchFirst, regex;
11+
import std.exception;
12+
import std.variant : Nullable;
13+
14+
// list of used APIs (overwritten by the test suite)
15+
string dTestAPI = "http://dtest.dlang.io";
16+
string circleCiAPI = "https://circleci.com/api/v1.1";
17+
string projectTesterAPI = "https://ci.dawg.eu";
18+
19+
// only since 2.073
20+
auto nullable(T)(T t) { return Nullable!T(t); }
21+
22+
/**
23+
There's no way to get the PR number from a GitHub status event (or other API
24+
endpoints).
25+
Hence we have to check the sender for this information.
26+
*/
27+
Nullable!uint getPRForStatus(string repoSlug, string url, string context)
28+
{
29+
Nullable!uint prNumber;
30+
31+
try {
32+
logDebug("getPRNumber (repo: %s, ci: %s)", repoSlug, context);
33+
switch (context) {
34+
case "auto-tester":
35+
prNumber = checkAutoTester(url);
36+
break;
37+
case "ci/circleci":
38+
prNumber = checkCircleCi(url);
39+
break;
40+
case "continuous-integration/travis-ci/pr":
41+
prNumber = checkTravisCi(url);
42+
break;
43+
case "CyberShadow/DAutoTest":
44+
prNumber = checkDTest(url);
45+
break;
46+
case "Project Tester":
47+
prNumber = checkProjectTester(url);
48+
break;
49+
// CodeCov provides no way atm
50+
default:
51+
}
52+
} catch (Exception e) {
53+
logDebug("PR number for: %s (by CI: %s) couldn't be detected", repoSlug, context);
54+
logDebug("Exception", e);
55+
}
56+
57+
return prNumber;
58+
}
59+
60+
class PRDetectionException : Exception
61+
{
62+
this()
63+
{
64+
super("Failure to detected PR number");
65+
}
66+
}
67+
68+
Nullable!uint checkCircleCi(string url)
69+
{
70+
import std.algorithm.iteration : splitter;
71+
import std.array : array;
72+
import std.range : back, retro;
73+
74+
// https://circleci.com/gh/dlang/dmd/2827?utm_campaign=v...
75+
static circleCiRe = regex(`circleci.com/gh/(.*)/([0-9]+)`);
76+
Nullable!uint pr;
77+
78+
auto m = url.matchFirst(circleCiRe);
79+
enforce(!m.empty);
80+
81+
string repoSlug = m[1];
82+
ulong buildNumber = m[2].to!ulong;
83+
84+
auto resp = requestHTTP("%s/project/github/%s/%d"
85+
.format(circleCiAPI, repoSlug, buildNumber)).readJson;
86+
if (auto prs = resp["pull_requests"][])
87+
{
88+
pr = prs[0]["url"].get!string
89+
.splitter("/")
90+
.array // TODO: splitter is not bidirectional
91+
// https://issues.dlang.org/show_bug.cgi?id=17047
92+
.back.to!uint;
93+
}
94+
// branch in upstream
95+
return pr;
96+
}
97+
98+
99+
Nullable!uint checkTravisCi(string url)
100+
{
101+
import dlangbot.travis : getPRNumber;
102+
103+
// https://travis-ci.org/dlang/dmd/builds/203056613
104+
static travisCiRe = regex(`travis-ci.org/(.*)/builds/([0-9]+)`);
105+
Nullable!uint pr;
106+
107+
auto m = url.matchFirst(travisCiRe);
108+
enforce(!m.empty);
109+
110+
string repoSlug = m[1];
111+
ulong buildNumber = m[2].to!ulong;
112+
113+
return getPRNumber(repoSlug, buildNumber);
114+
}
115+
116+
// tests PRs only
117+
auto checkAutoTester(string url)
118+
{
119+
// https://auto-tester.puremagic.com/pull-history.ghtml?projectid=1&repoid=1&pullid=6552
120+
static autoTesterRe = regex(`pullid=([0-9]+)`);
121+
122+
auto m = url.matchFirst(autoTesterRe);
123+
enforce(!m.empty);
124+
return m[1].to!uint.nullable;
125+
}
126+
127+
// tests PRs only
128+
auto checkDTest(string url)
129+
{
130+
import vibe.stream.operations : readAllUTF8;
131+
132+
// http://dtest.dlang.io/results/f3f364ddcf96e98d1a6566b04b130c3f8b37a25f/378ec2f7616ec7ca4554c5381b45561473b0c218/
133+
static dTestRe = regex(`results/([0-9a-f]+)/([0-9a-f]+)`);
134+
static dTestReText = regex(`<tr>.*Pull request.*<a href=".*\/pull\/([0-9]+)"`);
135+
136+
// to enable testing: don't use link directly
137+
auto shas = url.matchFirst(dTestRe);
138+
enforce(!shas.empty);
139+
string headSha = shas[1]; // = PR
140+
string baseSha= shas[2]; // e.g upstream/master
141+
142+
auto m = requestHTTP("%s/results/%s/%s/".format(dTestAPI, headSha, baseSha))
143+
.bodyReader
144+
.readAllUTF8
145+
.matchFirst(dTestReText);
146+
147+
enforce(!m.empty);
148+
return m[1].to!uint.nullable;
149+
}
150+
151+
// tests PRs only ?
152+
Nullable!uint checkProjectTester(string url)
153+
{
154+
import vibe.stream.operations : readAllUTF8;
155+
import vibe.inet.url : URL;
156+
157+
// 1: repoSlug, 2: pr
158+
static projectTesterReText = `href="https:\/\/github[.]com\/(.*)\/pull\/([0-9]+)`;
159+
160+
auto uri = URL(url);
161+
162+
auto m = requestHTTP("%s%s".format(projectTesterAPI, uri.path))
163+
.bodyReader
164+
.readAllUTF8
165+
.matchFirst(projectTesterReText);
166+
167+
enforce(!m.empty, "Project tester detection failed");
168+
return m[2].to!uint.nullable;
169+
}

‎source/dlangbot/github.d‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,14 @@ struct PullRequest
143143
string commitsURL() const { return "%s/repos/%s/pulls/%d/commits".format(githubAPIURL, repoSlug, number); }
144144
string eventsURL() const { return "%s/repos/%s/issues/%d/events".format(githubAPIURL, repoSlug, number); }
145145
string htmlURL() const { return "https://github.com/%s/pull/%d".format(repoSlug, number); }
146+
147+
static PullRequest fetch(string repoSlug, uint number)
148+
{
149+
return ghGetRequest("%s/repos/%s/pulls/%d"
150+
.format(githubAPIURL, repoSlug, number))
151+
.readJson
152+
.deserializeJson!PullRequest;
153+
}
146154
}
147155

148156
alias LabelsAndCommits = Tuple!(Json[], "labels", Json[], "commits");
@@ -286,4 +294,66 @@ void searchForAutoMergePrs(string repoSlug)
286294
}
287295
}
288296

297+
auto getPassingCiCount(string repoSlug, string sha)
298+
{
299+
auto json = ghGetRequest("%s/repos/%s/status/%s"
300+
.format(githubAPIURL, repoSlug, sha))
301+
.readJson["statuses"][];
302+
return json.filter!((e){
303+
if (e["state"] == "success")
304+
switch (e["context"].get!string) {
305+
case "auto-tester":
306+
case "CyberShadow/DAutoTest":
307+
case "continuous-integration/travis-ci/pr":
308+
case "ci/circleci":
309+
return true;
310+
default:
311+
return false;
312+
}
313+
return false;
314+
}).walkLength;
315+
}
316+
317+
/**
318+
Marks a PR as reviewable if
319+
- there hasn't been a review yet
320+
- there is at least one successful CI
321+
*/
322+
void checkPRForReviewNeed(string repoSlug, Json statusPayload)
323+
{
324+
import dlangbot.ci : getPRForStatus;
325+
326+
import std.stdio;
327+
auto passingCi = getPassingCiCount(repoSlug, statusPayload["sha"].get!string);
328+
329+
auto prNumber = getPRForStatus(repoSlug,
330+
statusPayload["target_url"].get!string,
331+
statusPayload["context"].get!string);
289332

333+
if (!prNumber.isNull)
334+
{
335+
PullRequest pr = {number: prNumber};
336+
pr.base.repo.fullName = repoSlug;
337+
logInfo("repo(%s): found a valid PR number: %d", repoSlug, prNumber);
338+
auto reviewsURL = "%s/repos/%s/pulls/%d/reviews"
339+
.format(githubAPIURL, repoSlug, prNumber);
340+
auto reviews = requestHTTP(reviewsURL, (scope req) {
341+
// custom media type is required during preview period:
342+
// preview review api: https://developer.github.com/changes/2016-12-14-reviews-api
343+
req.headers["Accept"] = "application/vnd.github.black-cat-preview+json";
344+
req.headers["Authorization"] = githubAuth;
345+
})
346+
.readJson[];
347+
348+
if (reviews.length == 0 && passingCi >= 2)
349+
{
350+
logInfo("repo(%s): No review found", repoSlug);
351+
// do the cool stuff here
352+
pr.addLabels(["needs review"]);
353+
}
354+
else if (reviews.length > 0)
355+
{
356+
pr.removeLabel("needs review");
357+
}
358+
}
359+
}

‎source/dlangbot/travis.d‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,31 @@ string travisAPIURL = "https://api.travis-ci.org";
44
string travisAuth;
55

66
import vibe.core.log;
7+
import vibe.http.client : requestHTTP, HTTPClientRequest;
8+
import vibe.http.common : HTTPMethod;
9+
10+
import std.functional : toDelegate;
11+
import std.format : format;
12+
import std.variant : Nullable;
713

814
//==============================================================================
915
// Dedup Travis-CI builds
1016
//==============================================================================
1117

18+
void travisAuthReq(scope HTTPClientRequest req)
19+
{
20+
req.headers["Accept"] = "application/vnd.travis-ci.2+json";
21+
req.headers["Authorization"] = travisAuth;
22+
}
23+
1224
void cancelBuild(size_t buildId)
1325
{
14-
import std.format : format;
15-
import vibe.http.client : requestHTTP;
16-
import vibe.http.common : HTTPMethod;
1726
import vibe.stream.operations : readAllUTF8;
1827

1928
auto url = "%s/builds/%s/cancel".format(travisAPIURL, buildId);
2029
requestHTTP(url, (scope req) {
21-
req.headers["Authorization"] = travisAuth;
2230
req.method = HTTPMethod.POST;
31+
travisAuthReq(req);
2332
}, (scope res) {
2433
if (res.statusCode / 100 == 2)
2534
logInfo("Canceled Build %s\n", buildId);
@@ -32,9 +41,7 @@ void cancelBuild(size_t buildId)
3241
void dedupTravisBuilds(string action, string repoSlug, uint pullRequestNumber)
3342
{
3443
import std.algorithm.iteration : filter;
35-
import std.format : format;
3644
import std.range : drop;
37-
import vibe.http.client : requestHTTP;
3845

3946
if (action != "synchronize" && action != "merged")
4047
return;
@@ -49,10 +56,7 @@ void dedupTravisBuilds(string action, string repoSlug, uint pullRequestNumber)
4956
}
5057

5158
auto url = "%s/repos/%s/builds?event_type=pull_request".format(travisAPIURL, repoSlug);
52-
auto activeBuildsForPR = requestHTTP(url, (scope req) {
53-
req.headers["Authorization"] = travisAuth;
54-
req.headers["Accept"] = "application/vnd.travis-ci.2+json";
55-
})
59+
auto activeBuildsForPR = requestHTTP(url, (&travisAuthReq).toDelegate)
5660
.readJson["builds"][]
5761
.filter!(b => activeState(b["state"].get!string))
5862
.filter!(b => b["pull_request_number"].get!uint == pullRequestNumber);
@@ -63,4 +67,16 @@ void dedupTravisBuilds(string action, string repoSlug, uint pullRequestNumber)
6367
cancelBuild(b["id"].get!size_t);
6468
}
6569

70+
Nullable!uint getPRNumber(string repoSlug, ulong buildId)
71+
{
72+
Nullable!uint pr;
73+
74+
auto url = "%s/repos/%s/builds/%s".format(travisAPIURL, repoSlug, buildId);
75+
auto res = requestHTTP(url, (&travisAuthReq).toDelegate)
76+
.readJson["build"];
6677

78+
if (res["event_type"] == "pull_request")
79+
pr = res["pull_request_number"].get!uint;
80+
81+
return pr;
82+
}

0 commit comments

Comments
 (0)