Skip to content

refine ms_conflict and enable retry failed jobs #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yijingyan2 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

refine ms_conflict and enable retry failed jobs #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 159 additions & 3 deletions azure-pipelines-wrapper/conflict_detect.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const util = require('util');
const { setTimeout } = require('timers/promises');
const eventhub = require('./eventhub');
const akv = require('./keyvault');
const { EmailClient } = require("@azure/communication-email");
const InProgress = 'in_progress'
const MsConflict = 'ms_conflict'
const MsChecker = 'ms_checker'
Expand All @@ -12,6 +13,131 @@ const COMPLETED = 'completed'
const FAILURE = 'failure'
const SUCCESS = 'success'

async function is_msft_user(octokit, username) {
// Check Microsoft GitHub org membership using the bot token
try {
const resp = await octokit.rest.orgs.checkMembershipForUser({
org: 'microsoft',
username,
});
// 204 = member, 302 = not a member (redirect)
if (resp.status === 204) return true;
} catch (e) {
// 404 = not a member, or token lacks org:read scope
}

// Check public GitHub email
try {
const { data } = await octokit.rest.users.getByUsername({ username });
if (data.email && data.email.toLowerCase().endsWith('@microsoft.com')) return true;
} catch (e) {
// Fall through
}

// Check commit emails
try {
const { data: searchResult } = await octokit.rest.search.commits({
q: `author:${username}`,
sort: 'author-date',
order: 'desc',
per_page: 10,
});
for (const item of searchResult.items) {
const email = item.commit && item.commit.author && item.commit.author.email;
if (email && email.toLowerCase().endsWith('@microsoft.com')) {
return true;
}
if (item.commit && item.commit.message) {
const signOffMatch = item.commit.message.match(/Signed-off-by:.*<([^>]+)>/gi);
if (signOffMatch) {
for (const line of signOffMatch) {
const emailMatch = line.match(/<([^>]+)>/);
if (emailMatch && emailMatch[1].toLowerCase().endsWith('@microsoft.com')) {
return true;
}
}
}
}
}
} catch (e) {
// Fall through
}

return false;
}

async function send_conflict_email(app, uuid, url, number, owner, mspr, conflict_ai_result, conflict_ai_description) {
try {
const acs_connection_string = await akv.getSecretFromCache("ACS_EMAIL_CONNECTION_STRING");
const sender_email = await akv.getSecretFromCache("CONFLICT_EMAIL_SENDER");
const notification_email = await akv.getSecretFromCache("CONFLICT_NOTIFICATION_EMAIL");

if (!acs_connection_string || !notification_email || !sender_email) {
app.log.error(`[ CONFLICT DETECT ] [${uuid}] Missing email configuration: ` +
`ACS_EMAIL_CONNECTION_STRING=${!!acs_connection_string}, NOTIFICATION_EMAIL=${!!notification_email}, CONFLICT_EMAIL_SENDER=${!!sender_email}`);
return;
}

const email_client = new EmailClient(acs_connection_string);

const message = {
senderAddress: sender_email,
recipients: {
to: notification_email.split(',').map(e => ({ address: e.trim() })),
},
content: {
subject: `[CODE CONFLICT] Code Conflict Detected - PR #${number}`,
plainText: [
`A code conflict has been detected for an approved community PR.`,
``,
`PR #${number}: ${url}`,
`PR Owner: ${owner}`,
`MS Internal PR: ${mspr}`,
``,
`The community PR is conflict with the MS internal repo.`,
`Please resolve this conflict in the internal branch: sonicbld/precheck/head/${number}`,
``,
`${conflict_ai_result}`,
`${conflict_ai_description}`,
`If the fix commit pushed by Copilot does not correctly resolve the conflict or Copilot failed to push a fix commit, Please follow these steps to resolve:`,
`1. Go to Networking-acs-buildimage repo: https://dev.azure.com/msazure/One/_git/Networking-acs-buildimage and "git fetch origin"`,
`2. Checkout the branch: "git checkout sonicbld/precheck/head/${number}"`,
`3. Resolve the conflict manually and push the fix commit to the same branch`,
`4. Approve the MS internal PR ${mspr}`,
`5. Comment "/azpw ms_conflict" in the original GitHub PR to trigger MS conflict detect again`,
].join('\n'),
html: [
`<h3>Code Conflict Detected - Community PR</h3>`,
`<p>A code conflict has been detected for an approved community PR.</p>`,
`<table border="0" cellpadding="4">`,
`<tr><td><strong>PR</strong></td><td><a href="${url}">#${number}</a></td></tr>`,
`<tr><td><strong>PR Owner</strong></td><td>${owner}</td></tr>`,
`<tr><td><strong>MS Internal PR</strong></td><td><a href="${mspr}">${mspr}</a></td></tr>`,
`</table>`,
`<p>The community PR is conflict with the MS internal repo.<br>`,
`Please resolve this conflict in the internal branch: <code>sonicbld/precheck/head/${number}</code></p>`,
`<p>${conflict_ai_result.replace(/\n/g, '<br>')}</p>`,
`<p>${conflict_ai_description.replace(/\n/g, '<br>')}</p>`,
`<h4>If the fix commit pushed by Copilot does not correctly resolve the conflict or Copilot failed to push a fix commit, Please follow these steps to resolve::</h4>`,
`<ol>`,
`<li>Go to <a href="https://dev.azure.com/msazure/One/_git/Networking-acs-buildimage">Networking-acs-buildimage repo</a> and <code>git fetch origin</code></li>`,
`<li>Checkout the branch: <code>git checkout sonicbld/precheck/head/${number}</code></li>`,
`<li>Resolve the conflict manually and push the fix commit to the same branch</li>`,
`<li>Approve the MS internal PR <a href="${mspr}">${mspr}</a></li>`,
`<li>Comment <code>/azpw ms_conflict</code> in the GitHub PR to trigger MS conflict detect again</li>`,
`</ol>`,
].join('\n'),
},
};

const poller = await email_client.beginSend(message);
const result = await poller.pollUntilDone();
app.log.info(`[ CONFLICT DETECT ] [${uuid}] Conflict email sent to ${notification_email} for PR #${number}, status: ${result.status}`);
} catch (error) {
app.log.error(`[ CONFLICT DETECT ] [${uuid}] Failed to send conflict email: ${error}`);
}
}

async function check_create(app, context, uuid, owner, repo, url, commit, check_name, result, status, output_title, output_summary){
if (! result) {
app.log.error(`[ CONFLICT DETECT ] [${uuid}] check_create: result=BLANK`)
Expand Down Expand Up @@ -159,9 +285,10 @@ function init(app) {
param.push(`PR_OWNER=${pr_owner}`)
param.push(`PR_BASE_BRANCH=${base_branch}`)
param.push(`PR_HEAD_COMMIT=${commit}`)
param.push(`GITHUB_COPILOT_TOKEN=${await akv.getSecretFromCache("GITHUB_COPILOT_TOKEN") || ''}`)

// If it belongs to ms, comment on PR.
var description = '', comment_at = '', mspr = '', tmp = '', ms_conflict_result = '', ms_checker_result = '', output = ''
var description = '', comment_at = '', mspr = '', tmp = '', ms_conflict_result = '', ms_checker_result = '', conflict_ai_result = '', conflict_ai_description = '', output = ''
var run = spawnSync('./bash_action.sh', param, { encoding: 'utf-8' })
for (const line of run.stdout.split(/\r?\n/)){
output = line
Expand All @@ -183,15 +310,44 @@ function init(app) {
if (line.includes("ms_checker.result: ")){
ms_checker_result = line.split(' ').pop()
}
if (line.includes("conflict_ai: ")){
conflict_ai_result += line.substring(line.indexOf("conflict_ai: ") + "conflict_ai: ".length) + '\n'
}
if (line.includes("conflict_ai_description: ")){
conflict_ai_description += line.substring(line.indexOf("conflict_ai_description: ") + "conflict_ai_description: ".length) + '\n'
}
}
app.log.info(`[ CONFLICT DETECT ] [${uuid}] ${mspr}, ${tmp}`)
if ( ['ALL',MsConflict].includes(check_suite) ) {
if (run.status == 254) {
app.log.info([`[ CONFLICT DETECT ] [${uuid}] Conflict detected!`, url].join(" "))
description = `@${comment_at} PR: ${url} is conflict with MS internal repo<br>${mspr}<br>Please push fix commit to sonicbld/precheck/head/${number}<br>Then approve PR and comment "/azpw ${MsConflict}" in github PR.`
const is_msft = await is_msft_user(context.octokit, pr_owner);
if (is_msft) {
description = `@${comment_at} PR: ${url} is conflict with MS internal repo<br><br>MS internal PR: ${mspr}<br>${conflict_ai_result}${conflict_ai_description}<br>If the fix commit pushed by Copilot does not correctly resolve the conflict or Copilot failed to push a fix commit, Please follow the instructions to resolve the conflict:<br>1. Go to Networking-acs-buildimage repo: https://dev.azure.com/msazure/One/_git/Networking-acs-buildimage and \`git fetch origin\`<br>2. Checkout the branch: \`git checkout sonicbld/precheck/head/${number}\`<br>3. Resolve the conflict manually and push the fix commit to the same branch<br>4. Approve the MS internal PR ${mspr}<br>5. Comment \`/azpw ${MsConflict}\` in your GitHub PR to trigger MS conflict detect again.`
} else {
// Only send conflict email if the PR has been approved
let is_approved = false;
try {
const reviews = await context.octokit.rest.pulls.listReviews({
owner,
repo,
pull_number: parseInt(number),
});
is_approved = reviews.data.some(review => review.state === 'APPROVED');
} catch (error) {
app.log.error(`[ CONFLICT DETECT ] [${uuid}] Failed to check PR approval status: ${error}`);
}
if (is_approved) {
description = `@${comment_at} PR: ${url} is conflict with MS internal repo<br><br>The MSFT team will take care of this conflict.<br>A notification email has been sent to sonicelastictest@microsoft.com.`
await send_conflict_email(app, uuid, url, number, pr_owner, mspr, conflict_ai_result, conflict_ai_description);
} else {
description = `@${comment_at} PR: ${url} is conflict with MS internal repo<br><br>The MSFT team will take care of this conflict.<br>Once the PR is approved, please comment \`/azpw ${MsConflict}\` in your GitHub PR to trigger MS conflict detect again and an notification email will be sent to MSFT team.`
app.log.info(`[ CONFLICT DETECT ] [${uuid}] PR not approved yet, skipping email notification`);
}
}
} else if (run.status == 253){
app.log.info([`[ CONFLICT DETECT ] [${uuid}] Conflict already exists!`, url].join(" "))
description = `@${comment_at} Conflict already exists in ${base_branch}<br>Please wait a few hours to run ms_conflict again!<br>'/azpw ${MsConflict}'`
description = `@${comment_at} Conflict already exists in ${base_branch}<br>The MSFT team will take care of this conflict.<br>Please wait a few hours to trigger ms_conflict again by commenting \`/azpw ${MsConflict}\` in your GitHub PR!`
} else if (run.status == 252){
app.log.info([`[ CONFLICT DETECT ] [${uuid}] Github Branch Error!`, url].join(" "))
description = `@${comment_at} Github Branch not ready<br>Please wait a few minutes to run again!<br>'/azpw ${MsConflict}'`
Expand Down
2 changes: 1 addition & 1 deletion azure-pipelines-wrapper/env_init.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ const PRPrefix = 'https://dev.azure.com/msazure/One/_git/Networking-acs-buildima
async function init(app){
output = ''
try{
output = execFileSync('bash', ['-c', 'site/wwwroot/env_init.sh 2>&1 | while IFS= read -r line; do echo [$(date +%FT%TZ)] $line >> env_init.stderr; done;' ], { encoding: 'utf-8' })
output = execFileSync('bash', ['-c', '/home/site/wwwroot/env_init.sh 2>&1 | while IFS= read -r line; do echo [$(date +%FT%TZ)] $line >> env_init.stderr; done;' ], { encoding: 'utf-8' })
app.log.info('[ INIT ] Succeeded!!!')
} catch(e){
app.log.error(`[ INIT ] Failed!!! ${output}`)
Expand Down
3 changes: 2 additions & 1 deletion azure-pipelines-wrapper/env_init.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,5 @@ chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null
apt-get update
apt-get install git jq gh parallel -y
git config --global --add safe.directory '*'
git config --global --add safe.directory '*'
chmod +x /home/site/wwwroot/node_modules/.bin/copilot
Loading