const LABEL = "🤖 Flaky Test Report"; const TITLE_BOT_HEADER = "title: "; // Only check jobs that start with these. // Helps make sure we don't restart something like screenshot tests or linters, which are not known to be flaky. const CONSIDERED_JOBS = [ "Integration Tests", ]; async function getFailedJobsForRun(github, context, workflowRunId, runAttempt) { const { data: { jobs }, } = await github.rest.actions.listJobsForWorkflowRunAttempt({ owner: context.repo.owner, repo: context.repo.repo, run_id: workflowRunId, attempt_number: runAttempt, }); return jobs .filter((job) => job.conclusion === "failure") .filter((job) => CONSIDERED_JOBS.some((title) => job.name.startsWith(title)) ); } export async function rerunFlakyTests({ github, context }) { const failingJobs = await getFailedJobsForRun( github, context, context.payload.workflow_run.id, context.payload.workflow_run.run_attempt ); if (failingJobs.length > 1) { console.log("Multiple jobs failing. PROBABLY not flaky, not rerunning."); return; } if (failingJobs.length === 0) { throw new Error( "rerunFlakyTests should not have run on a run with no failing jobs" ); } github.rest.actions.reRunWorkflowFailedJobs({ owner: context.repo.owner, repo: context.repo.repo, run_id: context.payload.workflow_run.id, }); } // Tries its best to extract a useful error title and message for the given log export function extractDetails(log) { // Strip off timestamp const lines = log.split(/^[0-9.:T\-]*?Z /gm); const failureRegex = /^\t?FAILURE #(?[0-9]+): (?.+)/; const groupRegex = /^##\[group\](?.+)/; const failures = []; let lastGroup = "root"; let loggingFailure; const newFailure = (failureMatch) => { const { headline } = failureMatch.groups; loggingFailure = { headline, group: lastGroup.replace("/datum/unit_test/", ""), details: [], }; }; for (const line of lines) { const groupMatch = line.match(groupRegex); if (groupMatch) { lastGroup = groupMatch.groups.group.trim(); continue; } const failureMatch = line.match(failureRegex); if (loggingFailure === undefined) { if (!failureMatch) { continue; } newFailure(failureMatch); } else if (failureMatch || line.startsWith("##")) { failures.push(loggingFailure); loggingFailure = undefined; if (failureMatch) { newFailure(failureMatch); } } else { loggingFailure.details.push(line.trim()); } } // We had no logged failures, there's not really anything we can do here if (failures.length === 0) { return { title: "Flaky test failure with no obvious source", failures, }; } // We *could* create multiple failures for multiple groups. // This would be important if we had multiple flaky tests at the same time. // I'm choosing not to because it complicates this logic a bit, has the ability to go terribly wrong, // and also because there's something funny to me about that increasing the urgency of fixing // flaky tests. If it becomes a serious issue though, I would not mind this being fixed. const uniqueGroups = new Set(failures.map((failure) => failure.group)); if (uniqueGroups.size > 1) { return { title: `Multiple flaky test failures in ${Array.from(uniqueGroups) .sort() .join(", ")}`, failures, }; } const failGroup = failures[0].group; if (failures.length > 1) { return { title: `Multiple errors in flaky test ${failGroup}`, failures, }; } const failure = failures[0]; // Common patterns where we can always get a detailed title const runtimeMatch = failure.headline.match(/Runtime in .+?: (?.+)/); if (runtimeMatch) { const runtime = runtimeMatch.groups.error.trim(); const invalidTimerMatch = runtime.match(/^Invalid timer:.+object:(?[^[]+).*delegate:(?.+?), source:/); if (invalidTimerMatch) { return { title: `Flaky test ${failGroup}: Invalid timer: ${invalidTimerMatch.groups.proc.trim()} on ${invalidTimerMatch.groups.object.trim()}`, failures, }; } return { title: `Flaky test ${failGroup}: ${runtime}`, failures, }; } const hardDelMatch = failure.headline.match(/^(?\/[\w/]+) hard deleted .* times out of a total del count of/); if (hardDelMatch) { return { title: `Flaky hard delete: ${hardDelMatch.groups.object}`, failures, }; } // Try to normalize the title and remove anything that might be variable const normalizedError = failure.headline.replace(/\s*at .+?:[0-9]+.*/g, ""); // " at code.dm:123" return { title: `Flaky test ${failGroup}: ${normalizedError}`, failures, }; } async function getExistingIssueId(graphql, context, title) { // Hope you never have more than 100 of these open! const { repository: { issues: { nodes: openFlakyTestIssues }, }, } = await graphql( ` query ($owner: String!, $repo: String!, $label: String!) { repository(owner: $owner, name: $repo) { issues( labels: [$label] first: 100 orderBy: { field: CREATED_AT, direction: DESC } states: [OPEN] ) { nodes { number title body } } } } `, { owner: context.repo.owner, repo: context.repo.repo, label: LABEL, } ); const exactTitle = openFlakyTestIssues.find((issue) => issue.title === title); if (exactTitle !== undefined) { return exactTitle.number; } const foundInBody = openFlakyTestIssues.find((issue) => issue.body.includes(``) ); if (foundInBody !== undefined) { return foundInBody.number; } return undefined; } function createBody({ title, failures }, runUrl) { return ` Flaky tests were detected in [this test run](${runUrl}). This means that there was a failure that was cleared when the tests were simply restarted. Failures: \`\`\` ${failures .map( (failure) => `${failure.group}: ${failure.headline}\n\t${failure.details.join("\n")}` ) .join("\n")} \`\`\` `.replace(/^\s*/gm, ""); } export async function reportFlakyTests({ github, context }) { const failedJobsFromLastRun = await getFailedJobsForRun( github, context, context.payload.workflow_run.id, context.payload.workflow_run.run_attempt - 1 ); // This could one day be relaxed if we face serious enough flaky test problems, so we're going to loop anyway if (failedJobsFromLastRun.length !== 1) { console.log( "Multiple jobs failing after retry, assuming maintainer rerun." ); return; } for (const job of failedJobsFromLastRun) { const { data: log } = await github.rest.actions.downloadJobLogsForWorkflowRun({ owner: context.repo.owner, repo: context.repo.repo, job_id: job.id, }); const details = extractDetails(log); const existingIssueId = await getExistingIssueId( github.graphql, context, details.title ); if (existingIssueId !== undefined) { // Maybe in the future, if it's helpful, update the existing issue with new links console.log(`Existing issue found: #${existingIssueId}`); return; } await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title: details.title, labels: [LABEL], body: createBody( details, `https://github.com/${context.repo.owner}/${ context.repo.repo }/actions/runs/${context.payload.workflow_run.id}/attempts/${ context.payload.workflow_run.run_attempt - 1 }` ), }); } }