chore: add issue analyzer (#9480)

See internal note.
This commit is contained in:
jrandolf 2023-01-11 12:33:53 +01:00 committed by GitHub
parent 7620cb30c3
commit ef35ee7296
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 644 additions and 7 deletions

View File

@ -27,13 +27,11 @@ body:
attributes:
label: Bug behavior
description: >
How does the bug behave? Does it happen very rarely (flaky)? Is there a
lack of error (no error)? If there is a PDF problem, make sure the
script writes the PDF somewhere in the current working directory. *Note:
PDF implies no error.*
How does the bug behave? Does it happen very rarely (flaky)? If there is
a PDF problem, make sure the script writes the PDF somewhere in the
current working directory. *Note: PDF implies no error.*
options:
- label: Flaky
- label: No error
- label: PDF
- id: mvce
type: textarea
@ -44,6 +42,16 @@ body:
example](https://stackoverflow.com/help/minimal-reproducible-example).
*No need for backticks — this automatically gets formatted into code.*
render: TypeScript
placeholder: |
import puppeteer from 'puppeteer'; // TS/ESM are all supported.
(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://news.google.com/news/');
await page.screenshot({path: 'news.png', fullPage: true});
await browser.close();
})();
validations:
required: true
- id: error
@ -51,8 +59,10 @@ body:
attributes:
label: Error string
description: >
Provide the bug's error. For example, `throw new Error('test')` would
have the error `test`. *Do not include the entire error log.*
Provide the bug's error. For example, `throw new Error('Something went
wrong')` would have the error `Something went wrong`. **If the script
does not throw**, write `no error` (case insensitive).
placeholder: Something went wrong
validations:
required: true
- id: puppeteer-configuration

203
.github/workflows/issue-analyzer.yml vendored Normal file
View File

@ -0,0 +1,203 @@
name: Issue Analyzer
permissions: read-all
on:
issues:
types: [opened, reopened, edited]
concurrency:
group: ${{ format('issue-{0}', github.event.issue.number) }}
cancel-in-progress: true
jobs:
analyze-issue:
name: Analyze Issues
runs-on: ubuntu-latest
if: ${{ contains(github.event.issue.labels.*.name, 'bug') }}
env:
ISSUE_BODY: ${{ toJson(github.event.issue.body) }}
outputs:
runsOn: ${{ steps.issue-analysis.outputs.runsOn }}
nodeVersion: ${{ steps.issue-analysis.outputs.nodeVersion }}
packageManager: ${{ steps.issue-analysis.outputs.packageManager }}
errorMessage: ${{ steps.issue-analysis.outputs.errorMessage }}
permissions:
issues: write
steps:
- name: Remove labels
uses: actions/github-script@v6
with:
script: |
if (${{ contains(github.event.issue.labels.*.name, 'confirmed') }}) {
github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: ["confirmed"]
})
}
if (${{ contains(github.event.issue.labels.*.name, 'not-reproducible') }}) {
github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: ["not-reproducible"]
})
}
if (${{ contains(github.event.issue.labels.*.name, 'needs-feedback') }}) {
github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: ["needs-feedback"]
})
}
if (${{ contains(github.event.issue.labels.*.name, 'invalid') }}) {
github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: ["invalid"]
})
}
- name: Check out repository
uses: actions/checkout@v3.0.2
- name: Set up Node.js
uses: actions/setup-node@v3.5.1
with:
node-version: latest
- name: Install dependencies
run: npm install
- name: Analyze issue
id: issue-analysis
run: echo $ISSUE_BODY | ./analyze_issue.mjs >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@v3
with:
name: issue-files
path: out/
verify-issue:
name: Verify Issue
needs: analyze-issue
runs-on: ${{ needs.analyze-issue.outputs.runsOn }}
permissions:
issues: write
env:
PACKAGE_MANAGER: ${{ needs.analyze-issue.outputs.packageManager }}
NODE_VERSION: ${{ needs.analyze-issue.outputs.nodeVersion }}
steps:
- uses: actions/download-artifact@v3
with:
name: issue-files
path: '.'
- name: Set up Node.js
uses: actions/setup-node@v3.5.1
with:
node-version: ${{ env.NODE_VERSION }}
- name: Enable corepack
run: corepack enable
- name: Install dependencies
run: ${{ env.PACKAGE_MANAGER }} install
- name: Verify issue
timeout-minutes: 10
run: ${{ env.PACKAGE_MANAGER }} run verify
- uses: actions/upload-artifact@v3
if: success() || failure()
with:
name: runtime-output
path: |
*.log
**/*.pdf
if-no-files-found: error
label-verified-issue:
needs: [verify-issue]
if: success()
name: Label verified issue
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Add labels
uses: actions/github-script@v6
with:
script: |
github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: ["confirmed"]
})
label-invalid-issue:
needs: [analyze-issue]
if: failure() && needs.analyze-issue.outputs.errorMessage != ''
name: Label invalid issue
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Add labels
uses: actions/github-script@v6
with:
script: |
github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: ["invalid"]
})
- name: Add comment
uses: peter-evans/create-or-update-comment@v2
with:
issue-number: ${{ github.event.issue.number }}
body: ${{ needs.analyze-issue.outputs.errorMessage }}
label-unverifiable-issue:
needs: [analyze-issue, verify-issue]
if: failure() && needs.analyze-issue.outputs.errorMessage == ''
name: Label unverifiable issue
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Add labels
uses: actions/github-script@v6
with:
script: |
github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: ["not-reproducible", "needs-feedback"]
})
- name: Add comment
uses: peter-evans/create-or-update-comment@v2
with:
issue-number: ${{ github.event.issue.number }}
body: |
This issue was not reproducible. Please check that your example runs locally and the following:
- Ensure the script does not rely on dependencies outside of `puppeteer` and `puppeteer-core`.
- Ensure the error string is just the error message.
- Bad:
```ts
Error: something went wrong
at Object.<anonymous> (/Users/username/repository/script.js:2:1)
at Module._compile (node:internal/modules/cjs/loader:1159:14)
at Module._extensions..js (node:internal/modules/cjs/loader:1213:10)
at Module.load (node:internal/modules/cjs/loader:1037:32)
at Module._load (node:internal/modules/cjs/loader:878:12)
at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:81:12)
at node:internal/main/run_main_module:23:47
```
- Good: `Error: something went wrong`.
- Ensure your configuration file (if applicable) is valid.
- If the issue is flaky (does not reproduce all the time), make sure 'Flaky' is checked.
- If the issue is not expected to error, make sure to write 'no error'.
Once the above checks are satisfied, please edit your issue with the changes and we will
try to reproduce the bug again.

69
package-lock.json generated
View File

@ -12,6 +12,7 @@
"test/installation"
],
"devDependencies": {
"@actions/core": "1.10.0",
"@commitlint/cli": "17.3.0",
"@commitlint/config-conventional": "17.3.0",
"@microsoft/api-documenter": "7.19.26",
@ -82,6 +83,25 @@
"zod": "3.20.2"
}
},
"node_modules/@actions/core": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz",
"integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==",
"dev": true,
"dependencies": {
"@actions/http-client": "^2.0.1",
"uuid": "^8.3.2"
}
},
"node_modules/@actions/http-client": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz",
"integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==",
"dev": true,
"dependencies": {
"tunnel": "^0.0.6"
}
},
"node_modules/@angular-devkit/architect": {
"version": "0.1500.4",
"resolved": "https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.1500.4.tgz",
@ -8110,6 +8130,15 @@
"fsevents": "~2.3.2"
}
},
"node_modules/tunnel": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
"dev": true,
"engines": {
"node": ">=0.6.11 <=0.7.0 || >=0.7.3"
}
},
"node_modules/type-check": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
@ -8202,6 +8231,15 @@
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
},
"node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
"dev": true,
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/v8-compile-cache-lib": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
@ -8882,6 +8920,25 @@
}
},
"dependencies": {
"@actions/core": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz",
"integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==",
"dev": true,
"requires": {
"@actions/http-client": "^2.0.1",
"uuid": "^8.3.2"
}
},
"@actions/http-client": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz",
"integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==",
"dev": true,
"requires": {
"tunnel": "^0.0.6"
}
},
"@angular-devkit/architect": {
"version": "0.1500.4",
"resolved": "https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.1500.4.tgz",
@ -14929,6 +14986,12 @@
"fsevents": "~2.3.2"
}
},
"tunnel": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
"dev": true
},
"type-check": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
@ -14996,6 +15059,12 @@
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
},
"uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
"dev": true
},
"v8-compile-cache-lib": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",

View File

@ -35,6 +35,7 @@
"test": "cross-env PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 node tools/mochaRunner/lib/main.js"
},
"devDependencies": {
"@actions/core": "1.10.0",
"@commitlint/cli": "17.3.0",
"@commitlint/config-conventional": "17.3.0",
"@microsoft/api-documenter": "7.19.26",

286
tools/analyze_issue.mjs Executable file
View File

@ -0,0 +1,286 @@
#!/usr/bin/env node
// @ts-check
'use strict';
import {writeFile, mkdir, copyFile} from 'fs/promises';
import {dirname, join} from 'path';
import semver from 'semver';
import {fileURLToPath} from 'url';
import {
versionsPerRelease,
lastMaintainedChromiumVersion,
} from '../versions.js';
import core from '@actions/core';
const LAST_SUPPORTED_PUPPETEER_VERSION = versionsPerRelease.get(
lastMaintainedChromiumVersion
);
if (!LAST_SUPPORTED_PUPPETEER_VERSION) {
core.setFailed('No maintained version found.');
}
const LAST_SUPPORTED_NODE_VERSION = '14.19.0';
const SUPPORTED_OSES = ['windows', 'macos', 'linux'];
const SUPPORTED_PACKAGE_MANAGERS = ['yarn', 'npm', 'pnpm'];
const codifyAndJoinValues = values => {
return values
.map(value => {
return `\`${value}\``;
})
.join(' ,');
};
const formatMessage = value => {
return value.trim();
};
const removeVersionPrefix = value => {
return value.startsWith('v') ? value.slice(1) : value;
};
const ERROR_MESSAGES = {
unsupportedOs(value) {
return formatMessage(`
This issue has an unsupported OS: \`${value}\`. Only the following operating systems are supported: ${codifyAndJoinValues(
SUPPORTED_OSES
)}. Please verify the issue on a supported OS and update the form.
`);
},
unsupportedPackageManager(value) {
return formatMessage(`
This issue has an unsupported package manager: \`${value}\`. Only the following package managers are supported: ${codifyAndJoinValues(
SUPPORTED_PACKAGE_MANAGERS
)}. Please verify the issue using a supported package manager and update the form.
`);
},
invalidPackageManagerVersion(value) {
return formatMessage(`
This issue has an invalid package manager version: \`${value}\`. Versions must follow [SemVer](https://semver.org/) formatting. Please update the form with a valid version.
`);
},
unsupportedNodeVersion(value) {
return formatMessage(`
This issue has an unsupported Node.js version: \`${value}\`. Only versions above \`v${LAST_SUPPORTED_NODE_VERSION}\` are supported. Please verify the issue on a supported version of Node.js and update the form.
`);
},
invalidNodeVersion(value) {
return formatMessage(`
This issue has an invalid Node.js version: \`${value}\`. Versions must follow [SemVer](https://semver.org/) formatting. Please update the form with a valid version.
`);
},
unsupportedPuppeteerVersion(value) {
return formatMessage(`
This issue has an unsupported Puppeteer version: \`${value}\`. Only versions above \`v${LAST_SUPPORTED_PUPPETEER_VERSION}\` are supported. Please verify the issue on a supported version of Puppeteer and update the form.
`);
},
invalidPuppeteerVersion(value) {
return formatMessage(`
This issue has an invalid Puppeteer version: \`${value}\`. Versions must follow [SemVer](https://semver.org/) formatting. Please update the form with a valid version.
`);
},
};
(async () => {
let input = '';
// @ts-expect-error: `iterator` is new and experimental.
for await (const chunk of process.stdin.iterator({
destroyOnReturn: false,
})) {
input += chunk;
}
input = JSON.parse(input).trim();
let mvce = '';
let error = '';
let configuration = '';
let puppeteerVersion = '';
let nodeVersion = '';
let packageManagerVersion = '';
let packageManager = '';
let os = '';
const behavior = {};
const lines = input.split('\n');
{
/** @type {(value: string) => void} */
let set = () => {
return void 0;
};
let j = 1;
let i = 1;
for (; i < lines.length; ++i) {
if (lines[i].startsWith('### Bug behavior')) {
set(lines.slice(j, i).join('\n').trim());
j = i + 1;
set = value => {
if (value.match(/\[x\] Flaky/i)) {
behavior.flaky = true;
}
if (value.match(/\[x\] pdf/i)) {
behavior.noError = true;
}
};
} else if (lines[i].startsWith('### Minimal, reproducible example')) {
set(lines.slice(j, i).join('\n').trim());
j = i + 1;
set = value => {
mvce = value;
};
} else if (lines[i].startsWith('### Error string')) {
set(lines.slice(j, i).join('\n').trim());
j = i + 1;
set = value => {
if (value.match(/no error/i)) {
behavior.noError = true;
} else {
error = value;
}
};
} else if (lines[i].startsWith('### Puppeteer configuration')) {
set(lines.slice(j, i).join('\n').trim());
j = i + 1;
set = value => {
configuration = value;
};
} else if (lines[i].startsWith('### Puppeteer version')) {
set(lines.slice(j, i).join('\n').trim());
j = i + 1;
set = value => {
puppeteerVersion = removeVersionPrefix(value);
};
} else if (lines[i].startsWith('### Node version')) {
set(lines.slice(j, i).join('\n').trim());
j = i + 1;
set = value => {
nodeVersion = removeVersionPrefix(value);
};
} else if (lines[i].startsWith('### Package manager version')) {
set(lines.slice(j, i).join('\n').trim());
j = i + 1;
set = value => {
packageManagerVersion = removeVersionPrefix(value);
};
} else if (lines[i].startsWith('### Package manager')) {
set(lines.slice(j, i).join('\n').trim());
j = i + 1;
set = value => {
packageManager = value.toLowerCase();
};
} else if (lines[i].startsWith('### Operating system')) {
set(lines.slice(j, i).join('\n').trim());
j = i + 1;
set = value => {
os = value.toLowerCase();
};
}
}
set(lines.slice(j, i).join('\n').trim());
}
let runsOn;
switch (os) {
case 'windows':
runsOn = 'windows-latest';
break;
case 'macos':
runsOn = 'macos-latest';
break;
case 'linux':
runsOn = 'ubuntu-latest';
break;
default:
core.setOutput('errorMessage', ERROR_MESSAGES.unsupportedOs(os));
core.setFailed(`Unsupported OS: ${os}`);
}
if (!SUPPORTED_PACKAGE_MANAGERS.includes(packageManager)) {
core.setOutput(
'errorMessage',
ERROR_MESSAGES.unsupportedPackageManager(packageManager)
);
core.setFailed(`Unsupported package manager: ${packageManager}`);
}
if (!semver.valid(nodeVersion)) {
core.setOutput(
'errorMessage',
ERROR_MESSAGES.invalidNodeVersion(nodeVersion)
);
core.setFailed('Invalid Node version');
}
if (semver.lt(nodeVersion, LAST_SUPPORTED_NODE_VERSION)) {
core.setOutput(
'errorMessage',
ERROR_MESSAGES.unsupportedNodeVersion(nodeVersion)
);
core.setFailed(`Unsupported node version: ${nodeVersion}`);
}
if (!semver.valid(puppeteerVersion)) {
core.setOutput(
'errorMessage',
ERROR_MESSAGES.invalidPuppeteerVersion(puppeteerVersion)
);
core.setFailed(`Invalid puppeteer version: ${puppeteerVersion}`);
}
if (
!LAST_SUPPORTED_PUPPETEER_VERSION ||
semver.lt(puppeteerVersion, LAST_SUPPORTED_PUPPETEER_VERSION)
) {
core.setOutput(
'errorMessage',
ERROR_MESSAGES.unsupportedPuppeteerVersion(puppeteerVersion)
);
core.setFailed(`Unsupported puppeteer version: ${puppeteerVersion}`);
}
if (!semver.valid(packageManagerVersion)) {
core.setOutput(
'errorMessage',
ERROR_MESSAGES.invalidPackageManagerVersion(packageManagerVersion)
);
core.setFailed(`Invalid package manager version: ${packageManagerVersion}`);
}
core.setOutput('errorMessage', '');
core.setOutput('runsOn', runsOn);
core.setOutput('nodeVersion', nodeVersion);
core.setOutput('packageManager', packageManager);
await mkdir('out');
Promise.all([
writeFile(join('out', 'main.ts'), mvce.split('\n').slice(1, -1).join('\n')),
writeFile(
join('out', 'puppeteer-error.txt'),
error.split('\n').slice(1, -1).join('\n')
),
writeFile(
join('out', 'puppeteer.config.js'),
configuration.split('\n').slice(1, -1).join('\n')
),
writeFile(join('out', 'puppeteer-behavior.json'), JSON.stringify(behavior)),
writeFile(
join('out', 'package.json'),
JSON.stringify({
packageManager: `${packageManager}@${packageManagerVersion}`,
scripts: {
start: 'tsx main.ts',
verify: 'tsx verify_issue.ts',
},
dependencies: {
puppeteer: puppeteerVersion,
},
devDependencies: {
tsx: 'latest',
},
})
),
copyFile(
join(
dirname(fileURLToPath(import.meta.url)),
'assets',
'verify_issue.ts'
),
join('out', 'verify_issue.ts')
),
]);
})();

68
tools/assets/verify_issue.ts Executable file
View File

@ -0,0 +1,68 @@
import {spawnSync} from 'child_process';
import {readFile, writeFile} from 'fs/promises';
(async () => {
const error = await readFile('puppeteer-error.txt', 'utf-8');
const behavior = JSON.parse(
await readFile('puppeteer-behavior.json', 'utf-8')
) as {flaky?: boolean; noError?: boolean};
let maxRepetitions = 1;
if (behavior.flaky) {
maxRepetitions = 100;
}
let status: number | null = null;
let stderr = '';
let stdout = '';
const preHook = async () => {
console.log('Writing output and error logs...');
await Promise.all([
writeFile('output.log', stdout),
writeFile('error.log', stderr),
]);
};
let checkStatusWithError: () => Promise<void>;
if (behavior.noError) {
checkStatusWithError = async () => {
if (status === 0) {
await preHook();
console.log('Script ran successfully; no error found.');
process.exit(0);
}
};
} else {
checkStatusWithError = async () => {
if (status !== 0) {
await preHook();
if (stderr.toLowerCase().includes(error.toLowerCase())) {
console.log('Script failed; error found.');
process.exit(0);
}
console.error('Script failed; unknown error found.');
process.exit(1);
}
};
}
for (let i = 0; i < maxRepetitions; ++i) {
const result = spawnSync('npm', ['start'], {
shell: true,
encoding: 'utf-8',
});
status = result.status;
stdout = result.stdout ?? '';
stderr = result.stderr ?? '';
await checkStatusWithError();
}
await preHook();
if (behavior.noError) {
console.error('Script failed; unknown error found.');
} else {
console.error('Script ran successfully; no error found.');
}
process.exit(1);
})();