Speed up `lint-deps`
This commit is contained in:
Evan Hahn
Josh Perez
parent
7ee6584d8b
commit
401cdfdb63
|
|
@ -1,26 +1,27 @@
|
|||
// tslint:disable no-console
|
||||
|
||||
import { readFileSync } from 'fs';
|
||||
import * as fs from 'fs';
|
||||
import { join, relative } from 'path';
|
||||
import normalizePath from 'normalize-path';
|
||||
|
||||
import { sync as fgSync } from 'fast-glob';
|
||||
import { forEach, some, values } from 'lodash';
|
||||
import pMap from 'p-map';
|
||||
import FastGlob from 'fast-glob';
|
||||
|
||||
import { ExceptionType, REASONS, RuleType } from './types';
|
||||
import { ENCODING, loadJSON, sortExceptions } from './util';
|
||||
|
||||
const ALL_REASONS = REASONS.join('|');
|
||||
const now = new Date();
|
||||
|
||||
function getExceptionKey(exception: any) {
|
||||
function getExceptionKey(
|
||||
exception: Pick<ExceptionType, 'rule' | 'path' | 'lineNumber'>
|
||||
): string {
|
||||
return `${exception.rule}-${exception.path}-${exception.lineNumber}`;
|
||||
}
|
||||
|
||||
function createLookup(list: Array<any>) {
|
||||
function createLookup(
|
||||
list: ReadonlyArray<ExceptionType>
|
||||
): { [key: string]: ExceptionType } {
|
||||
const lookup = Object.create(null);
|
||||
|
||||
forEach(list, exception => {
|
||||
list.forEach((exception: ExceptionType) => {
|
||||
const key = getExceptionKey(exception);
|
||||
|
||||
if (lookup[key]) {
|
||||
|
|
@ -39,16 +40,7 @@ const basePath = join(__dirname, '../../..');
|
|||
|
||||
const searchPattern = normalizePath(join(basePath, '**/*.{js,ts,tsx}'));
|
||||
|
||||
const rules: Array<RuleType> = loadJSON(rulesPath);
|
||||
const exceptions: Array<ExceptionType> = loadJSON(exceptionsPath);
|
||||
const exceptionsLookup = createLookup(exceptions);
|
||||
let scannedCount = 0;
|
||||
|
||||
const allSourceFiles = fgSync(searchPattern, { onlyFiles: true });
|
||||
|
||||
const results: Array<ExceptionType> = [];
|
||||
|
||||
const excludedFiles = [
|
||||
const excludedFilesRegexps = [
|
||||
// Non-distributed files
|
||||
'\\.d\\.ts$',
|
||||
|
||||
|
|
@ -295,10 +287,10 @@ const excludedFiles = [
|
|||
'^node_modules/webpack-hot-middleware/.+',
|
||||
'^node_modules/webpack-merge/.+',
|
||||
'^node_modules/webpack/.+',
|
||||
];
|
||||
].map(str => new RegExp(str));
|
||||
|
||||
function setupRules(allRules: Array<RuleType>) {
|
||||
forEach(allRules, (rule, index) => {
|
||||
allRules.forEach((rule: RuleType, index: number) => {
|
||||
if (!rule.name) {
|
||||
throw new Error(`Rule at index ${index} is missing a name`);
|
||||
}
|
||||
|
|
@ -311,93 +303,108 @@ function setupRules(allRules: Array<RuleType>) {
|
|||
});
|
||||
}
|
||||
|
||||
setupRules(rules);
|
||||
async function main(): Promise<void> {
|
||||
const now = new Date();
|
||||
|
||||
forEach(allSourceFiles, file => {
|
||||
const relativePath = relative(basePath, file).replace(/\\/g, '/');
|
||||
if (
|
||||
some(excludedFiles, excluded => {
|
||||
const regex = new RegExp(excluded);
|
||||
const rules: Array<RuleType> = loadJSON(rulesPath);
|
||||
setupRules(rules);
|
||||
|
||||
return regex.test(relativePath);
|
||||
})
|
||||
) {
|
||||
return;
|
||||
const exceptions: Array<ExceptionType> = loadJSON(exceptionsPath);
|
||||
const exceptionsLookup = createLookup(exceptions);
|
||||
|
||||
const results: Array<ExceptionType> = [];
|
||||
let scannedCount = 0;
|
||||
|
||||
await pMap(
|
||||
await FastGlob(searchPattern, { onlyFiles: true }),
|
||||
async (file: string) => {
|
||||
const relativePath = relative(basePath, file).replace(/\\/g, '/');
|
||||
|
||||
const isFileExcluded = excludedFilesRegexps.some(excludedRegexp =>
|
||||
excludedRegexp.test(relativePath)
|
||||
);
|
||||
if (isFileExcluded) {
|
||||
return;
|
||||
}
|
||||
|
||||
scannedCount += 1;
|
||||
|
||||
const lines = (await fs.promises.readFile(file, ENCODING)).split(/\r?\n/);
|
||||
|
||||
rules.forEach((rule: RuleType) => {
|
||||
const excludedModules = rule.excludedModules || [];
|
||||
if (excludedModules.some(module => relativePath.startsWith(module))) {
|
||||
return;
|
||||
}
|
||||
|
||||
lines.forEach((line: string, lineIndex: number) => {
|
||||
if (!rule.regex.test(line)) {
|
||||
return;
|
||||
}
|
||||
// recreate this rule since it has g flag, and carries local state
|
||||
if (rule.expression) {
|
||||
rule.regex = new RegExp(rule.expression, 'g');
|
||||
}
|
||||
|
||||
const lineNumber = lineIndex + 1;
|
||||
|
||||
const exceptionKey = getExceptionKey({
|
||||
rule: rule.name,
|
||||
path: relativePath,
|
||||
lineNumber,
|
||||
});
|
||||
|
||||
const exception = exceptionsLookup[exceptionKey];
|
||||
if (exception && (!exception.line || exception.line === line)) {
|
||||
// tslint:disable-next-line no-dynamic-delete
|
||||
delete exceptionsLookup[exceptionKey];
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
results.push({
|
||||
rule: rule.name,
|
||||
path: relativePath,
|
||||
line: line.length < 300 ? line : undefined,
|
||||
lineNumber,
|
||||
reasonCategory: ALL_REASONS,
|
||||
updated: now.toJSON(),
|
||||
reasonDetail: '<optional>',
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
// Without this, we may run into "too many open files" errors.
|
||||
{ concurrency: 100 }
|
||||
);
|
||||
|
||||
const unusedExceptions = Object.values(exceptionsLookup);
|
||||
|
||||
console.log(
|
||||
`${scannedCount} files scanned.`,
|
||||
`${results.length} questionable lines,`,
|
||||
`${unusedExceptions.length} unused exceptions,`,
|
||||
`${exceptions.length} total exceptions.`
|
||||
);
|
||||
|
||||
if (results.length === 0 && unusedExceptions.length === 0) {
|
||||
process.exit();
|
||||
}
|
||||
|
||||
scannedCount += 1;
|
||||
|
||||
const fileContents = readFileSync(file, ENCODING);
|
||||
const lines = fileContents.split('\n');
|
||||
|
||||
forEach(rules, (rule: RuleType) => {
|
||||
const excludedModules = rule.excludedModules || [];
|
||||
if (some(excludedModules, module => relativePath.startsWith(module))) {
|
||||
return;
|
||||
}
|
||||
|
||||
forEach(lines, (rawLine, lineIndex) => {
|
||||
const line = rawLine.replace(/\r/g, '');
|
||||
|
||||
if (!rule.regex.test(line)) {
|
||||
return;
|
||||
}
|
||||
// recreate this rule since it has g flag, and carries local state
|
||||
if (rule.expression) {
|
||||
rule.regex = new RegExp(rule.expression, 'g');
|
||||
}
|
||||
|
||||
const path = relativePath;
|
||||
const lineNumber = lineIndex + 1;
|
||||
|
||||
const exceptionKey = getExceptionKey({
|
||||
rule: rule.name,
|
||||
path: relativePath,
|
||||
lineNumber,
|
||||
});
|
||||
|
||||
const exception = exceptionsLookup[exceptionKey];
|
||||
if (exception && (!exception.line || exception.line === line)) {
|
||||
// tslint:disable-next-line no-dynamic-delete
|
||||
delete exceptionsLookup[exceptionKey];
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
results.push({
|
||||
rule: rule.name,
|
||||
path,
|
||||
line: line.length < 300 ? line : undefined,
|
||||
lineNumber,
|
||||
reasonCategory: ALL_REASONS,
|
||||
updated: now.toJSON(),
|
||||
reasonDetail: '<optional>',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const unusedExceptions = values(exceptionsLookup);
|
||||
|
||||
console.log(
|
||||
`${scannedCount} files scanned.`,
|
||||
`${results.length} questionable lines,`,
|
||||
`${unusedExceptions.length} unused exceptions,`,
|
||||
`${exceptions.length} total exceptions.`
|
||||
);
|
||||
|
||||
if (results.length === 0 && unusedExceptions.length === 0) {
|
||||
process.exit();
|
||||
}
|
||||
|
||||
console.log();
|
||||
console.log('Questionable lines:');
|
||||
console.log(JSON.stringify(sortExceptions(results), null, ' '));
|
||||
|
||||
if (unusedExceptions.length) {
|
||||
console.log();
|
||||
console.log('Unused exceptions!');
|
||||
console.log(JSON.stringify(sortExceptions(unusedExceptions), null, ' '));
|
||||
console.log('Questionable lines:');
|
||||
console.log(JSON.stringify(sortExceptions(results), null, ' '));
|
||||
|
||||
if (unusedExceptions.length) {
|
||||
console.log();
|
||||
console.log('Unused exceptions!');
|
||||
console.log(JSON.stringify(sortExceptions(unusedExceptions), null, ' '));
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
main().catch(err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue