149 lines
4.5 KiB
Bash
149 lines
4.5 KiB
Bash
#!/usr/bin/env sh
|
|
get_test_report() {
|
|
workdir="$1"
|
|
remote_url="$2"
|
|
branch="$3"
|
|
spec_basedir="$4"
|
|
output_path="$5"
|
|
|
|
cd "$workdir"
|
|
|
|
sh -cx "git clone --depth 1 --single-branch --branch '$branch' '$remote_url' ."
|
|
|
|
sh -cx "npm install"
|
|
|
|
sh -cx "node bin/mocha.js '$spec_basedir'/**/*.spec.js --reporter json-stream" \
|
|
| tee "$output_path"
|
|
}
|
|
|
|
compare_test_reports() {
|
|
spec_basedir="$1"
|
|
diff_out_file="$2"
|
|
workdir="$(mktemp -d)"
|
|
|
|
cd "$workdir"
|
|
|
|
# define the working directories for execution beforehand so that we can
|
|
# normalize spec paths and make them canonical for comparison.
|
|
test_report_1_workdir="$(mktemp -d)"
|
|
test_report_2_workdir="$(mktemp -d)"
|
|
|
|
# paths to output the test reports to
|
|
test_report_1_path="$workdir/main-test-report.json"
|
|
test_report_2_path="$workdir/byteb4rb1e-feature-5049-test-report.json"
|
|
|
|
# execute integration test suites on main branch
|
|
get_test_report "$test_report_1_workdir" \
|
|
'https://github.com/mochajs/mocha.git' \
|
|
'main' \
|
|
"$spec_basedir" \
|
|
"$test_report_1_path" &
|
|
jid="$!"
|
|
|
|
# execute integration test suites on PR branch
|
|
get_test_report "$test_report_2_workdir" \
|
|
'https://github.com/byteb4rb1e/mocha.git' \
|
|
'feature/5049' \
|
|
"$spec_basedir" \
|
|
"$test_report_2_path" &
|
|
jid="$jid $!"
|
|
|
|
wait $jid
|
|
|
|
# make the path pretty for NT environments
|
|
# only applicable for Cygwin/MSYS2 - in my case
|
|
diff_out_file="$(
|
|
cygpath -w "$diff_out_file" | sed 's|\\|\\\\|g'
|
|
)"
|
|
test_report_1_workdir="$(
|
|
cygpath -w "$test_report_1_workdir" | sed 's|\\|\\\\|g'
|
|
)"
|
|
test_report_2_workdir="$(
|
|
cygpath -w "$test_report_2_workdir" | sed 's|\\|\\\\|g'
|
|
)"
|
|
|
|
test_report_1_path="$(
|
|
cygpath -w "$test_report_1_path" | sed 's|\\|\\\\|g'
|
|
)"
|
|
test_report_2_path="$(
|
|
cygpath -w "$test_report_2_path" | sed 's|\\|\\\\|g'
|
|
)"
|
|
|
|
node << EOF
|
|
import { open } from 'node:fs/promises';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { createHash } from 'node:crypto';
|
|
|
|
const testReport1Workdir = '$test_report_1_workdir';
|
|
const testReport2Workdir = '$test_report_2_workdir';
|
|
|
|
const testReport1Path = '$test_report_1_path';
|
|
const testReport2Path = '$test_report_2_path';
|
|
|
|
const diffOutputPath = '$diff_out_file';
|
|
|
|
const mainResultHashes = {
|
|
pass: [],
|
|
fail: []
|
|
};
|
|
|
|
(async () => {
|
|
const f1 = await open(testReport1Path);
|
|
const f2 = await open(testReport2Path);
|
|
|
|
// iterate over every streamed JSON object, extract the case state and
|
|
// metadata, calculate a canonicalized hash and store it in an array for
|
|
// lookup when iterating over the other test results
|
|
for await (const line of f1.readLines()) {
|
|
const [caseState, caseMeta] = JSON.parse(line);
|
|
|
|
// skip anything that has nothing to do with a concrete test case
|
|
if (!['fail', 'pass'].includes(caseState)) continue;
|
|
|
|
//canonicalize the test suite file path
|
|
const canonFilePath = path.relative(testReport1Workdir, caseMeta.file);
|
|
|
|
// calculate a hash of the test case
|
|
const hash = createHash('sha256');
|
|
hash.update(canonFilePath);
|
|
hash.update(caseMeta.fullTitle);
|
|
const hashDigest = hash.copy().digest('hex');
|
|
|
|
mainResultHashes[caseState].push(hashDigest);
|
|
}
|
|
|
|
for await (const line of f2.readLines()) {
|
|
const [caseState, caseMeta] = JSON.parse(line);
|
|
|
|
// skip anything that has nothing to do with a concrete test case
|
|
if (!['fail', 'pass'].includes(caseState)) continue;
|
|
|
|
//canonicalize the test suite file path
|
|
const canonFilePath = path.relative(testReport2Workdir, caseMeta.file);
|
|
|
|
// calculate a hash of the test case
|
|
const hash = createHash('sha256');
|
|
hash.update(canonFilePath);
|
|
hash.update(caseMeta.fullTitle);
|
|
const hashDigest = hash.copy().digest('hex');
|
|
|
|
if (!mainResultHashes[caseState].includes(hashDigest)) {
|
|
var msg = \`$spec_basedir: mismatched, (is) '\${caseState}': \`;
|
|
msg += JSON.stringify(caseMeta, null, 4);
|
|
msg += '\\n\\n';
|
|
|
|
console.log(msg);
|
|
|
|
fs.appendFileSync(diffOutputPath, msg);
|
|
}
|
|
}
|
|
})()
|
|
EOF
|
|
}
|
|
|
|
out_file="$(pwd)/mocha-5094-test-reports-diff";
|
|
|
|
compare_test_reports 'test/integration' "$out_file"
|
|
compare_test_reports 'test/unit' "$out_file"
|
|
compare_test_reports 'test/node-unit' "$out_file"
|