#!/usr/bin/env sh get_test_report() { workdir="$1" remote_url="$2" branch="$3" spec_basedir="$4" output_path="$5" cd "$workdir" sh -cx "git clone --depth 1 --single-branch --branch '$branch' '$remote_url' ." sh -cx "npm install" sh -cx "node bin/mocha.js '$spec_basedir'/**/*.spec.js --reporter json-stream" \ | tee "$output_path" } compare_test_reports() { spec_basedir="$1" diff_out_file="$2" workdir="$(mktemp -d)" cd "$workdir" # define the working directories for execution beforehand so that we can # normalize spec paths and make them canonical for comparison. test_report_1_workdir="$(mktemp -d)" test_report_2_workdir="$(mktemp -d)" # paths to output the test reports to test_report_1_path="$workdir/main-test-report.json" test_report_2_path="$workdir/byteb4rb1e-feature-5049-test-report.json" # execute integration test suites on main branch get_test_report "$test_report_1_workdir" \ 'https://github.com/mochajs/mocha.git' \ 'main' \ "$spec_basedir" \ "$test_report_1_path" & jid="$!" # execute integration test suites on PR branch get_test_report "$test_report_2_workdir" \ 'https://github.com/byteb4rb1e/mocha.git' \ 'feature/5049' \ "$spec_basedir" \ "$test_report_2_path" & jid="$jid $!" wait $jid # make the path pretty for NT environments # only applicable for Cygwin/MSYS2 - in my case diff_out_file="$( cygpath -w "$diff_out_file" | sed 's|\\|\\\\|g' )" test_report_1_workdir="$( cygpath -w "$test_report_1_workdir" | sed 's|\\|\\\\|g' )" test_report_2_workdir="$( cygpath -w "$test_report_2_workdir" | sed 's|\\|\\\\|g' )" test_report_1_path="$( cygpath -w "$test_report_1_path" | sed 's|\\|\\\\|g' )" test_report_2_path="$( cygpath -w "$test_report_2_path" | sed 's|\\|\\\\|g' )" node << EOF import { open } from 'node:fs/promises'; import fs from 'node:fs'; import path from 'node:path'; import { createHash } from 'node:crypto'; const testReport1Workdir = '$test_report_1_workdir'; const testReport2Workdir = '$test_report_2_workdir'; const testReport1Path = '$test_report_1_path'; const testReport2Path = '$test_report_2_path'; const diffOutputPath = '$diff_out_file'; const mainResultHashes = { pass: [], fail: [] }; (async () => { const f1 = await open(testReport1Path); const f2 = await open(testReport2Path); // iterate over every streamed JSON object, extract the case state and // metadata, calculate a canonicalized hash and store it in an array for // lookup when iterating over the other test results for await (const line of f1.readLines()) { const [caseState, caseMeta] = JSON.parse(line); // skip anything that has nothing to do with a concrete test case if (!['fail', 'pass'].includes(caseState)) continue; //canonicalize the test suite file path const canonFilePath = path.relative(testReport1Workdir, caseMeta.file); // calculate a hash of the test case const hash = createHash('sha256'); hash.update(canonFilePath); hash.update(caseMeta.fullTitle); const hashDigest = hash.copy().digest('hex'); mainResultHashes[caseState].push(hashDigest); } for await (const line of f2.readLines()) { const [caseState, caseMeta] = JSON.parse(line); // skip anything that has nothing to do with a concrete test case if (!['fail', 'pass'].includes(caseState)) continue; //canonicalize the test suite file path const canonFilePath = path.relative(testReport2Workdir, caseMeta.file); // calculate a hash of the test case const hash = createHash('sha256'); hash.update(canonFilePath); hash.update(caseMeta.fullTitle); const hashDigest = hash.copy().digest('hex'); if (!mainResultHashes[caseState].includes(hashDigest)) { var msg = \`$spec_basedir: mismatched, (is) '\${caseState}': \`; msg += JSON.stringify(caseMeta, null, 4); msg += '\\n\\n'; console.log(msg); fs.appendFileSync(diffOutputPath, msg); } } })() EOF } out_file="$(pwd)/mocha-5094-test-reports-diff"; compare_test_reports 'test/integration' "$out_file" compare_test_reports 'test/unit' "$out_file" compare_test_reports 'test/node-unit' "$out_file"