/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import * as fs from 'fs'; import * as mobx from 'mobx'; import * as readline from 'readline'; import { ObservablePromise } from '../utils/utils'; import { parseScoredPredictionsCsv } from './amlResults'; import { AMLProvider, AMLRun } from './amlSimulations'; import { SimulationRunner, TestRuns } from './simulationRunner'; export class ResolvedAMLRun { @mobx.computed public get tests(): ObservablePromise { const selected = this.amlProvider.selected; if (!selected) { return ObservablePromise.resolve([]); } return new ObservablePromise((async () => { return this.testsForRun(selected); })(), []); } @mobx.computed public get testsToCompareAgainst(): ObservablePromise { const compareAgainstRun = this.amlProvider.compareAgainstRun; if (!compareAgainstRun) { return ObservablePromise.resolve([]); } return new ObservablePromise((async () => { return this.testsForRun(compareAgainstRun); })(), []); } private async testsForRun(run: AMLRun): Promise { const testRuns = await SimulationRunner.readFromStdoutJSON(run.stdoutPath, run.simulationInputPath); if (run.scoredPredictionsJSONL) { const contents = await this.readContentsLineByLine(run.scoredPredictionsJSONL); const evals = parseScoredPredictionsCsv(run.kind, contents); const testRunsMap = new Map(); for (const testRun of testRuns) { testRunsMap.set(testRun.name, testRun); } for (const evaluation of evals) { const testRun = testRunsMap.get(evaluation.caseName); if (!testRun) { console.warn(`Could not find test run for ${evaluation.caseName}`); continue; } testRun.activeEditorLanguageId = evaluation.activeEditorLanguageId; testRun.runs.forEach((run, i) => { run.pass = evaluation.isEachTestRunSuccess[i]; run.errorsOnlyInBefore = evaluation.errorsOnlyInBefore; run.errorsOnlyInAfter = evaluation.errorsOnlyInAfter; run.stdout = evaluation.stdout; run.stderr = evaluation.stderr; run.error ??= evaluation.evaluatorError; if (evaluation.annotations) { run.annotations.push(...evaluation.annotations); } run.generatedTestCaseCount = evaluation.generatedTestCaseCount; run.generatedAssertCount = evaluation.generatedAssertCount; run.expectedDiff = evaluation.expectedDiff; }); } } return testRuns; } private async readContentsLineByLine(filePath: string): Promise { return new Promise((resolve) => { const contents: string[] = []; const rd = readline.createInterface({ input: fs.createReadStream(filePath) }); rd.on('line', function (line) { contents.push(line); }); rd.on('close', function () { resolve(contents); }); }); } constructor( private readonly amlProvider: AMLProvider ) { mobx.makeObservable(this); } }