diff --git a/pom.xml b/pom.xml index d3216a94..4e1a5137 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,7 @@ tasks/xml tasks/zoom runtime/opentelemetry + runtime/codecoverage diff --git a/runtime/codecoverage/README.md b/runtime/codecoverage/README.md new file mode 100644 index 00000000..1ba8ed8d --- /dev/null +++ b/runtime/codecoverage/README.md @@ -0,0 +1,21 @@ +# codecoverage + +A plugin for Concord runtime-v2 that adds code coverage capabilities. + +## Usage + +To use the plugin, add the following dependency to your Concord process: + +```yaml +configuration: + dependencies: + - mvn://com.walmartlabs.concord.plugins:codecoverage: +``` + +## Generating HTML report with LCOV + +The plugin produces a file in [the LCOV format](https://github.com/linux-test-project/lcov). + +1. Download coverage info: `/api/v1/process/${INSTANCE_ID}/attachment/coverage.info` +2. Download and unzip process flows: `/api/v1/process/${INSTANCE_ID}/attachment/flows.zip` +3. Generate HTML with: `genhtml "coverage.info" --output-directory "html"` diff --git a/runtime/codecoverage/pom.xml b/runtime/codecoverage/pom.xml new file mode 100644 index 00000000..86de1136 --- /dev/null +++ b/runtime/codecoverage/pom.xml @@ -0,0 +1,142 @@ + + + + 4.0.0 + + + com.walmartlabs.concord.plugins + concord-plugins-parent + 2.6.1-SNAPSHOT + ../../pom.xml + + + codecoverage + takari-jar + + + + + + + com.walmartlabs.concord + concord-common + provided + + + com.walmartlabs.concord + concord-client2 + provided + + + com.walmartlabs.concord.runtime.v2 + concord-runtime-model-v2 + provided + + + com.walmartlabs.concord.runtime.v2 + concord-runner-v2 + provided + + + com.walmartlabs.concord.runtime.v2 + concord-runtime-vm-v2 + provided + + + com.walmartlabs.concord + concord-sdk + provided + + + com.walmartlabs.concord.runtime.v2 + concord-runtime-sdk-v2 + provided + + + + org.apache.commons + commons-compress + provided + + + + org.slf4j + slf4j-api + provided + + + + javax.inject + javax.inject + provided + + + com.google.inject + guice + provided + + + + com.fasterxml.jackson.core + jackson-core + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + provided + + + + + org.immutables + value + provided + + + org.immutables + builder + provided + + + com.google.code.findbugs + jsr305 + provided + + + com.google.errorprone + error_prone_annotations + provided + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + + + + + dev.ybrig.concord + concord-maven-plugin + + + + diff --git a/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/CodeCoverage.java b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/CodeCoverage.java new file mode 100644 index 00000000..63ce9cc0 --- /dev/null +++ b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/CodeCoverage.java @@ -0,0 +1,180 @@ +package com.walmartlabs.concord.plugins.codecoverage; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2024 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.IOUtils; +import com.walmartlabs.concord.runtime.v2.ProcessDefinitionUtils; +import com.walmartlabs.concord.runtime.v2.model.FlowCall; +import com.walmartlabs.concord.runtime.v2.model.ProcessDefinition; +import com.walmartlabs.concord.runtime.v2.model.Step; +import com.walmartlabs.concord.runtime.v2.runner.PersistenceService; +import com.walmartlabs.concord.runtime.v2.runner.vm.ElementEventProducer; +import com.walmartlabs.concord.runtime.v2.runner.vm.FlowCallCommand; +import com.walmartlabs.concord.runtime.v2.runner.vm.TaskCallCommand; +import com.walmartlabs.concord.runtime.v2.sdk.WorkingDirectory; +import com.walmartlabs.concord.svm.*; +import com.walmartlabs.concord.svm.Runtime; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Objects; +import java.util.stream.Collectors; + +public class CodeCoverage implements ExecutionListener { + + public static final Logger log = LoggerFactory.getLogger(CodeCoverage.class); + + private static final String COVERAGE_INFO_FILENAME = "coverage.info"; + private static final String FLOWS_FILENAME = "flows.zip"; + + private final StepsRecorder steps; + private final PersistenceService persistenceService; + private final Path workDir; + + @Inject + public CodeCoverage(StepsRecorder steps, PersistenceService persistenceService, WorkingDirectory workingDirectory) { + this.steps = steps; + this.persistenceService = persistenceService; + this.workDir = workingDirectory.getValue(); + } + + @Override + public void beforeProcessStart(Runtime runtime, State state) { + saveFlows(runtime.getService(ProcessDefinition.class)); + } + + @Override + public Result beforeCommand(Runtime runtime, VM vm, State state, ThreadId threadId, Command cmd) { + // we need the name of the flow, so we can handle the call step only in `afterCommand` + if (cmd instanceof FlowCallCommand) { + return Result.CONTINUE; + } + + if (cmd instanceof ElementEventProducer eep) { + processStep(eep.getStep(), runtime, state, threadId); + } else if (cmd instanceof TaskCallCommand tcc) { + processStep(tcc.getStep(), runtime, state, threadId); + } + + return Result.CONTINUE; + } + + @Override + public Result afterCommand(Runtime runtime, VM vm, State state, ThreadId threadId, Command cmd) { + if (cmd instanceof FlowCallCommand fcc) { + processStep(fcc.getStep(), runtime, state, threadId); + } + return Result.CONTINUE; + } + + private void processStep(Step step, Runtime runtime, State state, ThreadId threadId) { + var loc = step.getLocation(); + if (loc == null || loc.lineNum() < 0 || loc.fileName() == null) { + return; + } + + var pd = runtime.getService(ProcessDefinition.class); + + steps.record(StepInfo.builder() + .fileName(Objects.requireNonNull(loc.fileName())) + .line(loc.lineNum()) + .processDefinitionId(ProcessDefinitionUtils.getCurrentFlowName(pd, step)) + .flowCallName(flowCallName(step, state, threadId)) + .build()); + } + + @Override + public void onProcessError(Runtime runtime, State state, Exception e) { + generateReport(runtime); + } + + @Override + public void afterProcessEnds(Runtime runtime, State state, Frame lastFrame) { + if (isSuspended(state)) { + return; + } + + generateReport(runtime); + } + + private void generateReport(Runtime runtime) { + log.info("Generating code coverage info..."); + + try { + var reportProducer = new LcovReportProducer(runtime.getService(ProcessDefinition.class)); + reportProducer.onSteps(steps.list()); + + steps.cleanup(); + + persistenceService.persistFile(COVERAGE_INFO_FILENAME, reportProducer::produce, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + } catch (Exception e) { + throw new RuntimeException("Can't generate code coverage report", e); + } + + log.info("Coverage info saved as attachment with name '{}'", COVERAGE_INFO_FILENAME); + } + + private static boolean isSuspended(State state) { + return state.threadStatus().entrySet().stream() + .anyMatch(e -> e.getValue() == ThreadStatus.SUSPENDED); + } + + private static String flowCallName(Step step, State state, ThreadId threadId) { + if (step instanceof FlowCall) { + return FlowCallCommand.getFlowName(state, threadId); + } + return null; + } + + private void saveFlows(ProcessDefinition processDefinition) { + var fileNames = processDefinition.flows().values().stream() + .map(v -> v.location().fileName()) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + + persistenceService.persistFile(FLOWS_FILENAME, out -> { + try (ZipArchiveOutputStream zip = new ZipArchiveOutputStream(out)) { + + for (var fileName : fileNames) { + var file = workDir.resolve(fileName); + if (Files.notExists(file)) { + log.warn("CodeCoverage: can't save flow '{}' -> file not exists. This is most likely a bug", fileName); + continue; + } + + try { + IOUtils.zipFile(zip, file, fileName); + } catch (IOException ex) { + log.error("CodeCoverage: failed to add file '{}'. Error: {}", fileName, ex.getMessage()); + throw ex; + } + } + } + }); + log.debug("CodeCoverage: flows saved as '{}' process attachment", FLOWS_FILENAME); + } +} diff --git a/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/CodecoverageModule.java b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/CodecoverageModule.java new file mode 100644 index 00000000..283e034d --- /dev/null +++ b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/CodecoverageModule.java @@ -0,0 +1,38 @@ +package com.walmartlabs.concord.plugins.codecoverage; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2024 Walmart Inc., Concord Authors + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.multibindings.Multibinder; +import com.walmartlabs.concord.svm.ExecutionListener; + +import javax.inject.Named; + +@Named +public class CodecoverageModule implements Module { + + @Override + public void configure (Binder binder){ + var executionListeners = Multibinder.newSetBinder(binder, ExecutionListener.class); + executionListeners.addBinding().to(CodeCoverage.class); + } +} diff --git a/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/LcovReportProducer.java b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/LcovReportProducer.java new file mode 100644 index 00000000..548933bb --- /dev/null +++ b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/LcovReportProducer.java @@ -0,0 +1,216 @@ +package com.walmartlabs.concord.plugins.codecoverage; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2024 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.runtime.v2.model.ProcessDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class LcovReportProducer { + + private static final Logger log = LoggerFactory.getLogger(LcovReportProducer.class); + + private final Map statsPerFile = new HashMap<>(); + + public LcovReportProducer(ProcessDefinition processDefinition) { + init(processDefinition); + } + + public void onSteps(List steps) { + steps.forEach(this::onStep); + } + + public void onStep(StepInfo step) { + var fileName = step.fileName(); + + var stats = statsPerFile.get(fileName); + if (stats == null) { + log.warn("Can't find definitions for '{}'. This is most likely a bug.", fileName); + return; + } + + stats.onLineExecuted( step.line()); + + var flowCallName = step.flowCallName(); + if (flowCallName != null) { + var statsForFlow = findStatsForFlow(flowCallName); + if (statsForFlow != null) { + statsForFlow.onFlowCall(flowCallName); + } + } + + var processDefinitionId = step.processDefinitionId(); + if (processDefinitionId != null) { + // as we do not have call step for entry point + var statsForFlow = findStatsForFlow(processDefinitionId); + if (statsForFlow != null) { + statsForFlow.markCalled(processDefinitionId); + } + } + } + + + public void produce(OutputStream out) throws IOException { + try (var writer = new BufferedWriter(new OutputStreamWriter(out))) { + for (var statsEntry : statsPerFile.entrySet()) { + // TN (Test Name): + // Optional. Can be left empty or used to specify the name of the test. + writer.write("TN:"); + writer.newLine(); + + // SF (Source File Path): + // Specifies the path to the source file for which the coverage data is provided. + writer.write("SF:" + statsEntry.getKey()); + writer.newLine(); + + var stats = statsEntry.getValue(); + for (var flowEntry : stats.flowLocationByName.entrySet()) { + // FN (Function): + // The start line and name of the function in the source file. + writer.write(String.format("FN:%d,%s", flowEntry.getValue(), flowEntry.getKey())); + writer.newLine(); + } + + for (var execEntry : stats.flowExecCountByName.entrySet()) { + // FNDA (Function Data): + // The number of times the function was executed, followed by the function name. + writer.write(String.format("FNDA:%d,%s", execEntry.getValue(), execEntry.getKey())); + writer.newLine(); + } + + // Function summary + // FNF (Functions Found): + // The number of functions found in the file. + writer.write("FNF:" + stats.flowLocationByName.size()); + writer.newLine(); + + // FNH (Functions Hit): + // The number of functions that were executed at least once. + writer.write("FNH:" + stats.flowExecCountByName.size()); + writer.newLine(); + + // Line coverage data + // DA (Data Array): + // Line number and the execution count for that line. + for (var e : stats.stepExecCountByLineNumber.entrySet()) { + writer.write(String.format("DA:%d,%d", e.getKey(), e.getValue())); + writer.newLine(); + } + + // Branch coverage data + // BRDA (Branch Data Array): + // Line number, block number, branch number, and the execution count for branches. + // + // Branch summary + // BRF (Branches Found): + // The total number of branches found. + // + // BRH (Branches Hit): + // The number of branches that were taken. + + // line summary + // LF (Lines Found): + // The total number of lines found in the source file. + writer.write("LF:7"); + writer.newLine(); + // LH (Lines Hit): + // The number of lines that were executed at least once. + writer.write("LH:6"); + writer.newLine(); + + // End of record + writer.write("end_of_record"); + writer.newLine(); + } + } + } + + private void init(ProcessDefinition processDefinition) { + for (var fd : processDefinition.flows().entrySet()) { + var fileName = fd.getValue().location().fileName(); + + var stats = statsPerFile.computeIfAbsent(fileName, k -> new Stats()); + stats.init(fd.getKey(), fd.getValue().location().lineNum()); + } + } + + private Stats findStatsForFlow(String flow) { + for (var stats : statsPerFile.entrySet()) { + if (stats.getValue().containsFlow(flow)) { + return stats.getValue(); + } + } + log.warn("Can't find stats for {} flow. This is most likely a bug.", flow); + return null; + } + + private static class Stats { + + private final Map flowLocationByName = new HashMap<>(); + + private final Map flowExecCountByName = new HashMap<>(); + + private final Map stepExecCountByLineNumber = new HashMap<>(); + + public void init(String flowName, int location) { + flowLocationByName.put(flowName, location); + stepExecCountByLineNumber.put(location, 0); + } + + public void onLineExecuted(int lineNum) { + stepExecCountByLineNumber.compute(lineNum, (k, v) -> v == null ? 1 : v + 1); + } + + public void onFlowCall(String flow) { + assertContainsFlow(flow); + + flowExecCountByName.compute(flow, (k, v) -> v == null ? 1 : v + 1); + onLineExecuted(flowLocationByName.get(flow)); + } + + public void markCalled(String flow) { + assertContainsFlow(flow); + + Integer prev = flowExecCountByName.putIfAbsent(flow, 1); + if (prev == null) { + onLineExecuted(flowLocationByName.get(flow)); + } + } + + public boolean containsFlow(String flow) { + return flowLocationByName.containsKey(flow); + } + + private void assertContainsFlow(String flow) { + if (!flowLocationByName.containsKey(flow)) { + throw new IllegalArgumentException("Trying update stats for flow without flow definition " + flow); + } + } + } +} diff --git a/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/StepInfo.java b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/StepInfo.java new file mode 100644 index 00000000..6d8c3e4f --- /dev/null +++ b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/StepInfo.java @@ -0,0 +1,52 @@ +package com.walmartlabs.concord.plugins.codecoverage; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2024 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.immutables.value.Value; + +import javax.annotation.Nullable; + +@Value.Immutable +@Value.Style(jdkOnly = true) +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(as = ImmutableStepInfo.class) +@JsonDeserialize(as = ImmutableStepInfo.class) +public interface StepInfo { + + String fileName(); + + int line(); + + @Nullable + String processDefinitionId(); + + @Nullable + String flowCallName(); + + static ImmutableStepInfo.Builder builder() { + return ImmutableStepInfo.builder(); + } +} diff --git a/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/StepsRecorder.java b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/StepsRecorder.java new file mode 100644 index 00000000..00900de3 --- /dev/null +++ b/runtime/codecoverage/src/main/java/com/walmartlabs/concord/plugins/codecoverage/StepsRecorder.java @@ -0,0 +1,80 @@ +package com.walmartlabs.concord.plugins.codecoverage; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2024 Walmart Inc., Concord Authors + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import com.walmartlabs.concord.runtime.v2.runner.PersistenceService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.io.IOException; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.stream.Stream; + +@Singleton +public class StepsRecorder { + + private static final Logger log = LoggerFactory.getLogger(StepsRecorder.class); + + private static final TypeReference> STEPS_TYPE = new TypeReference<>() { + }; + + private static final String FILE_NAME = "code-coverage-steps.yaml"; + + private final PersistenceService persistenceService; + private final ObjectMapper objectMapper; + + @Inject + public StepsRecorder(PersistenceService persistenceService) { + this.persistenceService = persistenceService; + this.objectMapper = new ObjectMapper( + new YAMLFactory() + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER)); + } + + public synchronized void record(StepInfo step) { + persistenceService.persistFile(FILE_NAME, + out -> objectMapper.writeValue(out, List.of(step)), + StandardOpenOption.CREATE, StandardOpenOption.APPEND); + } + + public List list() { + var result = persistenceService.loadPersistedFile(FILE_NAME, in -> objectMapper.readValue(in, STEPS_TYPE)); + if (result == null) { + return List.of(); + } + return result; + } + + public synchronized void cleanup() { + try { + persistenceService.deletePersistedFile(FILE_NAME); + } catch (IOException e) { + log.warn("Can't cleanup steps from state: {}", e.getMessage()); + } + } +} diff --git a/runtime/codecoverage/src/test/resources/gen_html.bash b/runtime/codecoverage/src/test/resources/gen_html.bash new file mode 100755 index 00000000..7dda4678 --- /dev/null +++ b/runtime/codecoverage/src/test/resources/gen_html.bash @@ -0,0 +1,54 @@ +#!/bin/bash + +if [[ $# -lt 3 ]]; then + echo "Usage: $0 " + exit 1 +fi + +BASE_URL=$1 +INSTANCE_ID=$2 +TOKEN=$3 + +OUTPUT_DIR="./code-coverage" + +FILES=("coverage.info" "flows.zip") +UNZIP_DIR="${OUTPUT_DIR}" + +mkdir -p "$OUTPUT_DIR" "$UNZIP_DIR" + +download_file() { + local file_name=$1 + local download_url="${BASE_URL}/api/v1/process/${INSTANCE_ID}/attachment/${file_name}" + local output_file="${OUTPUT_DIR}/${file_name}" + + echo "Downloading ${file_name} from ${download_url}..." + + if curl -H "Authorization: ${TOKEN}" -o "${output_file}" -L "${download_url}"; then + echo "File downloaded successfully: ${output_file}" + else + echo "Failed to download the file: ${download_url}" + exit 1 + fi +} + +for file in "${FILES[@]}"; do + download_file "$file" +done + +ZIP_FILE="${OUTPUT_DIR}/flows.zip" + +echo "Unzipping ${ZIP_FILE} into ${UNZIP_DIR}..." + +if unzip -o "$ZIP_FILE" -d "$UNZIP_DIR"; then + echo "File unzipped successfully to: ${UNZIP_DIR}" +else + echo "Failed to unzip file: ${ZIP_FILE}" + exit 1 +fi + +if (cd "${OUTPUT_DIR}" && genhtml "coverage.info" --output-directory "html"); then + echo "HTML report generated successfully in: ${OUTPUT_DIR}/html" +else + echo "Failed to generate HTML report" + exit 1 +fi