Skip to content

Commit

Permalink
[RM] Feature #55153. Provide possibility to generate Aggregate Report…
Browse files Browse the repository at this point in the history
… on Statistics tab via REST API

(cherry picked from commit c5b88b4776ed478743c4a97e79210ecf0ad7e51f)

Change-Id: I7a28e883e466baca1ad6691d5279ed2afe8eb366
  • Loading branch information
IvanSmetanin authored and Nikita-Smirnov-Exactpro committed Apr 26, 2019
1 parent 7dad611 commit b015090
Show file tree
Hide file tree
Showing 8 changed files with 207 additions and 41 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,28 @@
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;

import com.exactpro.sf.embedded.statistics.entities.SfInstance;
import com.exactpro.sf.embedded.statistics.entities.Tag;
import com.exactpro.sf.embedded.statistics.handlers.IStatisticsReportHandler;
import com.exactpro.sf.embedded.statistics.storage.IStatisticsStorage;
import com.exactpro.sf.embedded.statistics.storage.StatisticsReportingStorage;
import com.google.common.collect.Iterables;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DurationFormatUtils;
Expand Down Expand Up @@ -92,6 +102,9 @@ public class StatisticsUtils {
EXECUTION_TIME_COLUMN, USER_NAME_COLUMN, SF_COLUMN, SERVICES_USED_COLUMN,
COMMENT_COLUMN, KNOWN_BUGS_COLUMN };

// average number of test cases per matrix is 100 so we decided to choose this limit for loading matrices
private static final int LOADING_MATRIX_LIMIT = 100;

private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final ObjectWriter LIST_JSON_WRITER = OBJECT_MAPPER.writer();
private static final ObjectReader LIST_JSON_READER = OBJECT_MAPPER.reader().forType(List.class);
Expand Down Expand Up @@ -172,6 +185,31 @@ public static void writeTagGroupReportToCsv(OutputStream outputStream, List<TagG
}
}

public static void generateAggregatedReport(StatisticsService statisticsService,
AggregateReportParameters params,
IStatisticsReportHandler statisticsReportHandler) {
statisticsReportHandler.reset();
StatisticsReportingStorage reportingStorage = statisticsService.getReportingStorage();
SortedMap<Long, List<Long>> matrixToTestCaseIds = reportingStorage.getMatrixRunAndTestCaseRunIDs(params);
if (!matrixToTestCaseIds.isEmpty()) {
Iterator<List<Long>> matrixIdPartsIterator =
Iterables.partition(matrixToTestCaseIds.keySet(), LOADING_MATRIX_LIMIT).iterator();
Iterator<List<List<Long>>> testCaseIdPartsIterator =
Iterables.partition(matrixToTestCaseIds.values(), LOADING_MATRIX_LIMIT).iterator();
while (matrixIdPartsIterator.hasNext() && testCaseIdPartsIterator.hasNext()) {
List<Long> batchMatrixIds = matrixIdPartsIterator.next();
List<Long> batchTestCaseIds = testCaseIdPartsIterator.next()
.stream()
.flatMap(List::stream)
.collect(Collectors.toList());
params.setMatrixRunIds(batchMatrixIds);
params.setTestCaseRunIds(batchTestCaseIds);
statisticsReportHandler.handleMatrixRunTestCases(reportingStorage.generateAggregatedReport(params), reportingStorage);
}
}
statisticsReportHandler.finalize(reportingStorage);
}

public static String createStatsPerTagsName() {
return createCsvFileName("sf_stats_per_tags");
}
Expand Down Expand Up @@ -612,4 +650,35 @@ public static void extractMatrixInfoRows(List<AggregatedReportRow> rows) {
}
}
}

public static void loadSfInstanceIdsFromDb(IStatisticsStorage statisticsStorage,
AggregateReportParameters parameters) {
if (CollectionUtils.isNotEmpty(parameters.getSfInstances())) {
for (SfInstance instance : parameters.getSfInstances()) {
SfInstance sfInstanceFromDb =
statisticsStorage.getSfInstance(instance.getHost(), String.valueOf(instance.getPort()), instance.getName());
if (sfInstanceFromDb == null) {
String message = String.format("Not registered SF Instance [host:%s, port:%s, name:%s] " +
"in the stat db", instance.getHost(), instance.getPort(), instance.getName());
throw new IllegalArgumentException(message);
}
instance.setId(sfInstanceFromDb.getId());
}

}
}

public static void loadTagIdsFromDb(IStatisticsStorage statisticsStorage, AggregateReportParameters parameters) {
if (CollectionUtils.isNotEmpty(parameters.getTags())) {
for (Tag tag : parameters.getTags()) {
Tag tagFromDb = statisticsStorage.getTagByName(tag.getName());
if (tagFromDb == null) {
String message = String.format("Not registered Tag [name:%s] in the stat db", tag.getName());
throw new IllegalArgumentException(message);
}
tag.setId(tagFromDb.getId());
}

}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
public interface IStatisticsStorage extends IHibernateStorage {

SfInstance loadSfInstance(String host, String port, String sfName);

SfInstance getSfInstance(String host, String port, String sfName);

TestCase loadUnknownTestCase();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -559,13 +559,18 @@ private Map<Long, List<Tag>> readTagsResultSet(List<Object[]> resultSet, BiConsu
}

public SortedMap<Long, List<Long>> getMatrixRunAndTestCaseRunIDs(AggregateReportParameters params) {
boolean tagsFilter = params.getTags() != null && !params.getTags().isEmpty();
boolean tagsFilter = CollectionUtils.isNotEmpty(params.getTags());
boolean sfInstancesFilter = CollectionUtils.isNotEmpty(params.getSfInstances());

String queryString = "select MR.id, TCR.id "
+ "from TestCaseRun as TCR "
+ "right join TCR.matrixRun as MR "
+ "join MR.sfInstance as SF "
+ "where MR.startTime >= :from and MR.finishTime <= :to and SF.id in (:ids) ";
+ "where MR.startTime >= :from and MR.finishTime <= :to ";

if (sfInstancesFilter) {
queryString += "and SF.id in (:ids) ";
}

if (tagsFilter) {
queryString += "and (exists (select MR2.id from "
Expand All @@ -589,8 +594,9 @@ public SortedMap<Long, List<Long>> getMatrixRunAndTestCaseRunIDs(AggregateReport
Query query = session.createQuery(queryString);
query.setParameter("from", params.getFrom());
query.setParameter("to", params.getTo());
query.setParameterList("ids", toIds(params.getSfInstances()));

if (sfInstancesFilter) {
query.setParameterList("ids", toIds(params.getSfInstances()));
}
if (tagsFilter) {
Object[] tagsIds = tagsToIds(params.getTags());
query.setParameterList("mrTags", tagsIds);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.List;
import java.util.Map;

import org.apache.commons.collections.CollectionUtils;
import org.hibernate.SessionFactory;
import org.hibernate.cfg.Configuration;
import org.hibernate.cfg.DefaultComponentSafeNamingStrategy;
Expand Down Expand Up @@ -111,16 +112,10 @@ protected void configure(HibernateStorageSettings settings, SessionFactory sessi

@Override
public SfInstance loadSfInstance(String host, String port, String sfName) {

SfInstance result;

List<Criterion> criterions = new ArrayList<Criterion>();

criterions.add(Restrictions.eq("host", host));
criterions.add(Restrictions.eq("port", Integer.parseInt( port )));
criterions.add(Restrictions.eq("name", sfName));

List<SfInstance> queryResults = storage.getAllEntities(SfInstance.class, criterions);
List<SfInstance> queryResults = loadSfInstancesBy(host, port, sfName);

if(!queryResults.isEmpty()) {

Expand All @@ -145,8 +140,17 @@ public SfInstance loadSfInstance(String host, String port, String sfName) {
return result;

}

@Override

@Override
public SfInstance getSfInstance(String host, String port, String sfName) {
List<SfInstance> instances = loadSfInstancesBy(host, port, sfName);
if (CollectionUtils.isNotEmpty(instances)) {
return instances.get(0);
}
return null;
}

@Override
public TestCase loadUnknownTestCase() {

TestCase result = storage.getEntityByField(TestCase.class, "testCaseId", UNKNOWN_TC_ID);
Expand Down Expand Up @@ -539,4 +543,12 @@ public TestCaseRun getTestCaseRunById(long id) {
public MatrixRun getMatrixRunById(long id) {
return (MatrixRun)storage.getEntityById(MatrixRun.class, id);
}

private List<SfInstance> loadSfInstancesBy(String host, String port, String sfName) {
List<Criterion> criterions = new ArrayList<Criterion>();
criterions.add(Restrictions.eq("host", host));
criterions.add(Restrictions.eq("port", Integer.parseInt( port )));
criterions.add(Restrictions.eq("name", sfName));
return storage.getAllEntities(SfInstance.class, criterions);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,17 @@
import com.exactpro.sf.embedded.statistics.entities.SfInstance;
import com.exactpro.sf.embedded.statistics.entities.Tag;
import com.exactpro.sf.util.DateTimeUtility;
import com.fasterxml.jackson.annotation.JsonSetter;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;

import java.time.LocalDateTime;

public class AggregateReportParameters {

@JsonDeserialize(using = LocalDateTimeDeserializer.class)
private LocalDateTime from;

@JsonDeserialize(using = LocalDateTimeDeserializer.class)
private LocalDateTime to;

private String matrixNamePattern;
Expand Down Expand Up @@ -59,10 +64,13 @@ public class AggregateReportParameters {

private boolean includeExecutedInFromToRange;

private String reportType;

public LocalDateTime getFrom() {
return from;
}

@JsonSetter
public void setFrom(LocalDateTime from) {
this.from = from;
}
Expand All @@ -75,6 +83,7 @@ public LocalDateTime getTo() {
return to;
}

@JsonSetter
public void setTo(LocalDateTime to) {
this.to = to;
}
Expand Down Expand Up @@ -202,4 +211,12 @@ public boolean isIncludeExecutedInFromToRange() {
public void setIncludeExecutedInFromToRange(boolean includeExecutedInFromToRange) {
this.includeExecutedInFromToRange = includeExecutedInFromToRange;
}

public String getReportType() {
return reportType;
}

public void setReportType(String reportType) {
this.reportType = reportType;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/******************************************************************************
* Copyright (c) 2009-2019, Exactpro Systems LLC
* www.exactpro.com
* Build Software to Test Software
*
* All rights reserved.
* This is unpublished, licensed software, confidential and proprietary
* information which is the property of Exactpro Systems LLC or its licensors.
******************************************************************************/
package com.exactpro.sf.embedded.statistics.storage.reporting;

import com.exactpro.sf.util.DateTimeUtility;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;

import java.io.IOException;
import java.time.LocalDateTime;

public class LocalDateTimeDeserializer extends JsonDeserializer<LocalDateTime> {
@Override
public LocalDateTime deserialize(JsonParser arg0, DeserializationContext arg1) throws IOException {
return LocalDateTime.parse(arg0.getText(), DateTimeUtility.createFormatter("yyyy-MM-dd HH:mm:ss"));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;

import com.exactpro.sf.configuration.suri.SailfishURI;
import com.exactpro.sf.embedded.statistics.entities.SfInstance;
import com.exactpro.sf.embedded.statistics.handlers.IStatisticsReportHandler;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -617,7 +621,7 @@ public Response migrate() {
@GET
@Path("stats_per_tags")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.TEXT_PLAIN)
@Produces({MediaType.TEXT_PLAIN, MediaType.APPLICATION_XML})
public Response getStatsPerTagsFromJson(DimensionMap dimensionMap) {
XmlResponse xmlResponse = new XmlResponse();
Status status;
Expand All @@ -636,6 +640,38 @@ public Response getStatsPerTagsFromJson(DimensionMap dimensionMap) {
build();
}

@GET
@Path("aggregated_report")
@Consumes(MediaType.APPLICATION_JSON)
@Produces({MediaType.TEXT_PLAIN, MediaType.APPLICATION_XML})
public Response getAggregatedReport(AggregateReportParameters parameters) {
XmlResponse xmlResponse = new XmlResponse();
Status status;
try {
StatisticsService statisticsService = getStatisticsService();
String reportType = parameters.getReportType();
if (reportType == null) {
throw new IllegalArgumentException("ReportType is not set");
}
IStatisticsReportHandler statisticsReportHandler =
statisticsService.getStatisticsReportHandler(SailfishURI.parse(reportType));
IStatisticsStorage statisticsStorage = statisticsService.getStorage();
StatisticsUtils.loadSfInstanceIdsFromDb(statisticsStorage, parameters);
StatisticsUtils.loadTagIdsFromDb(statisticsStorage, parameters);
StatisticsUtils.generateAggregatedReport(statisticsService, parameters, statisticsReportHandler);
return generateAggregatedReport(statisticsReportHandler, parameters);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
status = Status.INTERNAL_SERVER_ERROR;
xmlResponse.setMessage(e.getMessage());
xmlResponse.setRootCause((e.getCause() != null) ? e.getCause().getMessage() : null);
}
return Response.
status(status).
entity(xmlResponse).
build();
}

public ServletContext getContext() {
return context;
}
Expand Down Expand Up @@ -667,6 +703,25 @@ private Response generateStatsPerTags(List<TagGroupDimension> selectedDimensions
+ reportName).build();
}

private Response generateAggregatedReport(IStatisticsReportHandler statisticsReportHandler,
AggregateReportParameters parameters) {

String reportName = statisticsReportHandler.getReportName(parameters);
StreamingOutput stream = out -> {
try {
statisticsReportHandler.writeReport(out);
} catch(Exception e) {
logger.error(e.getMessage(), e);
throw new WebApplicationException(e);
}
};
return Response
.ok(stream)
.header("content-disposition",
"attachment; filename = "
+ reportName).build();
}

private List<TagGroupDimension> parseDimensions(DimensionMap dimensions) {
IStatisticsStorage storage = getStatisticsService().getStorage();
List<TagGroupDimension> parsedDimensions = new ArrayList<>();
Expand Down Expand Up @@ -697,4 +752,6 @@ private List<TagGroupDimension> parseDimensions(DimensionMap dimensions) {
}
return parsedDimensions;
}


}
Loading

0 comments on commit b015090

Please sign in to comment.