Merge pull request #436 from TAMULib/sprint-7-staging

Sprint 7 Staging

Author: Dbreck-TAMU

src/main/java/edu/tamu/scholars/middleware/discovery/argument/BoostArg.java

@@ -29,4 +29,9 @@ public static BoostArg of(String parameter) {
         return new BoostArg(parts[0], parts.length > 1 ? Float.valueOf(parts[1]) : 1.0f);
     }
 
+    @Override
+    public String toString() {
+        return "BoostArg [field=" + field + ", value=" + value + "]";
+    }
+
 }

src/main/java/edu/tamu/scholars/middleware/discovery/argument/FilterArg.java

@@ -54,4 +54,9 @@ public static FilterArg of(String field, Optional<String> value, Optional<String
         return new FilterArg(field, valueParam, opKeyParam, tagParam);
     }
 
+    @Override
+    public String toString() {
+        return "FilterArg [field=" + field + ", value=" + value + ", opKey=" + opKey + ", tag=" + tag + "]";
+    }
+
 }

src/main/java/edu/tamu/scholars/middleware/discovery/argument/QueryArg.java

@@ -81,4 +81,11 @@ public static QueryArg of(
         return new QueryArg(expression, defaultField, minimumShouldMatch, queryField, boostQuery, field);
     }
 
+    @Override
+    public String toString() {
+        return "QueryArg [expression=" + expression + ", defaultField=" + defaultField + ", minimumShouldMatch="
+                + minimumShouldMatch + ", queryField=" + queryField + ", boostQuery=" + boostQuery + ", fields="
+                + fields + "]";
+    }
+
 }

src/main/java/edu/tamu/scholars/middleware/discovery/model/Individual.java

@@ -118,4 +118,9 @@ public static Individual from(Map<String, Object> content) {
         return new Individual(content);
     }
 
+    @Override
+    public String toString() {
+        return "Individual [content=" + content + "]";
+    }
+
 }

src/main/java/edu/tamu/scholars/middleware/discovery/model/repo/IndividualRepo.java

@@ -16,6 +16,8 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -208,25 +210,37 @@ public Flux<Individual> export(QueryArg query, List<FilterArg> filters, List<Boo
             .withSort(sort)
             .withRows(Integer.MAX_VALUE);
 
+        logger.info("{}: Exporting {} {} {} {}", builder.getId(), query, filters, boosts, sort);
+
         return Flux.create(emitter -> {
             try {
                 solrClient.queryAndStreamResponse(collectionName, builder.query(), new StreamingResponseCallback() {
                     private final AtomicLong remaining = new AtomicLong(0);
+                    private final AtomicBoolean docListInfoReceived  = new AtomicBoolean(false);
 
                     @Override
                     public void streamSolrDocument(SolrDocument document) {
-                        emitter.next(Individual.from(document));
-
-                        if (remaining.decrementAndGet() == 0) {
+                        Individual individual = Individual.from(document);
+                        logger.debug("{}: streamSolrDocument: {}", builder.getId(), individual);
+                        emitter.next(individual);
+
+                        long numRemaining = remaining.decrementAndGet();
+                        logger.debug("{}: streamSolrDocument remaining: {}", builder.getId(), numRemaining);
+                        if (numRemaining == 0 && docListInfoReceived.get()) {
+                            logger.info("{}: streamSolrDocument COMPLETE", builder.getId());
                             emitter.complete();
                         }
                     }
 
                     @Override
                     public void streamDocListInfo(long numFound, long start, Float maxScore) {
-                        if (numFound > 0) {
-                            remaining.set(numFound);
-                        } else {
+                        logger.debug("{}: streamDocListInfo {} {} {}", builder.getId(), numFound, start, maxScore);
+
+                        remaining.set(numFound);
+                        docListInfoReceived.set(true);
+
+                        if (numFound == 0) {
+                            logger.info("{}: streamDocListInfo COMPLETE", builder.getId());
                             emitter.complete();
                         }
                     }
@@ -443,14 +457,16 @@ private Page<Individual> findAllQuery(SolrQuery query, Pageable pageable) {
 
     private List<String> getValues(SolrDocument document, List<String> dataFields) {
         return dataFields.stream()
-            .filter(v -> document.containsKey(v))
+            .filter(document::containsKey)
             .flatMap(v -> document.getFieldValues(v).stream())
             .map(v -> (String) v)
             .collect(Collectors.toList());
     }
 
     private class SolrQueryBuilder {
 
+        private final UUID id;
+
         private final SolrQuery query;
 
         private final List<String> filters;
@@ -460,13 +476,18 @@ private SolrQueryBuilder() {
         }
 
         private SolrQueryBuilder(String query) {
+            this.id = UUID.randomUUID();
             this.query = new SolrQuery()
                 .setParam("defType", defType)
                 .setParam("q.op", defaultOperator)
                 .setQuery(query);
             this.filters = new ArrayList<>();
         }
 
+        public String getId() {
+            return id.toString();
+        }
+
         public SolrQueryBuilder withQuery(QueryArg query) {
 
             if (StringUtils.isNotEmpty(query.getDefaultField())) {

src/main/java/edu/tamu/scholars/middleware/export/argument/ExportArg.java

@@ -39,4 +39,9 @@ public static ExportArg of(String parameter) {
         return new ExportArg(parts[0], parts.length > 1 ? parts[1] : parts[0]);
     }
 
+    @Override
+    public String toString() {
+        return "ExportArg [field=" + field + ", label=" + label + "]";
+    }
+
 }

src/main/java/edu/tamu/scholars/middleware/export/controller/IndividualSearchExportController.java

@@ -11,6 +11,8 @@
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.data.domain.Sort;
@@ -40,6 +42,8 @@
 @RestController
 public class IndividualSearchExportController implements RepresentationModelProcessor<RepositorySearchesResource> {
 
+    private static final Logger logger = LoggerFactory.getLogger(IndividualSearchExportController.class);
+
     @Lazy
     @Autowired
     private IndividualRepo repo;
@@ -58,7 +62,9 @@ public ResponseEntity<StreamingResponseBody> export(
         List<BoostArg> boosts,
         List<ExportArg> export
     ) throws UnknownExporterTypeException, InterruptedException, ExecutionException {
+        logger.info("/individual/search/export {} {} {} {} {} {} {}", view, type, query, sort, filters, boosts, export);
         Exporter exporter = exporterRegistry.getExporter(type);
+
         return ResponseEntity.ok()
             .header(CONTENT_DISPOSITION, exporter.contentDisposition(FilenameUtility.normalizeExportFilename(view)))
             .header(CONTENT_TYPE, exporter.contentType())

src/main/java/edu/tamu/scholars/middleware/export/service/CsvExporter.java

@@ -5,6 +5,7 @@
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -21,6 +22,7 @@
 import edu.tamu.scholars.middleware.discovery.exception.InvalidValuePathException;
 import edu.tamu.scholars.middleware.discovery.model.Individual;
 import edu.tamu.scholars.middleware.export.argument.ExportArg;
+import edu.tamu.scholars.middleware.utility.DateFormatUtility;
 
 /**
  * 
@@ -67,7 +69,14 @@ public StreamingResponseBody streamIndividuals(Flux<Individual> individuals, Lis
                 .map(e -> e.getField())
                 .collect(Collectors.toList());
             try (CSVPrinter printer = new CSVPrinter(outputStreamWriter, format)) {
-                individuals.subscribe(
+                individuals.doOnComplete(() -> {
+                    try {
+                        printer.flush();
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                        throw new RuntimeException("Failed to flush CSV printer", e);
+                    }
+                }).subscribe(
                     individual -> {
                         try {
                             List<Object> row = getRow(individual, properties);
@@ -78,16 +87,11 @@ public StreamingResponseBody streamIndividuals(Flux<Individual> individuals, Lis
                             | IOException e
                         ) {
                             e.printStackTrace();
+                            throw new RuntimeException("Failed mapping and printing individuals", e);
                         }
-                    }, error -> {
-                        throw new RuntimeException("Failed attempting to stream individuals", error);
-                    },
-                    () -> {
-                        try {
-                            printer.flush();
-                        } catch (IOException e) {
-                            e.printStackTrace();
-                        }
+                    }, e -> {
+                        e.printStackTrace();
+                        throw new RuntimeException("Failed attempting to stream individuals", e);
                     }
                 );
             } catch (IllegalArgumentException e) {
@@ -136,6 +140,8 @@ private List<Object> getRow(
                             .collect(Collectors.toList()));
                     }
 
+                } else if (Date.class.isAssignableFrom(value.getClass())) {
+                    data = DateFormatUtility.format((Date) value);
                 } else {
                     data = (String) value;
                 }

src/main/java/edu/tamu/scholars/middleware/utility/DateFormatUtility.java

@@ -4,6 +4,7 @@
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.Date;
 import java.util.Locale;
 
@@ -15,35 +16,50 @@
  */
 public class DateFormatUtility {
 
-    private static final String[] datePatterns = {
-        "yyyy",
-        "E MMM dd HH:mm:ss z yyyy",
-        "yyyy-MM-dd'T'HH:mm:ss'Z'",
-        "yyyy-MM-dd'T'HH:mm:ss",
-        "dd-MM-yy",
-        "MM-dd-yyyy",
-        "yyyy-MM-dd HH:mm:ss",
-        "EEEEE MMMMM yyyy HH:mm:ss.SSSZ"
+    private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
+
+    private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern(DATE_FORMAT);
+
+    private static final String[] DATE_PATTERNS = {
+            DATE_FORMAT,
+            "yyyy",
+            "E MMM dd HH:mm:ss z yyyy",
+            "yyyy-MM-dd'T'HH:mm:ss'Z'",
+            "yyyy-MM-dd'T'HH:mm:ss",
+            "dd-MM-yy",
+            "MM-dd-yyyy",
+            "EEEEE MMMMM yyyy HH:mm:ss.SSSZ"
     };
 
-    public static String parseYear(String value) throws IllegalArgumentException, ParseException {
+    private DateFormatUtility() {
+
+    }
+
+    public static String parseYear(String value) throws ParseException {
         return String.valueOf(parseZonedDateTime(value).getYear());
     }
 
-    public static ZonedDateTime parseZonedDateTime(String value) throws IllegalArgumentException, ParseException {
+    public static String format(Date date) {
+        return date.toInstant()
+                .atZone(ZoneId.systemDefault())
+                .toLocalDateTime()
+                .format(DATE_FORMATTER);
+    }
+
+    public static ZonedDateTime parseZonedDateTime(String value) throws ParseException {
         return parseDate(value)
-            .toInstant()
-            .atZone(ZoneId.systemDefault());
+                .toInstant()
+                .atZone(ZoneId.systemDefault());
     }
 
-    public static Date parseDate(String value) throws IllegalArgumentException, ParseException {
+    public static Date parseDate(String value) throws ParseException {
         Locale locale = LocaleContextHolder.getLocale();
-        return DateUtils.parseDate(value, locale, datePatterns);
+        return DateUtils.parseDate(value, locale, DATE_PATTERNS);
     }
 
     public static int getYear() {
         return ZonedDateTime.ofInstant(Instant.now(), ZoneId.systemDefault())
-            .getYear();
+                .getYear();
     }
 
 }

src/main/resources/defaults/displayViews.yml

@@ -322,9 +322,22 @@
                 - field: title
                   direction: ASC
               template: "defaults/displayViews/persons/publications/selectedPublications/default.html"
-            - name: Theses
+            - name: Datasets
               field: publications
               order: 6
+              filters:
+                - field: type
+                  value: Dataset
+              sort:
+                - field: publicationDate
+                  direction: DESC
+                  date: true
+                - field: title
+                  direction: ASC
+              template: "defaults/displayViews/persons/publications/selectedPublications/default.html"
+            - name: Theses
+              field: publications
+              order: 7
               filters:
                 - field: type
                   value: Thesis
@@ -337,7 +350,7 @@
               template: "defaults/displayViews/persons/publications/selectedPublications/thesis.html"
             - name: Working Papers
               field: publications
-              order: 7
+              order: 8
               filters:
                 - field: type
                   value: WorkingPaper
@@ -350,7 +363,7 @@
               template: "defaults/displayViews/persons/publications/selectedPublications/default.html"
             - name: Internet Publications
               field: publications
-              order: 8
+              order: 9
               filters:
                 - field: type
                   value: Webpage
@@ -363,7 +376,7 @@
               template: "defaults/displayViews/persons/publications/selectedPublications/internetPublication.html"
             - name: Reports
               field: publications
-              order: 9
+              order: 10
               filters:
                 - field: type
                   value: Report
@@ -376,7 +389,7 @@
               template: "defaults/displayViews/persons/publications/selectedPublications/report.html"
             - name: Reviews
               field: publications
-              order: 10
+              order: 11
               filters:
                 - field: type
                   value: Review
@@ -1439,20 +1452,21 @@
 - name: Documents
   types:
     - AcademicArticle
-    - ConferencePaper
-    - Thesis
-    - Chapter
     - Book
+    - Capstone
+    - Chapter
+    - ConferencePaper
+    - Dataset
     - GreyLiterature
-    - Patent
-    - WorkingPaper
     - NewsRelease
-    - ERO_0000071
-    - Capstone
-    - Webpage
+    - Patent
     - Report
     - Review
     - TeachingMaterial
+    - Thesis
+    - Webpage
+    - WorkingPaper
+    - ERO_0000071
   mainContentTemplate: "defaults/displayViews/documents/mainContentTemplate.html"
   rightScanTemplate: "defaults/displayViews/documents/rightScanTemplate.html"
   metaTemplates: