diff --git a/pom.xml b/pom.xml index aa7a1d9c..0e0e5d7c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.univocity univocity-parsers - 2.7.3-SNAPSHOT + 2.7.3 univocity-parsers jar univocity's open source parsers for processing different text formats using a consistent API diff --git a/src/main/java/com/univocity/parsers/annotations/Validate.java b/src/main/java/com/univocity/parsers/annotations/Validate.java index 92a4e4cf..9da99d92 100644 --- a/src/main/java/com/univocity/parsers/annotations/Validate.java +++ b/src/main/java/com/univocity/parsers/annotations/Validate.java @@ -74,4 +74,12 @@ * @return the sequence of disallowed values */ String[] noneOf() default {}; + + /** + * User provided implementations of {@link Validator} which will be executed + * in sequence after the validations specified in this annotation execute. + * + * @return custom classes to be used to validate any value associated with this field. + */ + Class[] validators() default {}; } diff --git a/src/main/java/com/univocity/parsers/annotations/helpers/AnnotationHelper.java b/src/main/java/com/univocity/parsers/annotations/helpers/AnnotationHelper.java index d94f879c..d9c20882 100644 --- a/src/main/java/com/univocity/parsers/annotations/helpers/AnnotationHelper.java +++ b/src/main/java/com/univocity/parsers/annotations/helpers/AnnotationHelper.java @@ -125,7 +125,8 @@ private static Conversion getConversion(Class fieldType, AnnotatedElement target String[] noneOf = AnnotationRegistry.getValue(target, validate, "noneOf", validate.noneOf()); String matches = AnnotationRegistry.getValue(target, validate, "matches", validate.matches()); - return Conversions.validate(nullable, allowBlanks, oneOf, noneOf, matches); + Class[] validators = AnnotationRegistry.getValue(target, validate, "validators", validate.validators()); + return new ValidatedConversion(nullable, allowBlanks, oneOf, noneOf, matches, validators); } else if (annType == EnumOptions.class) { if (!fieldType.isEnum()) { if (target == null) { diff --git a/src/main/java/com/univocity/parsers/common/DefaultConversionProcessor.java b/src/main/java/com/univocity/parsers/common/DefaultConversionProcessor.java index 5f394bd4..b3cc6380 100644 --- a/src/main/java/com/univocity/parsers/common/DefaultConversionProcessor.java +++ b/src/main/java/com/univocity/parsers/common/DefaultConversionProcessor.java @@ -130,13 +130,28 @@ public final Object[] applyConversions(String[] row, Context context) { keepRow = applyConversionsByType(false, objectRow, convertedFlags); } - if (keepRow) { + if (keepRow && validateAllValues(objectRow)) { return objectRow; } return null; } + private boolean validateAllValues(Object[] objectRow) { + if (conversions != null && conversions.validatedIndexes != null) { + boolean keepRow = true; + for (int i = 0; keepRow && i < conversions.validatedIndexes.length && i < objectRow.length; i++) { + try { + conversions.executeValidations(i, objectRow[i]); + } catch (Throwable ex) { + keepRow = handleConversionError(ex, objectRow, i); + } + } + return keepRow; + } + return true; + } + /** * Executes the sequences of reverse conversions defined using {@link DefaultConversionProcessor#convertFields(Conversion...)}, {@link DefaultConversionProcessor#convertIndexes(Conversion...)} and {@link DefaultConversionProcessor#convertAll(Conversion...)}, for every field in the given row. * @@ -180,7 +195,7 @@ public final boolean reverseConversions(boolean executeInReverseOrder, Object[] keepRow = applyConversionsByType(true, row, convertedFlags); } - return keepRow; + return keepRow && validateAllValues(row); } private boolean applyConversionsByType(boolean reverse, Object[] row, boolean[] convertedFlags) { diff --git a/src/main/java/com/univocity/parsers/common/fields/FieldConversionMapping.java b/src/main/java/com/univocity/parsers/common/fields/FieldConversionMapping.java index a34b8f36..907b9ed6 100644 --- a/src/main/java/com/univocity/parsers/common/fields/FieldConversionMapping.java +++ b/src/main/java/com/univocity/parsers/common/fields/FieldConversionMapping.java @@ -31,6 +31,8 @@ public class FieldConversionMapping { @SuppressWarnings("rawtypes") private static final Conversion[] EMPTY_CONVERSION_ARRAY = new Conversion[0]; + public int[] validatedIndexes; + /** * This list contains the sequence of conversions applied to sets of fields over multiple calls. *

It is shared by {@link FieldConversionMapping#fieldNameConversionMapping}, {@link FieldConversionMapping#fieldIndexConversionMapping} and {@link FieldConversionMapping#convertAllMapping}. @@ -72,6 +74,8 @@ protected FieldSelector newFieldSelector() { */ private Map>> conversionsByIndex = Collections.emptyMap(); + private Map> validationsByIndex = Collections.emptyMap(); + /** * Prepares the conversions registered in this object to be executed against a given sequence of fields * @@ -99,6 +103,36 @@ public void prepareExecution(boolean writing, String[] values) { fieldEnumConversionMapping.prepareExecution(writing, next, conversionsByIndex, values); convertAllMapping.prepareExecution(writing, next, conversionsByIndex, values); } + + + Iterator>>> entryIterator = conversionsByIndex.entrySet().iterator(); + + while (entryIterator.hasNext()) { + Map.Entry>> e = entryIterator.next(); + Iterator> it = e.getValue().iterator(); + while (it.hasNext()) { + Conversion conversion = it.next(); + if (conversion instanceof ValidatedConversion) { + if (validationsByIndex.isEmpty()) { + validationsByIndex = new TreeMap>(); + } + + it.remove(); + List validations = validationsByIndex.get(e.getKey()); + if (validations == null) { + validations = new ArrayList(1); + validationsByIndex.put(e.getKey(), validations); + } + validations.add((ValidatedConversion) conversion); + } + } + + if (e.getValue().isEmpty()) { + entryIterator.remove(); + } + } + + validatedIndexes = ArgumentUtils.toIntArray(validationsByIndex.keySet()); } /** @@ -144,6 +178,20 @@ public FieldSet applyConversionsOnFieldEnums(Conversion... conv return fieldEnumConversionMapping.registerConversions(conversions); } + /** + * Applies any validations associated with a field at a given index in a record + * @param index The index of parsed value in a record + * @param value The value of the record at the given index + */ + public void executeValidations(int index, Object value) { + List validations = validationsByIndex.get(index); + if (validations != null) { + for (int i = 0; i < validations.size(); i++) { + validations.get(i).execute(value); + } + } + } + /** * Applies a sequence of conversions associated with an Object value at a given index in a record. * diff --git a/src/main/java/com/univocity/parsers/conversions/ValidatedConversion.java b/src/main/java/com/univocity/parsers/conversions/ValidatedConversion.java index 474597c2..c55ecf88 100644 --- a/src/main/java/com/univocity/parsers/conversions/ValidatedConversion.java +++ b/src/main/java/com/univocity/parsers/conversions/ValidatedConversion.java @@ -16,6 +16,7 @@ package com.univocity.parsers.conversions; +import com.univocity.parsers.annotations.helpers.*; import com.univocity.parsers.common.*; import java.util.*; @@ -32,6 +33,7 @@ public class ValidatedConversion implements Conversion { private final Set oneOf; private final Set noneOf; private final Matcher matcher; + private final Validator[] validators; public ValidatedConversion(String regexToMatch) { this(false, false, null, null, regexToMatch); @@ -41,14 +43,29 @@ public ValidatedConversion(boolean nullable, boolean allowBlanks) { this(nullable, allowBlanks, null, null, null); } - public ValidatedConversion(boolean nullable, boolean allowBlanks, String[] oneOf, String[] noneOf, String regexToMatch) { + this(nullable, allowBlanks, oneOf, noneOf, regexToMatch, null); + } + + + public ValidatedConversion(boolean nullable, boolean allowBlanks, String[] oneOf, String[] noneOf, String regexToMatch, Class[] validators) { this.regexToMatch = regexToMatch; this.matcher = regexToMatch == null || regexToMatch.isEmpty() ? null : Pattern.compile(regexToMatch).matcher(""); this.nullable = nullable; this.allowBlanks = allowBlanks; this.oneOf = oneOf == null || oneOf.length == 0 ? null : new HashSet(Arrays.asList(oneOf)); this.noneOf = noneOf == null || noneOf.length == 0 ? null : new HashSet(Arrays.asList(noneOf)); + this.validators = validators == null || validators.length == 0 ? new Validator[0] : instantiateValidators(validators); + } + + private Validator[] instantiateValidators(Class[] validators) { + Validator[] out = new Validator[validators.length]; + + for (int i = 0; i < validators.length; i++) { + out[i] = (Validator) AnnotationHelper.newInstance(Validator.class, validators[i], ArgumentUtils.EMPTY_STRING_ARRAY); + } + + return out; } @Override @@ -117,6 +134,13 @@ private void validate(Object value) { e = new DataValidationException("Value '{value}' is not allowed."); } + for (int i = 0; e == null && i < validators.length; i++) { + String error = validators[i].validate(value); + if (error != null && !error.trim().isEmpty()) { + e = new DataValidationException("Value '{value}' didn't pass validation: " + error); + } + } + if (e != null) { e.setValue(value); throw e; diff --git a/src/main/java/com/univocity/parsers/conversions/Validator.java b/src/main/java/com/univocity/parsers/conversions/Validator.java new file mode 100644 index 00000000..4552f074 --- /dev/null +++ b/src/main/java/com/univocity/parsers/conversions/Validator.java @@ -0,0 +1,38 @@ +/******************************************************************************* + * Copyright 2018 Univocity Software Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package com.univocity.parsers.conversions; + +/** + * Defines a custom validation process to be executed when reading or writing + * values into a field of a java bean that is annotated with {@link com.univocity.parsers.annotations.Validate} + * + * @param the expected type of the value to be validated + */ +public interface Validator { + + /** + * Executes the required validations over a given value, returning + * any validation error messages that are applicable. + * + * If no validation errors are found, returns a blank {@code String} or {@code null} + * + * @param value the value to be validated + * @return a validation error message if the given value fails the validation process. + * If the value is acceptable this method can return either a blank {@code String} or {@code null} + */ + String validate(T value); +} + diff --git a/src/test/java/com/univocity/parsers/issues/github/Github_251.java b/src/test/java/com/univocity/parsers/issues/github/Github_251.java index 3768e4f0..3f63ff0d 100644 --- a/src/test/java/com/univocity/parsers/issues/github/Github_251.java +++ b/src/test/java/com/univocity/parsers/issues/github/Github_251.java @@ -19,6 +19,7 @@ import com.univocity.parsers.annotations.*; import com.univocity.parsers.common.*; import com.univocity.parsers.common.processor.*; +import com.univocity.parsers.conversions.*; import com.univocity.parsers.csv.*; import org.testng.annotations.*; @@ -35,6 +36,16 @@ */ public class Github_251 { + public static class Positive implements Validator { + @Override + public String validate(Integer value) { + if (value < 0) { + return "value must be positive or zero"; + } + return null; + } + } + public static class A { @Parsed(index = 0) @Validate(nullable = true) @@ -53,8 +64,13 @@ public static class A { public String aOrBOrNull; @Parsed(index = 4) - @Validate(allowBlanks = true, matches = "^[^\\d\\s]+$") //yet regex disallows whitespace + @Validate(allowBlanks = true, matches = "^[^\\d\\s]+$") + //yet regex disallows whitespace public String noDigits; + + @Parsed(index = 5) + @Validate(validators = Positive.class) + public int positive; } @Test @@ -69,6 +85,7 @@ public void testValidationAnnotation() { s.setProcessorErrorHandler(new RetryableErrorHandler() { @Override public void handleError(DataProcessingException error, Object[] inputRow, ParsingContext context) { +// System.out.println(error.getMessage()); errorDetails.add(new Object[]{error.getRecordNumber(), error.getColumnIndex(), error.getValue()}); this.keepRecord(); } @@ -77,20 +94,22 @@ public void handleError(DataProcessingException error, Object[] inputRow, Parsin CsvParser p = new CsvParser(s); p.parse(new StringReader("" + - ",a,a,\" \",3z\n" + - "\" \",c,b,,a b")); + ",a,a,\" \",3z,4\n" + + "\" \",c,b,,a b,-5")); - assertEquals(errorDetails.size(), 6); + assertEquals(errorDetails.size(), 7); assertEquals(errorDetails.get(0), new Object[]{1L, 3, " "}); assertEquals(errorDetails.get(1), new Object[]{1L, 4, "3z"}); assertEquals(errorDetails.get(2), new Object[]{2L, 0, " "}); assertEquals(errorDetails.get(3), new Object[]{2L, 1, "c"}); assertEquals(errorDetails.get(4), new Object[]{2L, 2, "b"}); assertEquals(errorDetails.get(5), new Object[]{2L, 4, "a b"}); + assertEquals(errorDetails.get(6), new Object[]{2L, 5, -5}); List beans = processor.getBeans(); assertEquals(beans.size(), 2); assertEquals(beans.get(1).aNotB, null); + assertEquals(beans.get(1).positive, 0); } } \ No newline at end of file