From 5e0c54af174d215f14dbb5fba0cb8c4a82684be1 Mon Sep 17 00:00:00 2001 From: owent <> Date: Sat, 13 Jul 2024 00:35:48 +0800 Subject: [PATCH] Prepare v2.17.0 --- HISTORY.md | 5 + README.md | 2 + pom.xml | 2 +- src/org/xresloader/core/ProgramOptions.java | 9 ++ .../core/data/vfy/DataVerifyImpl.java | 126 ++++++++++++------ .../core/data/vfy/DataVerifyRegex.java | 89 +++++++++++++ 6 files changed, 191 insertions(+), 42 deletions(-) create mode 100644 src/org/xresloader/core/data/vfy/DataVerifyRegex.java diff --git a/HISTORY.md b/HISTORY.md index eeaa860c..bc7ab05a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,11 @@ ## Unrelease +## 2.17.0 + +1. 增加 `--disable-data-validator` 允许跳过数据验证。 +2. 增加正则表达式验证器 `Regex("正则表达式")` 。 + ## 2.16.1 1. 修复字段别名的一处错误 diff --git a/README.md b/README.md index e1c28bce..90403092 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ echo " | --javascript-global | 导出javascript全局空间 | 导出数据到全局时，可以指定写入的名字空间 | | --ignore-unknown-dependency | 忽略未知的依赖项 | 忽略未知的输入协议的依赖项(>=2.9.0版本) | | --validator-rules | 指定自定义验证器配置文件路径 | 指定自定义验证器配置文件路径 | +| --disable-data-validator | 允许忽略数据验证错误 | (>=2.17.0版本) | | --data-source-lru-cache-rows | 数据源的LRU Cache行数 | 仅缓存流式索引 | | --tolerate-max-empty-rows | 连续空行检测的行数 | 设置连续空行检测的行数(>=2.14.1版本) ，大量的连续空行通常是误操作 | @@ -287,6 +288,7 @@ Excel里的Key使用@后缀的字段名，@后面的部分都属于验证器。 + 函数: `InText("文件名"[, 第几个字段[, \"字段分隔正则表达式\"]])` : 从文本文件（UTF-8编码）,可以指定读第几个字段和用于字段分隔的正则表达式 + 函数: `InTableColumn("文件名", "Sheet名", 从第几行开始, 从第几列开始)` : 从Excel数据列读取可用值,指定数据行和数据列 + 函数: `InTableColumn("文件名", "Sheet名", 从第几行开始, KeyRow, KeyValue)` : 从Excel数据列读取可用值,指定数据行并通过某一行的的值获取数据列 ++ 函数: `Regex("正则表达式")` : 验证匹配正则表达式(>=2.17.0版本) + 自定义验证器名（通过 `--validator-rules` 加载） + 协议类型（对应protobuf的message里的每个field，excel里可以填field number或者field name） + 枚举类型（对应protobuf的enum里的每个number，excel里可以填enum number或者enum name） diff --git a/pom.xml b/pom.xml index e0d1ce26..72c3d1d0 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.xresloader xresloader - 2.16.1 + 2.17.0 jar xresloader diff --git a/src/org/xresloader/core/ProgramOptions.java b/src/org/xresloader/core/ProgramOptions.java index 85911bfe..9ac19d87 100644 --- a/src/org/xresloader/core/ProgramOptions.java +++ b/src/org/xresloader/core/ProgramOptions.java @@ -57,6 +57,7 @@ public class RenameRule { public int prettyIndent = 0; public boolean enableStdin = false; public String[] customValidatorRules = null; + public boolean enableDataValidator = true; public String protoDumpFile = ""; public ProtoDumpType protoDumpType = ProtoDumpType.NONE; @@ -231,6 +232,8 @@ private static synchronized Options get_options_group() { options.addOption(null, "lua-module", true, "module(MODULE_NAME, package.seeall) if in lua mode"); options.addOption(Option.builder().longOpt("validator-rules") .desc("set file to load custom validator").hasArg().argName("FILE PATH").build()); + options.addOption(null, "disable-data-validator", false, + "disable data validator, so it will not show warnings when data checking failed."); options.addOption(Option.builder().longOpt("data-source-lru-cache-rows") .desc("set row number for LRU cache").hasArg().argName("NUMBER").build()); options.addOption(Option.builder().longOpt("tolerate-max-empty-rows") @@ -492,6 +495,12 @@ public int init(String[] args) { // custom validator rule file customValidatorRules = cmd.getOptionValues("validator-rules"); + if (cmd.hasOption("disable-data-validator")) { + enableDataValidator = false; + } else { + enableDataValidator = true; + } + return 0; } diff --git a/src/org/xresloader/core/data/vfy/DataVerifyImpl.java b/src/org/xresloader/core/data/vfy/DataVerifyImpl.java index 515f91d0..3513e681 100644 --- a/src/org/xresloader/core/data/vfy/DataVerifyImpl.java +++ b/src/org/xresloader/core/data/vfy/DataVerifyImpl.java @@ -8,6 +8,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.xresloader.core.ProgramOptions; import org.xresloader.core.data.err.ConvException; /** @@ -149,10 +150,10 @@ static public double getAndVerify(List verifyEngine, String path return n; } - try { - DataVerifyResult verify_cache = new DataVerifyResult(); + DataVerifyResult verify_cache = new DataVerifyResult(); - for (DataVerifyImpl vfy : verifyEngine) { + for (DataVerifyImpl vfy : verifyEngine) { + try { if (vfy.get(n, verify_cache)) { if (verify_cache.value == null) { return 0; @@ -165,16 +166,22 @@ static public double getAndVerify(List verifyEngine, String path } return doubleValueOf(verify_cache.value.toString()); } + } catch (Exception e) { + String value; + if (n == (long) n) { + value = String.format("%d", (long) n); + } else { + value = String.format("%g", n); + } + String message = String.format("Check %s for %s with validator %s failed, %s", value, path, + vfy.getDescription(), e.getMessage()); + if (ProgramOptions.getInstance().enableDataValidator) { + throw new ConvException( + message); + } else { + ProgramOptions.getLoger().warn(message); + } } - } catch (Exception e) { - String value; - if (n == (long) n) { - value = String.format("%d", (long) n); - } else { - value = String.format("%g", n); - } - throw new ConvException(String.format("Check %s for %s with %s %s failed, %s", value, path, - getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine), e.getMessage())); } String value; @@ -183,9 +190,16 @@ static public double getAndVerify(List verifyEngine, String path } else { value = String.format("%g", n); } - throw new ConvException( - String.format("Check %s for %s with %s %s failed, check data failed.", value, path, - getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine))); + + String message = String.format("Check %s for %s with %s %s failed, check data failed.", value, path, + getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine)); + if (ProgramOptions.getInstance().enableDataValidator) { + throw new ConvException( + message); + } else { + ProgramOptions.getLoger().warn(message); + return 0.0; + } } static public double getAndVerifyToDouble(List verifyEngine, String path, String val) @@ -222,42 +236,60 @@ static public double getAndVerifyToDouble(List verifyEngine, Str DataVerifyResult verify_cache = new DataVerifyResult(); for (DataVerifyImpl vfy : verifyEngine) { - if (vfy.get(val, verify_cache)) { - if (verify_cache.value == null) { - return 0; - } - if (verify_cache.value instanceof Double) { - return (double) verify_cache.value; + try { + if (vfy.get(val, verify_cache)) { + if (verify_cache.value == null) { + return 0; + } + if (verify_cache.value instanceof Double) { + return (double) verify_cache.value; + } + if (verify_cache.value instanceof Long) { + return ((Long) verify_cache.value).doubleValue(); + } + return doubleValueOf(verify_cache.value.toString()); } - if (verify_cache.value instanceof Long) { - return ((Long) verify_cache.value).doubleValue(); + } catch (Exception e) { + String message = String.format("Check %s for %s with validator %s failed, %s", val, path, + vfy.getDescription(), e.getMessage()); + if (ProgramOptions.getInstance().enableDataValidator) { + throw new ConvException( + message); + } else { + ProgramOptions.getLoger().warn(message); } - return doubleValueOf(verify_cache.value.toString()); } } } catch (Exception e) { - if (verifyEngine == null || verifyEngine.isEmpty()) { - throw new ConvException(String.format("Convert %s for %s failed, %s", val, path, e.getMessage())); + String message = String.format("Convert %s for %s failed, %s", val, path, e.getMessage()); + if (ProgramOptions.getInstance().enableDataValidator) { + throw new ConvException( + message); } else { - throw new ConvException(String.format("Convert %s for %s with %s %s failed, %s", val, path, - getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine), e.getMessage())); + ProgramOptions.getLoger().warn(message); } } - throw new ConvException(String.format("Convert %s for %s with %s %s failed, check data failed.", val, - path, getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine))); + String message = String.format("Convert %s for %s with %s %s failed, check data failed.", val, + path, getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine)); + if (ProgramOptions.getInstance().enableDataValidator) { + throw new ConvException(); + } else { + ProgramOptions.getLoger().warn(message); + return 0.0; + } } static public String getAndVerifyToString(List verifyEngine, String path, String val) throws ConvException { - try { - if (verifyEngine == null || verifyEngine.isEmpty()) { - return val; - } + if (verifyEngine == null || verifyEngine.isEmpty()) { + return val; + } - DataVerifyResult verify_cache = new DataVerifyResult(); + DataVerifyResult verify_cache = new DataVerifyResult(); - for (DataVerifyImpl vfy : verifyEngine) { + for (DataVerifyImpl vfy : verifyEngine) { + try { if (vfy.get(val, verify_cache)) { if (verify_cache.value == null) { return ""; @@ -276,14 +308,26 @@ static public String getAndVerifyToString(List verifyEngine, Str } return verify_cache.value.toString(); } + } catch (Exception e) { + String message = String.format("Check %s for %s with validator %s failed, %s", val, path, + vfy.getDescription(), e.getMessage()); + if (ProgramOptions.getInstance().enableDataValidator) { + throw new ConvException( + message); + } else { + ProgramOptions.getLoger().warn(message); + } } - } catch (Exception e) { - throw new ConvException(String.format("Convert %s for %s with %s %s failed, %s", val, path, - getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine), e.getMessage())); } - throw new ConvException(String.format("Convert %s for %s with %s %s failed, check data failed.", val, - path, getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine))); + String message = String.format("Convert %s for %s with %s %s failed, check data failed.", val, + path, getValidatorWord(verifyEngine), collectValidatorNames(verifyEngine)); + if (ProgramOptions.getInstance().enableDataValidator) { + throw new ConvException(message); + } else { + ProgramOptions.getLoger().warn(message); + return ""; + } } static public long getAndVerifyToLong(List verifyEngine, String path, String val) diff --git a/src/org/xresloader/core/data/vfy/DataVerifyRegex.java b/src/org/xresloader/core/data/vfy/DataVerifyRegex.java new file mode 100644 index 00000000..87834960 --- /dev/null +++ b/src/org/xresloader/core/data/vfy/DataVerifyRegex.java @@ -0,0 +1,89 @@ +package org.xresloader.core.data.vfy; + +import java.util.ArrayList; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import org.xresloader.core.ProgramOptions; + +public class DataVerifyRegex extends DataVerifyImpl { + private boolean valid = false; + private ArrayList rules = new ArrayList<>(); + + static Pattern SPACE_SPLITOR = Pattern.compile("\\s+"); + + public DataVerifyRegex(ValidatorTokens tokens) { + super(tokens); + + this.valid = false; + if (tokens.parameters.size() < 2) { + ProgramOptions.getLoger().error("Invalid in regex validator %s", tokens.name); + return; + } + + this.valid = true; + for (int i = 1; i < tokens.parameters.size(); ++i) { + try { + this.rules.add(Pattern.compile(tokens.parameters.get(i))); + } catch (PatternSyntaxException e) { + ProgramOptions.getLoger().error("Can not parse regex %s for validator %s : %s", + tokens.parameters.get(i), + tokens.name, + e.getMessage()); + this.valid = false; + } + } + } + + public boolean isValid() { + return this.valid; + } + + @Override + public boolean get(double number, DataVerifyResult res) { + // 0 值永久有效,因为空数据项会被填充默认值 + if (0 == number) { + res.success = true; + res.value = number; + return true; + } + + String value; + if (number == (long) number) { + value = String.format("%d", (long) number); + } else { + value = String.format("%g", number); + } + + for (Pattern rule : this.rules) { + if (rule.matcher(value).matches()) { + res.success = true; + res.value = number; + return true; + } + } + res.success = false; + return false; + } + + @Override + public boolean get(String input, DataVerifyResult res) { + // 空值永久有效,因为空数据项会被填充默认值 + if (input.isEmpty()) { + res.success = true; + res.value = ""; + return true; + } + + for (Pattern rule : this.rules) { + if (rule.matcher(input.trim()).matches()) { + res.success = true; + res.value = input; + return true; + } + } + + res.success = false; + return false; + } +}