diff --git a/README.md b/README.md index a61184c..020caac 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,31 @@ patterns: 输入：`Error: file not found` 输出：file not found +### include + +支持引入其它规则文件，例如： + +```yaml +name: Rule +include: base #引入同级目录下的 base.yaml 或 base.yml +``` + +`include`支持引入一个或多个文件，例如: + +```yaml +name: Rule +include: + - base + - ../base + - base.yaml + - base/base1 + - base/base2.yaml + - ../base.yaml + - /usr/etc/rules/base.yml +``` + +` context`、`patterns`会按照引用顺序依次合并，如果有同名的context，后面的会替换之前的。 + ## License MIT \ No newline at end of file diff --git a/flog/flog.py b/flog/flog.py index 41f6d89..629ada5 100644 --- a/flog/flog.py +++ b/flog/flog.py @@ -5,6 +5,7 @@ from .rule import * from .engine import * +from .loader import * def cmd(): parser = argparse.ArgumentParser(description='Process logs.') @@ -16,7 +17,7 @@ def cmd(): def main(): args = cmd() - rules = Rule.load(args.rule) + rules = Loader(args.rule).load() engine = Engine(rules) output = None diff --git a/flog/loader.py b/flog/loader.py new file mode 100644 index 0000000..dcb952c --- /dev/null +++ b/flog/loader.py @@ -0,0 +1,74 @@ +import re +import yaml + +from typing import List +from pathlib import Path +from .rule import Rule + +class Loader: + suffix = ['.yaml', '.yml'] + + def __init__(self, base_path: str): + self.path = Path(base_path) + self.included = [] + self.gloabl_context = {} + self.rules = [] + + def __load_yaml(self, path: Path) -> dict: + with path.open() as f: + return yaml.load(f, Loader=yaml.FullLoader) + + def __load_include(self, name: str) -> dict: + path = Path(name).expanduser() + if path.is_absolute(): + # include /User/abc.yaml + self.__load_file(path) + return + + path = self.path.parent.joinpath(name) + if path.is_file(): + # include abc.yaml + self.__load_file(path) + return + + # include abc + for suffix in self.suffix: + new_path = path.with_suffix(suffix) + print(new_path) + if new_path.is_file(): + self.__load_file(new_path) + return + + raise Exception(f'include file not found: {name}') + + def __load_file(self, path: Path): + if path in self.included: + return + self.included.append(path) + data = self.__load_yaml(path) + + # check include + if include := data.get('include', None): + if isinstance(include, str): + self.__load_include(include) + if isinstance(include, list): + for item in include: + self.__load_include(item) + + # global context + if context := data.get('context', None): + for key, value in context.items(): + if key in ["lines", "captures", "content"]: + raise AttributeError(f'Invalid context key: {key}') + self.gloabl_context[key] = re.compile(value) + + if patterns := data.get('patterns', None): + for item in data['patterns']: + self.rules.append(Rule(item)) + + def load(self) -> List[Rule]: + self.__load_file(self.path) + + for rule in self.rules: + rule.global_context = self.gloabl_context + return self.rules diff --git a/flog/rule.py b/flog/rule.py index a3831c8..bb032f0 100644 --- a/flog/rule.py +++ b/flog/rule.py @@ -156,22 +156,4 @@ def __render_env(self, context, result): else: env[key] = '' - return env - - @classmethod - def load(cls, path): - rules = [] - with open(path) as f: - data = yaml.load(f, Loader=yaml.FullLoader) - - global_context = None - if context := data.get('context', None): - global_context = {} - for key, value in context.items(): - if key in ["lines", "captures", "content"]: - raise AttributeError(f'Invalid context key: {key}') - global_context[key] = re.compile(value) - - for item in data['patterns']: - rules.append(Rule(item, global_context)) - return rules + return env \ No newline at end of file diff --git a/setup.py b/setup.py index 033f55c..b990a05 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="flog", - version="0.0.4", + version="0.0.5", author="zqqf16", author_email="zqqf16@gmail.com", description="Yet another log parser", diff --git a/tests/test_loader.py b/tests/test_loader.py new file mode 100644 index 0000000..20f5c61 --- /dev/null +++ b/tests/test_loader.py @@ -0,0 +1,156 @@ +import pytest + +from flog.loader import * + +def str_to_rules(tmpdir, content): + rule = tmpdir.join('test.yaml') + rule.write(content) + loader = Loader(str(rule)) + return loader.load() + +def test_path(tmpdir): + base = tmpdir.join('base.yaml') + base.write(''' +patterns: + - name: Python version + match: Python (3.\\d+\\.\\d+\\.\\d+) + message: "Python version: {{0}}"''') + + rule = str_to_rules(tmpdir, 'include: base')[0] + assert rule.name == 'Python version' + + rule = str_to_rules(tmpdir, 'include: base.yaml')[0] + assert rule.name == 'Python version' + + rule = str_to_rules(tmpdir, 'include: base.yml')[0] + assert rule.name == 'Python version' + + rule = str_to_rules(tmpdir, 'include: '+str(base))[0] + assert rule.name == 'Python version' + + with pytest.raises(Exception) as exc_info: + str_to_rules(tmpdir, 'include: notfound') + assert str(exc_info.value) == 'include file not found: notfound' + + sub = tmpdir.mkdir('sub') + rule = str_to_rules(sub, 'include: ../base')[0] + assert rule.name == 'Python version' + + with pytest.raises(Exception) as exc_info: + str_to_rules(sub, 'include: base') + assert str(exc_info.value) == 'include file not found: base' + +def test_context(tmpdir): + base = tmpdir.join('base.yaml') + base.write(''' +context: + name: "version" +patterns: + - name: Python version + match: Python (3.\\d+\\.\\d+\\.\\d+) + message: "Python version: {{0}}"''') + + rule = str_to_rules(tmpdir, 'include: base')[0] + assert rule.global_context['name'].pattern == 'version' + + rule = str_to_rules(tmpdir, '''\ +include: base +context: + name: "new" +''')[0] + assert rule.global_context['name'].pattern == 'new' + +def test_recursive(tmpdir): + base = tmpdir.join('base.yaml') + base.write(''' +context: + name: "version" +patterns: + - name: Python version + match: Python (3.\\d+\\.\\d+\\.\\d+) + message: "Python version: {{0}}"''') + + base2 = tmpdir.join('base2.yaml') + base2.write(''' +include: base +context: + name: "version" +patterns: + - name: Build version + match: Build (\\d+) + message: "Buildd version: {{0}}"''') + + rules = str_to_rules(tmpdir, 'include: base2') + assert rules[0].name == 'Python version' + assert rules[1].name == 'Build version' + +def test_cycle(tmpdir): + base = tmpdir.join('base.yaml') + base.write(''' +include: base2 +context: + name: "version" +patterns: + - name: Python version + match: Python (3.\\d+\\.\\d+\\.\\d+) + message: "Python version: {{0}}"''') + + base2 = tmpdir.join('base2.yaml') + base2.write(''' +include: base +context: + name: "version" +patterns: + - name: Build version + match: Build (\\d+) + message: "Buildd version: {{0}}"''') + + rules = str_to_rules(tmpdir, 'include: base2') + assert len(rules) == 2 + +def test_context(tmpdir): + base = tmpdir.join('base.yaml') + base.write(''' +context: + name: "version" +patterns: + - name: Python version + match: Python (3.\\d+\\.\\d+\\.\\d+) + message: "Python version: {{0}}"''') + + rule = str_to_rules(tmpdir, 'include: base')[0] + assert rule.global_context['name'].pattern == 'version' + + rule = str_to_rules(tmpdir, '''\ +include: base +context: + name: "new" +''')[0] + assert rule.global_context['name'].pattern == 'new' + +def test_include_list(tmpdir): + base = tmpdir.join('base.yaml') + base.write(''' +context: + name: "version" +patterns: + - name: Python version + match: Python (3.\\d+\\.\\d+\\.\\d+) + message: "Python version: {{0}}"''') + + base2 = tmpdir.join('base2.yaml') + base2.write(''' +context: + name: "version" +patterns: + - name: Build version + match: Build (\\d+) + message: "Buildd version: {{0}}"''') + + rules = str_to_rules(tmpdir, ''' +include: + - base + - base2 + ''') + assert rules[0].name == 'Python version' + assert rules[1].name == 'Build version' \ No newline at end of file diff --git a/tests/test_rule.py b/tests/test_rule.py index c76d508..b73de63 100644 --- a/tests/test_rule.py +++ b/tests/test_rule.py @@ -106,27 +106,4 @@ def test_rule_message(): ctx = MatchingContext(result, content) msg = rule.message(ctx, result) - assert msg == content+' 1 1'+' 2022-04-08 16:52:37.152' - -def test_rule_loader(tmpdir): - rule = tmpdir.join('rule.yaml') - rule.write(r''' -name: test -context: - timestamp: "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}" -patterns: - - match: "hello ([^:]*):" - message: "{{ timestamp }}: {{ captures[0] }}" -''') - - rules = Rule.load(str(rule)) - rule = rules[0] - - assert rule.global_context['timestamp'].pattern == '\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}' - - content = '2022-04-08 16:52:37.152 hello world: this is a test message' - result = rule.match(content) - ctx = MatchingContext(result, content) - - msg = rule.message(ctx, result) - assert msg == '2022-04-08 16:52:37.152: world' \ No newline at end of file + assert msg == content+' 1 1'+' 2022-04-08 16:52:37.152' \ No newline at end of file