Support

README.md

@@ -172,6 +172,31 @@ patterns:
 输入：`Error: file not found`   
 输出：<font color="red">file not found</font> 
 
+### include
+
+支持引入其它规则文件，例如：
+
+```yaml
+name: Rule
+include: base #引入同级目录下的 base.yaml 或 base.yml
+```
+
+`include`支持引入一个或多个文件，例如:
+
+```yaml
+name: Rule
+include:
+	- base
+	- ../base
+	- base.yaml
+	- base/base1
+	- base/base2.yaml
+	- ../base.yaml
+	- /usr/etc/rules/base.yml
+```
+
+` context`、`patterns`会按照引用顺序依次合并，如果有同名的context，后面的会替换之前的。
+
 ## License
 
 MIT

flog/flog.py

@@ -5,6 +5,7 @@
 
 from .rule import *
 from .engine import *
+from .loader import *
 
 def cmd():
     parser = argparse.ArgumentParser(description='Process logs.')
@@ -16,7 +17,7 @@ def cmd():
 def main():
     args = cmd()
 
-    rules = Rule.load(args.rule)
+    rules = Loader(args.rule).load()
     engine = Engine(rules)
 
     output = None

flog/loader.py

@@ -0,0 +1,74 @@
+import re
+import yaml
+
+from typing import List
+from pathlib import Path
+from .rule import Rule
+
+class Loader:
+    suffix = ['.yaml', '.yml']
+
+    def __init__(self, base_path: str):
+        self.path = Path(base_path)
+        self.included = []
+        self.gloabl_context = {}
+        self.rules = []
+
+    def __load_yaml(self, path: Path) -> dict:
+        with path.open() as f:
+            return yaml.load(f, Loader=yaml.FullLoader)
+
+    def __load_include(self, name: str) -> dict:
+        path = Path(name).expanduser()
+        if path.is_absolute():
+            # include /User/abc.yaml
+            self.__load_file(path)
+            return
+
+        path = self.path.parent.joinpath(name)
+        if path.is_file():
+            # include abc.yaml
+            self.__load_file(path)
+            return
+
+        # include abc
+        for suffix in self.suffix:
+            new_path = path.with_suffix(suffix)
+            print(new_path)
+            if new_path.is_file():
+                self.__load_file(new_path)
+                return
+
+        raise Exception(f'include file not found: {name}')
+
+    def __load_file(self, path: Path):
+        if path in self.included:
+            return
+        self.included.append(path)
+        data = self.__load_yaml(path)
+
+        # check include
+        if include := data.get('include', None):
+            if isinstance(include, str):
+                self.__load_include(include)
+            if isinstance(include, list):
+                for item in include:
+                    self.__load_include(item)
+
+        # global context
+        if context := data.get('context', None):
+            for key, value in context.items():
+                if key in ["lines", "captures", "content"]:
+                    raise AttributeError(f'Invalid context key: {key}')
+                self.gloabl_context[key] = re.compile(value)
+
+        if patterns := data.get('patterns', None):
+            for item in data['patterns']:
+                self.rules.append(Rule(item))
+
+    def load(self) -> List[Rule]:
+        self.__load_file(self.path)
+
+        for rule in self.rules:
+            rule.global_context = self.gloabl_context
+        return self.rules

flog/rule.py

@@ -156,22 +156,4 @@ def __render_env(self, context, result):
             else:
                 env[key] = ''
 
-        return env
-
-    @classmethod
-    def load(cls, path):
-        rules = []
-        with open(path) as f:
-            data = yaml.load(f, Loader=yaml.FullLoader)
-
-        global_context = None
-        if context := data.get('context', None):
-            global_context = {}
-            for key, value in context.items():
-                if key in ["lines", "captures", "content"]:
-                    raise AttributeError(f'Invalid context key: {key}')
-                global_context[key] = re.compile(value)
-
-        for item in data['patterns']:
-            rules.append(Rule(item, global_context))
-        return rules
+        return env

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="flog",
-    version="0.0.4",
+    version="0.0.5",
     author="zqqf16",
     author_email="zqqf16@gmail.com",
     description="Yet another log parser",

tests/test_loader.py

@@ -0,0 +1,156 @@
+import pytest
+
+from flog.loader import *
+
+def str_to_rules(tmpdir, content):
+    rule = tmpdir.join('test.yaml')
+    rule.write(content)
+    loader = Loader(str(rule))
+    return loader.load()    
+
+def test_path(tmpdir):
+    base = tmpdir.join('base.yaml')
+    base.write('''
+patterns:
+    - name: Python version
+      match: Python (3.\\d+\\.\\d+\\.\\d+)
+      message: "Python version: {{0}}"''')
+
+    rule = str_to_rules(tmpdir, 'include: base')[0]
+    assert rule.name == 'Python version'
+
+    rule = str_to_rules(tmpdir, 'include: base.yaml')[0]
+    assert rule.name == 'Python version'
+
+    rule = str_to_rules(tmpdir, 'include: base.yml')[0]
+    assert rule.name == 'Python version'
+
+    rule = str_to_rules(tmpdir, 'include: '+str(base))[0]
+    assert rule.name == 'Python version'
+
+    with pytest.raises(Exception) as exc_info:   
+        str_to_rules(tmpdir, 'include: notfound')
+    assert str(exc_info.value) == 'include file not found: notfound'
+
+    sub = tmpdir.mkdir('sub')
+    rule = str_to_rules(sub, 'include: ../base')[0]
+    assert rule.name == 'Python version'
+
+    with pytest.raises(Exception) as exc_info:   
+        str_to_rules(sub, 'include: base')
+    assert str(exc_info.value) == 'include file not found: base'
+
+def test_context(tmpdir):
+    base = tmpdir.join('base.yaml')
+    base.write('''
+context:
+    name: "version"
+patterns:
+    - name: Python version
+      match: Python (3.\\d+\\.\\d+\\.\\d+)
+      message: "Python version: {{0}}"''')
+
+    rule = str_to_rules(tmpdir, 'include: base')[0]
+    assert rule.global_context['name'].pattern == 'version'
+
+    rule = str_to_rules(tmpdir, '''\
+include: base
+context:
+    name: "new"
+''')[0]
+    assert rule.global_context['name'].pattern == 'new'
+
+def test_recursive(tmpdir):
+    base = tmpdir.join('base.yaml')
+    base.write('''
+context:
+    name: "version"
+patterns:
+    - name: Python version
+      match: Python (3.\\d+\\.\\d+\\.\\d+)
+      message: "Python version: {{0}}"''')
+
+    base2 = tmpdir.join('base2.yaml')
+    base2.write('''
+include: base
+context:
+    name: "version"
+patterns:
+    - name: Build version
+      match: Build (\\d+)
+      message: "Buildd version: {{0}}"''')
+
+    rules = str_to_rules(tmpdir, 'include: base2')
+    assert rules[0].name == 'Python version'
+    assert rules[1].name == 'Build version'
+
+def test_cycle(tmpdir):
+    base = tmpdir.join('base.yaml')
+    base.write('''
+include: base2
+context:
+    name: "version"
+patterns:
+    - name: Python version
+      match: Python (3.\\d+\\.\\d+\\.\\d+)
+      message: "Python version: {{0}}"''')
+
+    base2 = tmpdir.join('base2.yaml')
+    base2.write('''
+include: base
+context:
+    name: "version"
+patterns:
+    - name: Build version
+      match: Build (\\d+)
+      message: "Buildd version: {{0}}"''')
+
+    rules = str_to_rules(tmpdir, 'include: base2')
+    assert len(rules) == 2
+
+def test_context(tmpdir):
+    base = tmpdir.join('base.yaml')
+    base.write('''
+context:
+    name: "version"
+patterns:
+    - name: Python version
+      match: Python (3.\\d+\\.\\d+\\.\\d+)
+      message: "Python version: {{0}}"''')
+
+    rule = str_to_rules(tmpdir, 'include: base')[0]
+    assert rule.global_context['name'].pattern == 'version'
+
+    rule = str_to_rules(tmpdir, '''\
+include: base
+context:
+    name: "new"
+''')[0]
+    assert rule.global_context['name'].pattern == 'new'
+
+def test_include_list(tmpdir):
+    base = tmpdir.join('base.yaml')
+    base.write('''
+context:
+    name: "version"
+patterns:
+    - name: Python version
+      match: Python (3.\\d+\\.\\d+\\.\\d+)
+      message: "Python version: {{0}}"''')
+
+    base2 = tmpdir.join('base2.yaml')
+    base2.write('''
+context:
+    name: "version"
+patterns:
+    - name: Build version
+      match: Build (\\d+)
+      message: "Buildd version: {{0}}"''')
+
+    rules = str_to_rules(tmpdir, '''
+include:
+    - base
+    - base2
+    ''')
+    assert rules[0].name == 'Python version'
+    assert rules[1].name == 'Build version'

tests/test_rule.py

@@ -106,27 +106,4 @@ def test_rule_message():
     ctx = MatchingContext(result, content)
 
     msg = rule.message(ctx, result)
-    assert msg == content+' 1 1'+' 2022-04-08 16:52:37.152'
-
-def test_rule_loader(tmpdir):
-    rule = tmpdir.join('rule.yaml')
-    rule.write(r'''
-name: test
-context:
-  timestamp: "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}"
-patterns:
-  - match: "hello ([^:]*):"
-    message: "{{ timestamp }}: {{ captures[0] }}"
-''')
-
-    rules = Rule.load(str(rule))
-    rule = rules[0]
-
-    assert rule.global_context['timestamp'].pattern == '\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}'
-
-    content = '2022-04-08 16:52:37.152 hello world: this is a test message'
-    result = rule.match(content)
-    ctx = MatchingContext(result, content)
-
-    msg = rule.message(ctx, result)
-    assert msg == '2022-04-08 16:52:37.152: world'
+    assert msg == content+' 1 1'+' 2022-04-08 16:52:37.152'