!4 把bug-fix和文档配置整合到主分支

Merge pull request !4 from WillQvQ/send_to_bot
fastnlp · Oct 30, 2020 · b747b15 · b747b15
2 parents 465b356 + 5c8c022
commit b747b15
Show file tree

Hide file tree

Showing 31 changed files with 100 additions and 45 deletions.
diff --git a/.Jenkinsfile b/.Jenkinsfile
@@ -0,0 +1,42 @@
+pipeline {
+    agent {
+        docker {
+            image 'ubuntu_tester'
+            args '-u root:root -v ${HOME}/html/docs:/docs -v ${HOME}/html/_ci:/ci'
+        }
+    }
+    environment {
+        PJ_NAME = 'fitlog'
+        POST_URL = 'https://open.feishu.cn/open-apis/bot/v2/hook/ab4aa33b-1c7a-403c-8a83-fdc27fc9d8d1'
+    }
+    stages {
+        stage('Package Installation') {
+            steps {
+                sh 'python setup.py install'
+            }
+        }
+        stage('Parallel Stages') {
+            parallel {
+                stage('Document Building') {
+                    steps {
+                        sh 'cd docs && make prod'
+                        sh 'rm -rf /docs/${PJ_NAME}'
+                        sh 'mv docs/build/html /docs/${PJ_NAME}'
+                    }
+                }
+                stage('Package Testing') {
+                    steps {
+                        sh 'pytest ./test --html=test_results.html --self-contained-html'
+                    }
+                }
+            }
+        }
+    }
+    post {
+        always {
+            sh 'post'
+        }
+
+    }
+
+}
diff --git a/README.md b/README.md
@@ -52,29 +52,22 @@ filog 是我们实验室内部使用的一款工具，大部分功能口口相
 fitlog.finish(status=1,
 	send_to_bot="https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxx")      
 ```
-
-#### 添加飞书机器人的方法
-
-如下图所示，打开群聊后点击 "设置-群机器人-添加机器人-Custom Bot" 可以打开机器人窗口，点击添加后可以获得飞书机器人的 webhook 地址。
-
-![add_bot_1](doc/source/figures/add_bot_1.png)  
-
-另外，可以参考[一个人也可以用的群聊机器人](https://getfeishu.cn/hc/zh-cn/articles/360024984973-%E5%9C%A8%E7%BE%A4%E8%81%8A%E4%B8%AD%E4%BD%BF%E7%94%A8%E6%9C%BA%E5%99%A8%E4%BA%BA)
+添加飞书机器人的方法参考[群聊机器人](https://getfeishu.cn/hc/zh-cn/articles/360024984973-%E5%9C%A8%E7%BE%A4%E8%81%8A%E4%B8%AD%E4%BD%BF%E7%94%A8%E6%9C%BA%E5%99%A8%E4%BA%BA)
 
 ## New Features(2020.06.11)
 以下的功能都没有经过实战使用检验，可能有bug。
 #### 1. 支持一些复杂筛选条件
  现在支持在Table那个页面的search框中输入复杂搜索逻辑进行筛选，例如下面log
 
-![before_search](doc/source/figures/before_search.jpg)  
+![before_search](docs/source/figures/before_search.jpg)  
 
 想要搜索**2020年06月11号15点到2020年06月11号16点之间，并且hidden_size大于60**的log, 可以通过下面的语法输入到search框中  
 ```
 ${'id':'log_20200611_150000<=&&<=log_20200611_160000', 'hyper-hidden_size':'>60'}$
 ```
 使用效果如下（只显示满足条件的log了）
 
-![after_search](./doc/source/figures/after_search.jpg)  
+![after_search](docs/source/figures/after_search.jpg)  
 
 
 其中开头结尾的$是特殊标记符号，申明这是特殊搜索，否则fitlog会只进行常规字符串匹配搜索。由于search框一旦有文字就会触发搜索，所以建议在其他地方把文本编辑好，直接复制到search框中。支持的search语法规则如下
@@ -88,10 +81,10 @@ ${'id':'log_20200611_150000<=&&<=log_20200611_160000', 'hyper-hidden_size':'>60'
 #### 2.支持多条log的metric收敛曲线对比
 操作如下图所示，先选择需要对比的log(默认只能选择10条)，然后点击右上角红色框处 
 
-![compare_metric](./doc/source/figures/compare_metric.jpg)
+![compare_metric](docs/source/figures/compare_metric.jpg)
 
 然后会弹出选择需要对比的metric，选好需要对比的metric后点击确认跳转，跳转后界面为
 
-![compare_metric_trend](./doc/source/figures/compare_metric_trend.jpg)
+![compare_metric_trend](docs/source/figures/compare_metric_trend.jpg)
 
 我十分垃圾的js代码能力坚定地告诉我：你们一定不可能需要对比loss曲线的，所以我就没有必要再做loss对比了🐶。
diff --git a/doc/source/inner_api.rst b/doc/source/inner_api.rst
diff --git a/doc/Makefile → docs/Makefile b/doc/Makefile → docs/Makefile
@@ -20,7 +20,10 @@ server:
 	cd build/html && python -m http.server
 
 dev:
-	rm -rf build && make html && make server
+	rm -f source/$(SPHINXPROJ).* source/modules.rst && rm -rf build && make apidoc && make html && make server
+
+prod:
+	make apidoc && make html
 
 .PHONY: help Makefile
 

diff --git a/doc/source/conf.py → docs/source/conf.py b/doc/source/conf.py → docs/source/conf.py
diff --git a/doc/source/figures/add_bot_1.png → docs/source/figures/add_bot_1.png b/doc/source/figures/add_bot_1.png → docs/source/figures/add_bot_1.png
diff --git a/doc/source/figures/after_search.jpg → docs/source/figures/after_search.jpg b/doc/source/figures/after_search.jpg → docs/source/figures/after_search.jpg
diff --git a/doc/source/figures/before_search.jpg → docs/source/figures/before_search.jpg b/doc/source/figures/before_search.jpg → docs/source/figures/before_search.jpg
diff --git a/doc/source/figures/compare_metric.jpg → docs/source/figures/compare_metric.jpg b/doc/source/figures/compare_metric.jpg → docs/source/figures/compare_metric.jpg
diff --git a/doc/source/figures/compare_metric_trend.jpg → docs/source/figures/compare_metric_trend.jpg b/doc/source/figures/compare_metric_trend.jpg → docs/source/figures/compare_metric_trend.jpg
diff --git a/doc/source/figures/fitlog_chart.png → docs/source/figures/fitlog_chart.png b/doc/source/figures/fitlog_chart.png → docs/source/figures/fitlog_chart.png
diff --git a/doc/source/figures/fitlog_table.png → docs/source/figures/fitlog_table.png b/doc/source/figures/fitlog_table.png → docs/source/figures/fitlog_table.png
diff --git a/doc/source/fitlog.fastgit.rst → docs/source/fitlog.fastgit.rst b/doc/source/fitlog.fastgit.rst → docs/source/fitlog.fastgit.rst
diff --git a/doc/source/fitlog.fastlog.rst → docs/source/fitlog.fastlog.rst b/doc/source/fitlog.fastlog.rst → docs/source/fitlog.fastlog.rst
diff --git a/doc/source/fitlog.rst → docs/source/fitlog.rst b/doc/source/fitlog.rst → docs/source/fitlog.rst
diff --git a/doc/source/index.rst → docs/source/index.rst b/doc/source/index.rst → docs/source/index.rst
@@ -24,13 +24,10 @@ fitlog是一款集成了自动版本管理和自动日志记录两种功能的 P
 API 文档
 -------------
 
-一般情况下你只需要阅读用户 API 文档即可，里面包括了你可以调用的函数接口。内部 API 是为开发者准备的。
-
 .. toctree::
     :maxdepth: 1
 
-    用户 API <fitlog>
-    内部 API <inner_api>
+    fitlog
 
 
 

diff --git a/doc/source/modules.rst → docs/source/modules.rst b/doc/source/modules.rst → docs/source/modules.rst
diff --git a/doc/source/user/command_line.rst → docs/source/user/command_line.rst b/doc/source/user/command_line.rst → docs/source/user/command_line.rst
diff --git a/doc/source/user/configs.rst → docs/source/user/configs.rst b/doc/source/user/configs.rst → docs/source/user/configs.rst
diff --git a/doc/source/user/guide.rst → docs/source/user/guide.rst b/doc/source/user/guide.rst → docs/source/user/guide.rst
diff --git a/doc/source/user/installation.rst → docs/source/user/installation.rst b/doc/source/user/installation.rst → docs/source/user/installation.rst
diff --git a/doc/source/user/quickstart.rst → docs/source/user/quickstart.rst b/doc/source/user/quickstart.rst → docs/source/user/quickstart.rst
diff --git a/doc/source/user/tips.rst → docs/source/user/tips.rst b/doc/source/user/tips.rst → docs/source/user/tips.rst
diff --git a/doc/source/user/website.rst → docs/source/user/website.rst b/doc/source/user/website.rst → docs/source/user/website.rst
diff --git a/fitlog/fastlog/logger.py b/fitlog/fastlog/logger.py
@@ -1,7 +1,6 @@
 import logging
 import os
 import requests
-import json
 from datetime import datetime
 from copy import deepcopy
 import argparse
@@ -236,12 +235,22 @@ def _create_log_files(self):
         if not hasattr(self, 'meta_logger'):
             if self._save_log_dir is None:
                 now = datetime.now().strftime('%Y%m%d_%H%M%S')
-                self._save_log_dir = os.path.join(self._log_dir, 'log_' + now)
-                while os.path.exists(self._save_log_dir):
+                _save_log_dir = os.path.join(self._log_dir, 'log_' + now)
+                while os.path.exists(_save_log_dir):
                     time.sleep(1)
                     now = datetime.now().strftime('%Y%m%d_%H%M%S')
-                    self._save_log_dir = os.path.join(self._log_dir, 'log_' + now)
-                os.mkdir(self._save_log_dir)
+                    _save_log_dir = os.path.join(self._log_dir, 'log_' + now)
+                while True:
+                    try:
+                        os.mkdir(_save_log_dir)
+                        break
+                    except FileExistsError:
+                        time.sleep(1)
+                        now = datetime.now().strftime('%Y%m%d_%H%M%S')
+                        _save_log_dir = os.path.join(self._log_dir, 'log_' + now)
+                    except Exception as e:
+                        raise e
+                self._save_log_dir = _save_log_dir
             # prepare logger
             formatter = logging.Formatter('%(message)s')  # 只保存记录的时间与记录的内容
             for name in ['meta', 'hyper', 'metric', 'other', 'loss', 'progress', 'best_metric', 'file']:

diff --git a/fitlog/fastserver/static/js/bootstrap-table-export.js b/fitlog/fastserver/static/js/bootstrap-table-export.js
@@ -24,7 +24,7 @@
     $.extend($.fn.bootstrapTable.defaults, {
         showExport: false,
         exportDataType: 'basic', // basic, all, selected
-        // 'json', 'xml', 'png', 'csv', 'txt', 'sql', 'doc', 'excel', 'powerpoint', 'pdf'
+        // 'json', 'xml', 'png', 'csv', 'txt', 'sql', 'docs', 'excel', 'powerpoint', 'pdf'
         exportTypes: ['json', 'xml', 'csv', 'txt', 'sql', 'excel'],
         exportOptions: {}
     });

diff --git a/fitlog/fastserver/static/js/jquery.min.js b/fitlog/fastserver/static/js/jquery.min.js
@@ -1115,7 +1115,7 @@ setDocument = Sizzle.setDocument = function( node ) {
 	var hasCompare, subWindow,
 		doc = node ? node.ownerDocument || node : preferredDoc;
 
-	// Return early if doc is invalid or already selected
+	// Return early if docs is invalid or already selected
 	if ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) {
 		return document;
 	}

diff --git a/readthedocs.yml b/readthedocs.yml
@@ -1,7 +1,7 @@
 version: 2
 
 sphinx:
-  configuration: doc/source/conf.py
+  configuration: docs/source/conf.py
 
 build:
   image: latest

diff --git a/test/fastgit/test_fastgit.py b/test/fastgit/test_fastgit.py
@@ -0,0 +1,20 @@
+import unittest
+import os
+import shutil
+from fitlog.fastgit import committer
+
+
+class TextExample(unittest.TestCase):
+    def setUp(self):  # 定义一个setup，放一些准备的工作，或者准备一些测试数据。
+        os.mkdir('testArea')
+
+    def test_init(self):
+        ret = committer.init_project('testArea/test_pj', git=False)
+        self.assertEqual(ret, 0)
+
+    def tearDown(self):  # 定义一个tearDown，在测试完的时候我要对测试有一个销毁的过程
+        shutil.rmtree('testArea', ignore_errors=True)
+
+    # 基本参考 https://docs.python.org/zh-cn/3/library/unittest.html
+    # mock 提供了更多的可能性 https://docs.python.org/zh-cn/3/library/unittest.mock.html
+
diff --git a/test/fastserver/test_app.py b/test/fastserver/test_app.py
@@ -1,12 +1,12 @@
-
 from fitlog.fastserver.app import start_app
 
-def run_app():
 
-    log_dir ='../../model_test/codes/V1/logs'
+def run_app():
+    log_dir = '../../model_test/codes/V1/logs'
     log_config_name = 'default.cfg'
     port = 5000
     start_app(log_dir, log_config_name, port, 1, '123')
 
+
 if __name__ == '__main__':
-    run_app()
+    run_app()
diff --git a/test/fastserver/test_summary_utils.py b/test/fastserver/test_summary_utils.py
@@ -1,8 +1,8 @@
-
 from itertools import groupby
 from operator import itemgetter
 import numpy as np
 
+
 def groupBy(data, key):
     """
 
@@ -11,9 +11,10 @@ def groupBy(data, key):
     :return: 可以看成[[key1, group1], [key2, group2]]
     """
     data.sort(key=itemgetter(key))
-    grouped_data = groupby(data, itemgetter(key)) # key + 迭代器。可以看成[[key1, group1], [key2, group2]]
+    grouped_data = groupby(data, itemgetter(key))  # key + 迭代器。可以看成[[key1, group1], [key2, group2]]
     return grouped_data
 
+
 def get_grouped_data(data, keys):
     """
 
@@ -22,7 +23,7 @@ def get_grouped_data(data, keys):
     :return: {}nested的dict，最后的value是符合这个group的结果
     """
     _dict = {}
-    if len(keys)==1:
+    if len(keys) == 1:
         grouped_data = groupBy(data, keys[0])
         for key, group in grouped_data:
             _dict[key] = list(group)
@@ -33,6 +34,7 @@ def get_grouped_data(data, keys):
             _dict[key] = get_grouped_data(list(group), keys[1:])
     return _dict
 
+
 def merge(a, b, path=None):
     "merges b into a"
     if path is None: path = []
@@ -41,7 +43,7 @@ def merge(a, b, path=None):
             if isinstance(a[key], dict) and isinstance(b[key], dict):
                 merge(a[key], b[key], path + [str(key)])
             elif a[key] == b[key]:
-                pass # same leaf value
+                pass  # same leaf value
             else:
                 raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
         else:
@@ -81,4 +83,4 @@ def merge(a, b, path=None):
     root_log_dir = '../../model_test/codes/V1/logs'
 
     print(generate_summary_table(vertical, horizontals, method, criteria, results, results, selected_data,
-                     root_log_dir))
+                                 root_log_dir))