diff --git a/.Jenkinsfile b/.Jenkinsfile new file mode 100644 index 0000000..26f9f0f --- /dev/null +++ b/.Jenkinsfile @@ -0,0 +1,42 @@ +pipeline { + agent { + docker { + image 'ubuntu_tester' + args '-u root:root -v ${HOME}/html/docs:/docs -v ${HOME}/html/_ci:/ci' + } + } + environment { + PJ_NAME = 'fitlog' + POST_URL = 'https://open.feishu.cn/open-apis/bot/v2/hook/ab4aa33b-1c7a-403c-8a83-fdc27fc9d8d1' + } + stages { + stage('Package Installation') { + steps { + sh 'python setup.py install' + } + } + stage('Parallel Stages') { + parallel { + stage('Document Building') { + steps { + sh 'cd docs && make prod' + sh 'rm -rf /docs/${PJ_NAME}' + sh 'mv docs/build/html /docs/${PJ_NAME}' + } + } + stage('Package Testing') { + steps { + sh 'pytest ./test --html=test_results.html --self-contained-html' + } + } + } + } + } + post { + always { + sh 'post' + } + + } + +} diff --git a/README.md b/README.md index 5d11aaf..a49c1af 100755 --- a/README.md +++ b/README.md @@ -52,21 +52,14 @@ filog 是我们实验室内部使用的一款工具，大部分功能口口相 fitlog.finish(status=1, send_to_bot="https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxx") ``` - -#### 添加飞书机器人的方法 - -如下图所示，打开群聊后点击 "设置-群机器人-添加机器人-Custom Bot" 可以打开机器人窗口，点击添加后可以获得飞书机器人的 webhook 地址。 - -![add_bot_1](doc/source/figures/add_bot_1.png) - -另外，可以参考[一个人也可以用的群聊机器人](https://getfeishu.cn/hc/zh-cn/articles/360024984973-%E5%9C%A8%E7%BE%A4%E8%81%8A%E4%B8%AD%E4%BD%BF%E7%94%A8%E6%9C%BA%E5%99%A8%E4%BA%BA) +添加飞书机器人的方法参考[群聊机器人](https://getfeishu.cn/hc/zh-cn/articles/360024984973-%E5%9C%A8%E7%BE%A4%E8%81%8A%E4%B8%AD%E4%BD%BF%E7%94%A8%E6%9C%BA%E5%99%A8%E4%BA%BA) ## New Features(2020.06.11) 以下的功能都没有经过实战使用检验，可能有bug。 #### 1. 支持一些复杂筛选条件 现在支持在Table那个页面的search框中输入复杂搜索逻辑进行筛选，例如下面log -![before_search](doc/source/figures/before_search.jpg) +![before_search](docs/source/figures/before_search.jpg) 想要搜索**2020年06月11号15点到2020年06月11号16点之间，并且hidden_size大于60**的log, 可以通过下面的语法输入到search框中 ``` @@ -74,7 +67,7 @@ ${'id':'log_20200611_150000<=&&<=log_20200611_160000', 'hyper-hidden_size':'>60' ``` 使用效果如下（只显示满足条件的log了） -![after_search](./doc/source/figures/after_search.jpg) +![after_search](docs/source/figures/after_search.jpg) 其中开头结尾的$是特殊标记符号，申明这是特殊搜索，否则fitlog会只进行常规字符串匹配搜索。由于search框一旦有文字就会触发搜索，所以建议在其他地方把文本编辑好，直接复制到search框中。支持的search语法规则如下 @@ -88,10 +81,10 @@ ${'id':'log_20200611_150000<=&&<=log_20200611_160000', 'hyper-hidden_size':'>60' #### 2.支持多条log的metric收敛曲线对比 操作如下图所示，先选择需要对比的log(默认只能选择10条)，然后点击右上角红色框处 -![compare_metric](./doc/source/figures/compare_metric.jpg) +![compare_metric](docs/source/figures/compare_metric.jpg) 然后会弹出选择需要对比的metric，选好需要对比的metric后点击确认跳转，跳转后界面为 -![compare_metric_trend](./doc/source/figures/compare_metric_trend.jpg) +![compare_metric_trend](docs/source/figures/compare_metric_trend.jpg) 我十分垃圾的js代码能力坚定地告诉我：你们一定不可能需要对比loss曲线的，所以我就没有必要再做loss对比了🐶。 diff --git a/doc/source/inner_api.rst b/doc/source/inner_api.rst deleted file mode 100644 index 4ea51f6..0000000 --- a/doc/source/inner_api.rst +++ /dev/null @@ -1,11 +0,0 @@ -============= -内部 API -============= - -内部 API 是为开发者准备的。你可以自行 fork 我们的 github 库进行再开发，或者联系我们提出建议。 - -.. toctree:: - :maxdepth: 1 - - fitlog.fastgit - fitlog.fastlog \ No newline at end of file diff --git a/doc/Makefile b/docs/Makefile similarity index 84% rename from doc/Makefile rename to docs/Makefile index 9b469c4..0cc6290 100644 --- a/doc/Makefile +++ b/docs/Makefile @@ -20,7 +20,10 @@ server: cd build/html && python -m http.server dev: - rm -rf build && make html && make server + rm -f source/$(SPHINXPROJ).* source/modules.rst && rm -rf build && make apidoc && make html && make server + +prod: + make apidoc && make html .PHONY: help Makefile diff --git a/doc/source/conf.py b/docs/source/conf.py similarity index 100% rename from doc/source/conf.py rename to docs/source/conf.py diff --git a/doc/source/figures/add_bot_1.png b/docs/source/figures/add_bot_1.png similarity index 100% rename from doc/source/figures/add_bot_1.png rename to docs/source/figures/add_bot_1.png diff --git a/doc/source/figures/after_search.jpg b/docs/source/figures/after_search.jpg similarity index 100% rename from doc/source/figures/after_search.jpg rename to docs/source/figures/after_search.jpg diff --git a/doc/source/figures/before_search.jpg b/docs/source/figures/before_search.jpg similarity index 100% rename from doc/source/figures/before_search.jpg rename to docs/source/figures/before_search.jpg diff --git a/doc/source/figures/compare_metric.jpg b/docs/source/figures/compare_metric.jpg similarity index 100% rename from doc/source/figures/compare_metric.jpg rename to docs/source/figures/compare_metric.jpg diff --git a/doc/source/figures/compare_metric_trend.jpg b/docs/source/figures/compare_metric_trend.jpg similarity index 100% rename from doc/source/figures/compare_metric_trend.jpg rename to docs/source/figures/compare_metric_trend.jpg diff --git a/doc/source/figures/fitlog_chart.png b/docs/source/figures/fitlog_chart.png similarity index 100% rename from doc/source/figures/fitlog_chart.png rename to docs/source/figures/fitlog_chart.png diff --git a/doc/source/figures/fitlog_table.png b/docs/source/figures/fitlog_table.png similarity index 100% rename from doc/source/figures/fitlog_table.png rename to docs/source/figures/fitlog_table.png diff --git a/doc/source/fitlog.fastgit.rst b/docs/source/fitlog.fastgit.rst similarity index 100% rename from doc/source/fitlog.fastgit.rst rename to docs/source/fitlog.fastgit.rst diff --git a/doc/source/fitlog.fastlog.rst b/docs/source/fitlog.fastlog.rst similarity index 100% rename from doc/source/fitlog.fastlog.rst rename to docs/source/fitlog.fastlog.rst diff --git a/doc/source/fitlog.rst b/docs/source/fitlog.rst similarity index 100% rename from doc/source/fitlog.rst rename to docs/source/fitlog.rst diff --git a/doc/source/index.rst b/docs/source/index.rst similarity index 77% rename from doc/source/index.rst rename to docs/source/index.rst index 17febcf..160db88 100644 --- a/doc/source/index.rst +++ b/docs/source/index.rst @@ -24,13 +24,10 @@ fitlog是一款集成了自动版本管理和自动日志记录两种功能的 P API 文档 ------------- -一般情况下你只需要阅读用户 API 文档即可，里面包括了你可以调用的函数接口。内部 API 是为开发者准备的。 - .. toctree:: :maxdepth: 1 - 用户 API - 内部 API + fitlog diff --git a/doc/source/modules.rst b/docs/source/modules.rst similarity index 100% rename from doc/source/modules.rst rename to docs/source/modules.rst diff --git a/doc/source/user/command_line.rst b/docs/source/user/command_line.rst similarity index 100% rename from doc/source/user/command_line.rst rename to docs/source/user/command_line.rst diff --git a/doc/source/user/configs.rst b/docs/source/user/configs.rst similarity index 100% rename from doc/source/user/configs.rst rename to docs/source/user/configs.rst diff --git a/doc/source/user/guide.rst b/docs/source/user/guide.rst similarity index 100% rename from doc/source/user/guide.rst rename to docs/source/user/guide.rst diff --git a/doc/source/user/installation.rst b/docs/source/user/installation.rst similarity index 100% rename from doc/source/user/installation.rst rename to docs/source/user/installation.rst diff --git a/doc/source/user/quickstart.rst b/docs/source/user/quickstart.rst similarity index 100% rename from doc/source/user/quickstart.rst rename to docs/source/user/quickstart.rst diff --git a/doc/source/user/tips.rst b/docs/source/user/tips.rst similarity index 100% rename from doc/source/user/tips.rst rename to docs/source/user/tips.rst diff --git a/doc/source/user/website.rst b/docs/source/user/website.rst similarity index 100% rename from doc/source/user/website.rst rename to docs/source/user/website.rst diff --git a/fitlog/fastlog/logger.py b/fitlog/fastlog/logger.py index 6bdc546..d7fe756 100755 --- a/fitlog/fastlog/logger.py +++ b/fitlog/fastlog/logger.py @@ -1,7 +1,6 @@ import logging import os import requests -import json from datetime import datetime from copy import deepcopy import argparse @@ -236,12 +235,22 @@ def _create_log_files(self): if not hasattr(self, 'meta_logger'): if self._save_log_dir is None: now = datetime.now().strftime('%Y%m%d_%H%M%S') - self._save_log_dir = os.path.join(self._log_dir, 'log_' + now) - while os.path.exists(self._save_log_dir): + _save_log_dir = os.path.join(self._log_dir, 'log_' + now) + while os.path.exists(_save_log_dir): time.sleep(1) now = datetime.now().strftime('%Y%m%d_%H%M%S') - self._save_log_dir = os.path.join(self._log_dir, 'log_' + now) - os.mkdir(self._save_log_dir) + _save_log_dir = os.path.join(self._log_dir, 'log_' + now) + while True: + try: + os.mkdir(_save_log_dir) + break + except FileExistsError: + time.sleep(1) + now = datetime.now().strftime('%Y%m%d_%H%M%S') + _save_log_dir = os.path.join(self._log_dir, 'log_' + now) + except Exception as e: + raise e + self._save_log_dir = _save_log_dir # prepare logger formatter = logging.Formatter('%(message)s') # 只保存记录的时间与记录的内容 for name in ['meta', 'hyper', 'metric', 'other', 'loss', 'progress', 'best_metric', 'file']: diff --git a/fitlog/fastserver/static/js/bootstrap-table-export.js b/fitlog/fastserver/static/js/bootstrap-table-export.js index 60c29e2..ca362c7 100755 --- a/fitlog/fastserver/static/js/bootstrap-table-export.js +++ b/fitlog/fastserver/static/js/bootstrap-table-export.js @@ -24,7 +24,7 @@ $.extend($.fn.bootstrapTable.defaults, { showExport: false, exportDataType: 'basic', // basic, all, selected - // 'json', 'xml', 'png', 'csv', 'txt', 'sql', 'doc', 'excel', 'powerpoint', 'pdf' + // 'json', 'xml', 'png', 'csv', 'txt', 'sql', 'docs', 'excel', 'powerpoint', 'pdf' exportTypes: ['json', 'xml', 'csv', 'txt', 'sql', 'excel'], exportOptions: {} }); diff --git a/fitlog/fastserver/static/js/jquery.min.js b/fitlog/fastserver/static/js/jquery.min.js index 7b91071..b7420d3 100755 --- a/fitlog/fastserver/static/js/jquery.min.js +++ b/fitlog/fastserver/static/js/jquery.min.js @@ -1115,7 +1115,7 @@ setDocument = Sizzle.setDocument = function( node ) { var hasCompare, subWindow, doc = node ? node.ownerDocument || node : preferredDoc; - // Return early if doc is invalid or already selected + // Return early if docs is invalid or already selected if ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) { return document; } diff --git a/readthedocs.yml b/readthedocs.yml index 8110d0f..e6d5baf 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -1,7 +1,7 @@ version: 2 sphinx: - configuration: doc/source/conf.py + configuration: docs/source/conf.py build: image: latest diff --git a/test/fastgit/test_fastgit.py b/test/fastgit/test_fastgit.py new file mode 100644 index 0000000..3a0b983 --- /dev/null +++ b/test/fastgit/test_fastgit.py @@ -0,0 +1,20 @@ +import unittest +import os +import shutil +from fitlog.fastgit import committer + + +class TextExample(unittest.TestCase): + def setUp(self): # 定义一个setup，放一些准备的工作，或者准备一些测试数据。 + os.mkdir('testArea') + + def test_init(self): + ret = committer.init_project('testArea/test_pj', git=False) + self.assertEqual(ret, 0) + + def tearDown(self): # 定义一个tearDown，在测试完的时候我要对测试有一个销毁的过程 + shutil.rmtree('testArea', ignore_errors=True) + + # 基本参考 https://docs.python.org/zh-cn/3/library/unittest.html + # mock 提供了更多的可能性 https://docs.python.org/zh-cn/3/library/unittest.mock.html + diff --git a/test/fastserver/test_app.py b/test/fastserver/test_app.py index 146e2d4..de69299 100644 --- a/test/fastserver/test_app.py +++ b/test/fastserver/test_app.py @@ -1,12 +1,12 @@ - from fitlog.fastserver.app import start_app -def run_app(): - log_dir ='../../model_test/codes/V1/logs' +def run_app(): + log_dir = '../../model_test/codes/V1/logs' log_config_name = 'default.cfg' port = 5000 start_app(log_dir, log_config_name, port, 1, '123') + if __name__ == '__main__': - run_app() \ No newline at end of file + run_app() diff --git a/test/fastserver/test_summary_utils.py b/test/fastserver/test_summary_utils.py index dfc4fc4..34b30d6 100644 --- a/test/fastserver/test_summary_utils.py +++ b/test/fastserver/test_summary_utils.py @@ -1,8 +1,8 @@ - from itertools import groupby from operator import itemgetter import numpy as np + def groupBy(data, key): """ @@ -11,9 +11,10 @@ def groupBy(data, key): :return: 可以看成[[key1, group1], [key2, group2]] """ data.sort(key=itemgetter(key)) - grouped_data = groupby(data, itemgetter(key)) # key + 迭代器。可以看成[[key1, group1], [key2, group2]] + grouped_data = groupby(data, itemgetter(key)) # key + 迭代器。可以看成[[key1, group1], [key2, group2]] return grouped_data + def get_grouped_data(data, keys): """ @@ -22,7 +23,7 @@ def get_grouped_data(data, keys): :return: {}nested的dict，最后的value是符合这个group的结果 """ _dict = {} - if len(keys)==1: + if len(keys) == 1: grouped_data = groupBy(data, keys[0]) for key, group in grouped_data: _dict[key] = list(group) @@ -33,6 +34,7 @@ def get_grouped_data(data, keys): _dict[key] = get_grouped_data(list(group), keys[1:]) return _dict + def merge(a, b, path=None): "merges b into a" if path is None: path = [] @@ -41,7 +43,7 @@ def merge(a, b, path=None): if isinstance(a[key], dict) and isinstance(b[key], dict): merge(a[key], b[key], path + [str(key)]) elif a[key] == b[key]: - pass # same leaf value + pass # same leaf value else: raise Exception('Conflict at %s' % '.'.join(path + [str(key)])) else: @@ -81,4 +83,4 @@ def merge(a, b, path=None): root_log_dir = '../../model_test/codes/V1/logs' print(generate_summary_table(vertical, horizontals, method, criteria, results, results, selected_data, - root_log_dir)) \ No newline at end of file + root_log_dir))