diff --git a/Makefile b/Makefile index 4fea7d1..04a0d24 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ python:=$(python_venv)/bin/python init: python -m venv $(python_venv) + $(python) -m pip install -r requirements-dev.txt reinit: @@ -37,3 +38,8 @@ publish-test: build publish: build twine upload dist/* + +.PHONY: run-exmaple +run-example: + $(python) -m pip install -e . + cd example && make build diff --git a/example/Makefile b/example/Makefile index 30747b6..db0ad61 100644 --- a/example/Makefile +++ b/example/Makefile @@ -1,7 +1,7 @@ root_dir:=$(abspath $(CURDIR)/..) -build_dir:=$(root_path)/example/build -excel2xx_bin:=python3 -m excel2xx -excel2xx_bin_debug:=EXCEL2XX_DEBUG=1 python3 -m excel2xx +build_dir:=$(root_dir)/example/build +excel2xx_bin:=$(root_dir)/.venv/bin/excel2xx +excel2xx_bin_debug:=EXCEL2XX_DEBUG=1 $(excel2xx_bin) build: clean diff --git a/example/test.lua.mako b/example/test.lua.mako index e628ffc..43f0fa0 100644 --- a/example/test.lua.mako +++ b/example/test.lua.mako @@ -1,6 +1,6 @@ local t = { -% for sheet in excel: - ${sheet.name} = { +% for name, sheet in excel.items(): + ${name} = { % for row in sheet: { % for key in row: diff --git a/excel2xx/__init__.py b/excel2xx/__init__.py index f550c75..d311991 100644 --- a/excel2xx/__init__.py +++ b/excel2xx/__init__.py @@ -8,7 +8,7 @@ import excel2xx.exporter as exporter -VERSION = "0.11.2" +VERSION = "0.11.3" __author__ = "cupen" __email__ = "xcupen@gmail.com" diff --git a/excel2xx/codegen/__init__.py b/excel2xx/codegen/__init__.py index c6dfc7c..af887d9 100644 --- a/excel2xx/codegen/__init__.py +++ b/excel2xx/codegen/__init__.py @@ -29,6 +29,7 @@ def generate(src: list, context={}, note=NOTE, encoding="utf-8", newline="\n"): print(f"\ttemplate files: {len(fpaths)}") for fpath in fpaths: + fpath_new = fpath.replace(".mako", "") print(f"\t{fpath} -> ", end="") if not fpath.endswith(".mako"): print(console.Colors.yellow(fpath_new)) @@ -39,7 +40,6 @@ def generate(src: list, context={}, note=NOTE, encoding="utf-8", newline="\n"): ctx["__dir__"] = os.path.dirname(fpath) tmpl = load_template(fpath) - fpath_new = fpath.replace(".mako", "") try: text = tmpl.render(**ctx) with open(fpath_new, "w", encoding=encoding, newline=newline) as fp: diff --git a/excel2xx/console/__init__.py b/excel2xx/console/__init__.py index c35645c..2a0af50 100644 --- a/excel2xx/console/__init__.py +++ b/excel2xx/console/__init__.py @@ -1,4 +1,7 @@ # coding: utf-8 +import colorama + +colorama.init(autoreset=True) from colorama import Fore, Back, Style from .colors import Colors diff --git a/excel2xx/export.py b/excel2xx/export.py index b037c89..3ab2392 100644 --- a/excel2xx/export.py +++ b/excel2xx/export.py @@ -7,6 +7,7 @@ from mako.template import Template from io import open from datetime import datetime, date +from excel2xx.exporter import auto def _defaultSerialize(obj): @@ -16,31 +17,18 @@ def _defaultSerialize(obj): def toJson(excel, output, encoding="utf-8"): - _dict = OrderedDict() - for sheet in excel: - _dict[sheet.name] = list(sheet) - - if sys.version_info[0] == 2: - with open(output, mode="wb") as f: - json.dump( - _dict, - f, - ensure_ascii=False, - encoding=encoding, - default=_defaultSerialize, - ) - return - - with open(output, mode="w", encoding=encoding) as f: - json.dump(_dict, f, ensure_ascii=False, indent=4, default=_defaultSerialize) + data = auto.export(excel) + with open(output, "w", encoding=encoding) as fp: + json.dump(data, fp, ensure_ascii=False, indent=4, default=_defaultSerialize) pass def toMako(excel, output, template, encoding="utf-8"): + data = auto.export(excel) with open(output, mode="w", encoding=encoding) as ouputfile: text = "" with open(template, mode="r", encoding=encoding) as f: - text = Template(f.read()).render(excel=excel, format=pformat) + text = Template(f.read()).render(excel=data, format=pformat) ouputfile.write(text) pass @@ -48,22 +36,13 @@ def toMako(excel, output, template, encoding="utf-8"): def toMsgPack(excel, output, encoding="utf-8"): import msgpack - _dict = OrderedDict() - for sheet in excel: - _dict[sheet.name] = list(sheet) - + data = auto.export(excel) with open(output, mode="wb") as f: - f.write(msgpack.packb(_dict, default=_defaultSerialize)) + f.write(msgpack.packb(data, default=_defaultSerialize)) pass def toCSV(excel, output, encoding="utf-8"): - import msgpack - - _dict = OrderedDict() - for sheet in excel: - _dict[sheet.name] = list(sheet) - - with open(output, mode="wb") as f: - f.write(msgpack.packb(_dict, default=_defaultSerialize)) + data = exporter.auto(excel) + raise NotImplementedError("toCSV") pass diff --git a/excel2xx/exporter/auto.py b/excel2xx/exporter/auto.py index 2019b21..5689b9f 100644 --- a/excel2xx/exporter/auto.py +++ b/excel2xx/exporter/auto.py @@ -7,6 +7,8 @@ def export(ex, ver=1): if str.startswith(sheet.name, "#"): continue name, _type = parse_sheet_name(sheet.name) + if not name: + raise Exception(f"invalid sheet {sheet.name}") if _type == "list": d[name] = sheet.toList() elif _type == "map": @@ -18,40 +20,52 @@ def export(ex, ver=1): return d -def parse_sheet_name(name) -> (str, str): +def parse_sheet_name(name: str) -> tuple[str, str]: """ - >>> _parse_sheet_name("name") -> ("name", "list") + >>> _parse_sheet_name("name") + ('name', 'list') + >>> _parse_sheet_name("name(list)") + ('name', 'list') + >>> _parse_sheet_name("name123(map)") + ('name123', 'map') + >>> _parse_sheet_name("name123((map)") + Traceback (most recent call last): + Exception: duplicated symbol '(' + >>> _parse_sheet_name("sheet-map(map)") + ('sheet-map', 'map') """ name = name.strip() - if "=" not in name: - return name, "list" - arr = list(map(lambda x: x.strip(), name.split("="))) - if len(arr) <= 1: - return "", "" - name = arr[1] + if "=" in name: + arr = list(map(lambda x: x.strip(), name.split("="))) + name = arr[1] return _parse_sheet_name(name) -def _parse_sheet_name(name) -> (str, str): +def _parse_sheet_name(name) -> tuple[str, str]: realName = "" realType = "" - state = 0 # 0-init, 1-type, 2-end + state = 0 # 0-start, 1-name, 2-type, 3-end for c in name: if c == "(": - if state != 0: + if state != 1: raise Exception(f"duplicated symbol '('") - state = 1 + state = 2 elif c == ")": - if state != 1: + if state != 2: raise Exception(f" ')' must behind of '('") if len(realType) <= 0: raise Exception(f" '()' must be one of list, map, kv") - state = 2 + state = 3 break # end else: + if not str.isalnum(c) and c not in ["-", "_"]: + raise Exception(f"invalid char '{c}'") if state == 0: - realName += c + realName = c + state = 1 elif state == 1: + realName += c + elif state == 2: realType += c else: raise Exception(f"BUG!") diff --git a/excel2xx/main.py b/excel2xx/main.py index acaaa00..9a6222e 100644 --- a/excel2xx/main.py +++ b/excel2xx/main.py @@ -12,13 +12,15 @@ Options: -h --help show this help message and exit. - -v --version show version and exit. + --version show version and exit. -o --output=FILE output to file. - -r --row-number=ROW_NUMBER first row. [default: 2] + --name-row=NAME_ROW name row number. [default: 1] + --type-row=TYPE_ROW type row number. [default: 2] + --desc-row=DESC_ROW desc row number. [default: 3] + --data-row=DATA_ROW data row number. [default: 4] -v --verbose show debug infomation. -vv --verbose2 show more debug infomation. """ -from __future__ import unicode_literals, print_function import os import traceback from docopt import docopt @@ -38,8 +40,13 @@ def main(args): print("Unexist file:" + excelFile) return 1 - excel = Excel(excelFile, fieldMeta=FieldMeta()) - + meta = FieldMeta( + name=int(args["--name-row"]) - 1, + type=int(args["--type-row"]) - 1, + desc=int(args["--desc-row"]) - 1, + data=int(args["--data-row"]) - 1, + ) + excel = Excel(excelFile, fieldMeta=meta) if args["json"]: export.toJson(excel, args["--output"] or "%(excelFile)s.json" % locals()) elif args["msgpack"]: @@ -51,12 +58,11 @@ def main(args): else: print("Invalid subcmd.") return 2 - return 0 -def main_docopt(): - args = docopt(__doc__) +def main_docopt(argv=None): + args = docopt(__doc__, argv) if args["--verbose2"]: print(args) try: @@ -69,4 +75,6 @@ def main_docopt(): if __name__ == "__main__": - exit(main_docopt()) + import sys + + exit(main_docopt(sys.argv)) diff --git a/excel2xx/validator/__init__.py b/excel2xx/validator/__init__.py index a11b357..8fef513 100644 --- a/excel2xx/validator/__init__.py +++ b/excel2xx/validator/__init__.py @@ -38,7 +38,7 @@ def run(data, data_dir: str, validator_name="_validator"): def list_files(_dir: str): import bisect - rs = [] + rs = [] # type: list[str] for base, dirs, files in os.walk(_dir): for fname in files: if not fname.endswith(".py"): @@ -51,19 +51,3 @@ def list_files(_dir: str): pass pass return rs - - -def green(text): - return Fore.LIGHTGREEN_EX + str(text) + Fore.RESET - - -def red(text): - return Fore.LIGHTRED_EX + str(text) + Fore.RESET - - -def blue(text): - return Fore.LIGHTBLUE_EX + str(text) + Fore.RESET - - -def yellow(text): - return Fore.LIGHTYELLOW_EX + str(text) + Fore.RESET diff --git a/setup.py b/setup.py index 2103df5..ab36db4 100644 --- a/setup.py +++ b/setup.py @@ -4,13 +4,23 @@ root_dir = os.path.dirname(__file__) -fpath = os.path.join(root_dir, "README.md") -readme = open(fpath, "r", encoding="utf-8").read() +_open = lambda fname: open(os.path.join(root_dir, fname), "r", encoding="utf-8") +with _open("README.md") as fp: + readme = fp.read() + pass + +with _open("requirements.txt") as fp: + deps = map(str.strip, fp.readlines()) + deps = filter(lambda line: bool(line), deps) + deps = list(deps) + pass + +print(deps) setup( name="excel2xx", - version="0.11.2", + version="0.11.3", packages=find_packages(), url="https://github.com/cupen/excel2xx", license="WTFPL", @@ -19,13 +29,7 @@ description="Extract data from excel file, and export to json, msgpack, or any code(mako template).", long_description=readme, long_description_content_type="text/markdown", - install_requires=[ - "xlrd == 1.2.*", - "docopt >= 0.6.0", - "mako == 1.2.*", - "msgpack-python >= 0.4.8", - "colorama >= 0.4.6", - ], + install_requires=deps, entry_points={ "console_scripts": [ "excel2xx=excel2xx.main:main_docopt", diff --git a/tests/test_excel.py b/tests/test_excel.py new file mode 100644 index 0000000..28ddecc --- /dev/null +++ b/tests/test_excel.py @@ -0,0 +1,60 @@ +import os +import json +from conftest import testdata_dir +import openpyxl +from openpyxl.styles import Font, Color + +excel_dir = os.path.join(testdata_dir, "_excel") +excel_files = [os.path.join(excel_dir, "test.xlsx")] + + +def F(fname): + return os.path.join(excel_dir, fname) + + +def gen_cases(): + os.makedirs(excel_dir, exist_ok=True) + w = openpyxl.Workbook() + s = w.create_sheet(title="auto") + s.append(["key", "type", "value"]) + s.append(["string", "auto", "auto"]) + s.append(["", "", ""]) + s.append(["key1", "int", 1]) + s.append(["key2", "string", "2"]) + s.append(["key3", "array", "1,2,3"]) + for i in range(3): + s = w.create_sheet(title=f"sheet{i}") + s.append(["key1", "key2", "key3", "key3"]) + s.append(["int", "string", "float", "array"]) + s.append(["", "", "", ""]) + s.append([1, "2", 3.0, "1,2,3"]) + pass + + s = w.create_sheet("sheet-map(map)") + s.append(["key1", "key2"]) + s.append(["int", "string"]) + s.append(["", ""]) + s.append([1, "1"]) + s.append([2, "2"]) + s.append([3, "3"]) + + w.save(excel_files[0]) + pass + + +def test_excel_json(): + gen_cases() + from excel2xx import main + + argv = ["json", "-v", "-o", F("yes.json"), excel_files[0]] + code = main.main_docopt(argv) + assert 0 == code + with open(F("yes.json")) as fp: + data = json.load(fp) + pass + assert 3 == len(data["auto"]) + assert 1 == len(data["sheet0"]) + assert 1 == len(data["sheet1"]) + assert 1 == len(data["sheet2"]) + assert 3 == len(data["sheet-map"]) + pass diff --git a/tox.ini b/tox.ini index 722297b..c04074f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,28 +1,32 @@ ; @see http://tox.readthedocs.org/en/latest/config.html [tox] requires = tox>=4 -env_list = lint, type, py{36,37,38,39,310,311} +env_list = lint, example, py{36,37,38,39,310,311,312} skipsdist=True +skip_missing_interpreters = true [testenv] description = run unit tests -deps = - pytest>=7 - pytest-sugar - xlrd == 2.0.* - docopt >= 0.6.0 - mako >= 1.0.0 - msgpack-python >= 0.4.8 - colorama >= 0.4.6 -commands = +deps = -r requirements-dev.txt +commands = + # mypy excel2xx + python -m doctest ./excel2xx/exporter/auto.py pytest {posargs:tests} + [testenv:lint] description = run linters skip_install = true -deps = - black==22.12 -commands = black {posargs:.} +deps = black==22.12 +commands = + black {posargs:.} + + +[testenv:example] +description = run examples +allowlist_externals=make +deps = -r requirements.txt +commands = make run-example [gh] @@ -33,3 +37,4 @@ python = 3.9 = py39 3.10 = py310, type 3.11 = py311 + 3.12 = py312