Skip to content

Commit c595750

Browse files
committed
update: foliant, workflow, lint settings, bump python version, tests
1 parent 53ad1cd commit c595750

File tree

11 files changed

+111
-128
lines changed

11 files changed

+111
-128
lines changed

.github/workflows/python-test.yml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@ jobs:
2424
version: 1.0.0
2525
- name: Install library
2626
run: poetry install --no-interaction
27-
- name: Lint and test with poetry
27+
- name: Lint with poetry
28+
run: poetry run pylint foliant
29+
- name: Test with poetry
2830
run: |
2931
poetry run pytest --cov=foliant
30-
poetry run codecov
31-
poetry run pylint foliant
32+
poetry run codecov

foliant/backends/base.py

Lines changed: 70 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
from datetime import date
77
from logging import Logger
88
from glob import glob
9+
from typing import Union, List, Set
910
from foliant.utils import spinner
10-
from typing import Union, List
1111

1212
class BaseBackend():
1313
'''Base backend. All backends must inherit from this one.'''
@@ -92,26 +92,42 @@ def partial_copy(
9292
root: Union[str, Path]
9393
) -> None:
9494
"""
95-
Copies files, a list of files, or files matching a glob pattern to the specified folder.
95+
Copies files, a list of files,
96+
or files matching a glob pattern to the specified folder.
9697
Creates all necessary directories if they don't exist.
97-
98-
:param source: A file path, a list of file paths, or a glob pattern (as a string or Path object).
99-
:param destination: Target folder (as a string or Path object).
100-
:param root: Base folder to calculate relative paths (optional). If not provided, the parent directory of the source is used.
10198
"""
102-
# Convert destination to a Path object
10399
destination_path = Path(destination)
100+
root_path = Path(root)
101+
image_extensions = {'.jpg', '.jpeg', '.png', '.svg', '.gif', '.bmp', '.webp'}
102+
image_pattern = re.compile(r'!\[.*?\]\((.*?)\)|<img.*?src=["\'](.*?)["\']', re.IGNORECASE)
103+
include_statement_pattern = re.compile(
104+
r'(?<!\<)\<(?:include)(?:\s[^\<\>]*)?\>(?P<path>.*?)\<\/(?:include)\>',
105+
flags=re.DOTALL
106+
)
104107

105-
def extract_first_header(file_path):
108+
def _extract_first_header(file_path):
106109
"""Extracts the first first-level header from the Markdown file."""
107110
with open(file_path, 'r', encoding='utf-8') as file:
108111
for line in file:
109112
match = re.match(r'^#\s+(.*)', line)
110113
if match:
111-
return match.group(0) # Returns the header
112-
return None # If the header is not found
114+
return match.group(0)
115+
return None
113116

114-
def copy_files_without_content(src_dir, dst_dir):
117+
def _find_referenced_images(file_path: Path) -> Set[Path]:
118+
"""Finds all image files referenced in the given file."""
119+
image_paths = set()
120+
with open(file_path, 'r', encoding='utf-8') as file:
121+
content = file.read()
122+
for match in image_pattern.findall(content):
123+
for group in match:
124+
if group:
125+
image_path = Path(file_path).parent / Path(group)
126+
if image_path.suffix.lower() in image_extensions:
127+
image_paths.add(image_path)
128+
return image_paths
129+
130+
def _copy_files_without_content(src_dir: str, dst_dir: str):
115131
"""Copies files, leaving only the first-level header."""
116132
if not os.path.exists(dst_dir):
117133
os.makedirs(dst_dir)
@@ -123,17 +139,54 @@ def copy_files_without_content(src_dir, dst_dir):
123139
dst_file_path = Path(os.path.join(dst_dir, dirs, file_name))
124140
dst_file_path.parent.mkdir(parents=True, exist_ok=True)
125141
if file_name.endswith('.md'):
126-
header = extract_first_header(src_file_path)
142+
header = _extract_first_header(src_file_path)
127143
if header:
128144
with open(dst_file_path, 'w', encoding='utf-8') as dst_file:
129145
dst_file.write(header + '\n')
130146
else:
131-
copy(src_file_path, dst_file_path)
147+
if Path(src_file_path).suffix.lower() not in image_extensions:
148+
copy(src_file_path, dst_file_path)
149+
150+
def _copy_files_recursive(files_to_copy: List):
151+
"""Recursively copies files and their dependencies."""
152+
referenced_images = set()
153+
154+
for file_path in files_to_copy:
155+
relative_path = file_path.relative_to(root_path)
156+
destination_file_path = destination_path / relative_path
157+
destination_file_path.parent.mkdir(parents=True, exist_ok=True)
158+
159+
# Find and copy includes
160+
include_paths = []
161+
match_includes = re.findall(include_statement_pattern,
162+
file_path.read_text(encoding='utf-8'))
163+
for path in match_includes:
164+
_path = Path(path)
165+
if not _path.exists():
166+
_path = relative_path / path
167+
if _path.exists():
168+
include_paths.append(_path)
169+
_copy_files_recursive(include_paths)
170+
171+
# Find referenced images
172+
referenced_images.update(_find_referenced_images(file_path))
173+
174+
# Copy the file
175+
copy(file_path, destination_file_path)
176+
177+
# Copy referenced images
178+
for image_path in referenced_images:
179+
src_image_path = Path(image_path).relative_to(root_path)
180+
dst_image_path = destination_path / src_image_path
181+
dst_image_path.parent.mkdir(parents=True, exist_ok=True)
182+
183+
if Path(image_path).exists():
184+
copy(image_path, dst_image_path)
185+
186+
# Basic logic
187+
_copy_files_without_content(root_path, destination_path)
132188

133-
copy_files_without_content(root, destination)
134-
# Handle case where source is a list of files
135189
if isinstance(source, str) and ',' in source:
136-
print( source)
137190
source = source.split(',')
138191
if isinstance(source, list):
139192
files_to_copy = []
@@ -143,38 +196,19 @@ def copy_files_without_content(src_dir, dst_dir):
143196
raise FileNotFoundError(f"Source '{item}' not found.")
144197
files_to_copy.append(item_path)
145198
else:
146-
# Convert source to a Path object if it's a string
147199
if isinstance(source, str):
148200
source_path = Path(source)
149201
else:
150202
source_path = source
151203

152-
# Check if the source is a glob pattern
153204
if isinstance(source, str) and ('*' in source or '?' in source or '[' in source):
154-
# Use glob to find files matching the pattern
155205
files_to_copy = [Path(file) for file in glob(source, recursive=True)]
156206
else:
157-
# Check if the source file or directory exists
158207
if not source_path.exists():
159208
raise FileNotFoundError(f"Source '{source_path}' not found.")
160209
files_to_copy = [source_path]
161210

162-
# Determine the root directory for calculating relative paths
163-
root = Path(root)
164-
165-
# Copy each file
166-
for file_path in files_to_copy:
167-
# Calculate the relative path
168-
relative_path = file_path.relative_to(root)
169-
170-
# Full path to the destination file
171-
destination_file_path = destination_path / relative_path
172-
173-
# Create directories if they don't exist
174-
destination_file_path.parent.mkdir(parents=True, exist_ok=True)
175-
176-
# Copy the file
177-
copy(file_path, destination_file_path)
211+
_copy_files_recursive(files_to_copy)
178212

179213
def preprocess_and_make(self, target: str) -> str:
180214
'''Apply preprocessors required by the selected backend and defined in the config file,

foliant/backends/pre.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ def __init__(self, *args, **kwargs):
2424
def make(self, target: str) -> str:
2525
rmtree(self._preprocessed_dir_name, ignore_errors=True)
2626
if self.context['only_partial']:
27-
self.partial_copy(self.working_dir / self.context['only_partial'], self._preprocessed_dir_name)
27+
self.partial_copy(self.working_dir / self.context['only_partial'],
28+
self._preprocessed_dir_name,
29+
self.working_dir)
2830
else:
2931
copytree(self.working_dir, self._preprocessed_dir_name)
3032

foliant/cli/__init__.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ class Foliant(*get_available_clis().values()):
1212

1313
@set_help({'version': 'show version and exit'})
1414
def _root(self, version=False):
15-
# pylint: disable=no-self-use
1615

1716
if version:
1817
print(f'Foliant v.{foliant_version}')

foliant/cli/make.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ def make(
163163
# pylint: disable=consider-using-sys-exit
164164

165165
self.logger.setLevel(DEBUG if debug else WARNING)
166+
result = None
166167

167168
if logs_dir:
168169
super().__init__(logs_dir)

foliant/preprocessors/base.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from logging import Logger
33
from typing import Dict
44
import yaml
5+
56
OptionValue = int or float or bool or str
67

78

pylintrc

Lines changed: 7 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -55,88 +55,11 @@ confidence=
5555
# no Warning level messages displayed, use"--disable=all --enable=classes
5656
# --disable=W"
5757
disable=missing-docstring,
58-
print-statement,
59-
parameter-unpacking,
60-
unpacking-in-except,
61-
old-raise-syntax,
62-
backtick,
63-
long-suffix,
64-
old-ne-operator,
65-
old-octal-literal,
66-
import-star-module-level,
67-
non-ascii-bytes-literal,
68-
invalid-unicode-literal,
69-
raw-checker-failed,
70-
bad-inline-option,
71-
locally-disabled,
72-
locally-enabled,
73-
file-ignored,
74-
suppressed-message,
75-
useless-suppression,
76-
deprecated-pragma,
77-
too-few-public-methods,
78-
apply-builtin,
79-
basestring-builtin,
80-
buffer-builtin,
81-
cmp-builtin,
82-
coerce-builtin,
83-
execfile-builtin,
84-
file-builtin,
85-
long-builtin,
86-
raw_input-builtin,
87-
reduce-builtin,
88-
standarderror-builtin,
89-
unicode-builtin,
90-
xrange-builtin,
91-
coerce-method,
92-
delslice-method,
93-
getslice-method,
94-
setslice-method,
95-
no-absolute-import,
96-
old-division,
97-
dict-iter-method,
98-
dict-view-method,
99-
next-method-called,
100-
metaclass-assignment,
101-
indexing-exception,
102-
raising-string,
103-
reload-builtin,
104-
oct-method,
105-
hex-method,
106-
nonzero-method,
107-
cmp-method,
108-
input-builtin,
109-
round-builtin,
110-
intern-builtin,
111-
unichr-builtin,
112-
map-builtin-not-iterating,
113-
zip-builtin-not-iterating,
114-
range-builtin-not-iterating,
115-
filter-builtin-not-iterating,
116-
using-cmp-argument,
117-
eq-without-hash,
118-
div-method,
119-
idiv-method,
120-
rdiv-method,
121-
exception-message-attribute,
122-
invalid-str-codec,
123-
sys-max-int,
124-
bad-python3-import,
125-
deprecated-string-function,
126-
deprecated-str-translate-call,
127-
deprecated-itertools-function,
128-
deprecated-types-field,
129-
next-method-defined,
130-
dict-items-not-iterating,
131-
dict-keys-not-iterating,
132-
dict-values-not-iterating,
133-
deprecated-operator-function,
134-
deprecated-urllib-function,
135-
xreadlines-attribute,
136-
deprecated-sys-function,
137-
exception-escape,
138-
comprehension-escape,
139-
duplicate-code
58+
too-many-locals,
59+
too-few-public-methods,
60+
too-many-statements,
61+
unreachable,
62+
duplicate-code # TODO: fix duplicate code in backends.base and preprocessors.base
14063

14164
# Enable the message, report, category or checker with the given id(s). You can
14265
# either give multiple identifier separated by comma (,) or put this option
@@ -314,8 +237,8 @@ max-module-lines=1000
314237
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
315238
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
316239
# `empty-line` allows space-only lines.
317-
no-space-check=trailing-comma,
318-
dict-separator
240+
# no-space-check=trailing-comma,
241+
# dict-separator
319242

320243
# Allow the body of a class to be on the same line as the declaration if body
321244
# contains single statement.

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ documentation = "https://foliant-docs.github.io/docs/"
1111
keywords = ["documentation", "markdown", "html", "docx", "pdf"]
1212

1313
[tool.poetry.dependencies]
14-
python = "^3.6"
14+
python = "^3.8"
1515
pyyaml = "^5.1.1"
1616
cliar = "^1.3.2"
1717
prompt_toolkit = "^2.0"

test.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,6 @@
44
poetry install --no-interaction
55

66
# run tests
7-
poetry run pytest --cov=foliant -v
7+
poetry run pylint foliant && \
8+
poetry run pytest --cov=foliant && \
9+
poetry run codecov

test_in_docker.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# Write Dockerfile
3+
# # Write Dockerfile
44
echo "FROM python:3.9.21-alpine3.20" > Dockerfile
55
echo "RUN apk add --no-cache --upgrade bash && \
66
pip install poetry==1 && \

0 commit comments

Comments
 (0)