Skip to content

Commit e3da072

Browse files
tkiliaskaklakariada
tkilias
and
kaklakariada
authored
#40: Moved Config data model from SLC-CI-Setup to SLC-CI (#41)
Co-authored-by: Christoph Pirkl <christoph.pirkl@exasol.com>
1 parent d4e7938 commit e3da072

20 files changed

+1200
-61
lines changed

‎.gitattributes

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
poetry.lock linguist-generated=true
2+
exasol_script_languages_container_ci/lib/config/config_data_model.py linguist-generated=true

‎doc/changes/changelog.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Changes
22

3-
* [1.2.0](changes_1.1.0.md)
3+
* [1.3.0](changes_1.3.0.md)
4+
* [1.2.0](changes_1.2.0.md)
45
* [1.1.0](changes_1.1.0.md)
56
* [1.0.0](changes_1.0.0.md)
67
* [0.6.0](changes_0.6.0.md)

‎doc/changes/changes_1.3.0.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Script-Languages-Container-CI 1.3.0, 2023-07-04
2+
3+
Code name: Move config definiton from SLC-CI-Setup to this project
4+
5+
## Summary
6+
7+
This release moves the config from SLC-CI-Setup to this project, because both project use the config.
8+
9+
## Bug Fixes
10+
11+
- #40: Move config definiton from SLC-CI-Setup to this project
12+
13+
## Features / Enhancements
14+
15+
n/a
16+
17+
## Documentation
18+
19+
n/a
20+
21+
## Refactoring
22+
23+
n/a

‎exasol_script_languages_container_ci/cli/commands/run_ci.py

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
from exasol_script_languages_container_ci.cli.cli import cli
66
from exasol_script_languages_container_ci.lib.ci import ci
7+
from exasol_script_languages_container_ci.lib.config.config_data_model import Config
78
from exasol_script_languages_container_ci.lib.git_access import GitAccess
89

910

@@ -35,12 +36,13 @@ def run_ci(ctx: click.Context,
3536
commit_sha: str,
3637
config_file: str):
3738
logging.basicConfig(level=logging.INFO)
38-
ci(flavor,
39-
branch_name,
40-
docker_user,
41-
docker_password,
42-
docker_build_repository,
43-
docker_release_repository,
44-
commit_sha,
45-
config_file,
46-
GitAccess())
39+
build_config = Config.parse_file(config_file)
40+
ci(flavor=flavor,
41+
branch_name=branch_name,
42+
docker_user=docker_user,
43+
docker_password=docker_password,
44+
docker_build_repository=docker_build_repository,
45+
docker_release_repository=docker_release_repository,
46+
commit_sha=commit_sha,
47+
build_config=build_config,
48+
git_access=GitAccess())

‎exasol_script_languages_container_ci/cli/commands/run_release.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
from exasol_script_languages_container_ci.cli.cli import cli
77
from exasol_script_languages_container_ci.lib.asset_uploader import AssetUploader
8+
from exasol_script_languages_container_ci.lib.config.config_data_model import Config
89
from exasol_script_languages_container_ci.lib.github_release_asset_uploader import GithubReleaseAssetUploader
910
from exasol_script_languages_container_ci.lib.release import release
1011
from exasol_script_languages_container_ci.lib.release_uploader import ReleaseUploader
@@ -42,11 +43,12 @@ def run_release(ctx: click.Context,
4243
github_release_asset_uploader = GithubReleaseAssetUploader(os.getenv("GITHUB_TOKEN"))
4344
asset_uploader = AssetUploader(release_asset_uploader=github_release_asset_uploader)
4445
release_uploader = ReleaseUploader(asset_uploader=asset_uploader)
46+
build_config = Config.parse_file(config_file)
4547
release(flavor=flavor,
4648
docker_user=docker_user,
4749
docker_password=docker_password,
4850
docker_release_repository=docker_release_repository,
49-
config_file=config_file,
51+
build_config=build_config,
5052
source_repo_url=source_repo_url,
5153
release_id=release_id,
5254
release_uploader=release_uploader,

‎exasol_script_languages_container_ci/lib/ci.py

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,15 @@
33
from pathlib import Path
44
from typing import Set
55

6-
import click
6+
from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY
77
from exasol_integration_test_docker_environment.lib.base import luigi_log_config
8-
from exasol_integration_test_docker_environment.lib.config import build_config
98

109
from exasol_script_languages_container_ci.lib.branch_config import BranchConfig
11-
from exasol_script_languages_container_ci.lib.common import get_config
1210
from exasol_script_languages_container_ci.lib.ci_build import CIBuild
1311
from exasol_script_languages_container_ci.lib.ci_push import CIPush
1412
from exasol_script_languages_container_ci.lib.ci_security_scan import CISecurityScan
1513
from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest
14+
from exasol_script_languages_container_ci.lib.config.config_data_model import Config
1615
from exasol_script_languages_container_ci.lib.git_access import GitAccess
1716

1817

@@ -26,16 +25,15 @@ def get_all_affected_files(git_access: GitAccess, base_branch: str) -> Set[str]:
2625
return changed_files
2726

2827

29-
def check_if_need_to_build(branch_name: str, config_file: str, flavor: str, git_access: GitAccess):
28+
def check_if_need_to_build(branch_name: str, config: Config, flavor: str, git_access: GitAccess):
3029
if BranchConfig.build_always(branch_name):
3130
return True
3231
if "[rebuild]" in git_access.get_last_commit_message():
3332
return True
34-
with get_config(config_file) as config:
35-
affected_files = list(get_all_affected_files(git_access, config["base_branch"]))
36-
logging.debug(f"check_if_need_to_build: Found files of last commits: {affected_files}")
37-
for ignore_path in config["build_ignore"]["ignored_paths"]:
38-
affected_files = list(filter(lambda file: not file.startswith(ignore_path), affected_files))
33+
affected_files = list(get_all_affected_files(git_access, config.build.base_branch))
34+
logging.debug(f"check_if_need_to_build: Found files of last commits: {affected_files}")
35+
for ignore_path in config.build.ignore.paths:
36+
affected_files = list(filter(lambda file: not file.startswith(ignore_path), affected_files))
3937

4038
if len(affected_files) > 0:
4139
# Now filter out also other flavor folders
@@ -53,7 +51,7 @@ def ci(flavor: str,
5351
docker_build_repository: str,
5452
docker_release_repository: str,
5553
commit_sha: str,
56-
config_file: str,
54+
build_config: Config,
5755
git_access: GitAccess,
5856
ci_build: CIBuild = CIBuild(),
5957
ci_execute_tests: CIExecuteTest = CIExecuteTest(),
@@ -71,9 +69,9 @@ def ci(flavor: str,
7169
flavor_path = (f"flavors/{flavor}",)
7270
test_container_folder = "test_container"
7371
rebuild = BranchConfig.rebuild(branch_name)
74-
needs_to_build = check_if_need_to_build(branch_name, config_file, flavor, git_access)
72+
needs_to_build = check_if_need_to_build(branch_name, build_config, flavor, git_access)
7573
if needs_to_build:
76-
log_path = Path(build_config.DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log"
74+
log_path = Path(DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log"
7775
os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME] = f"{log_path.absolute()}"
7876

7977
ci_build.build(flavor_path=flavor_path,

‎exasol_script_languages_container_ci/lib/config/__init__.py

Whitespace-only changes.

‎exasol_script_languages_container_ci/lib/config/config_data_model.py

Lines changed: 38 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎exasol_script_languages_container_ci/lib/config/data_model_generator.py

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import json
2+
import logging
3+
from pathlib import Path
4+
from tempfile import TemporaryDirectory
5+
6+
from datamodel_code_generator import generate, InputFileType
7+
8+
from exasol_script_languages_container_ci.lib.render_template import render_template
9+
10+
logger = logging.getLogger(__name__)
11+
12+
CONFIG_DATA_MODEL_FILE_NAME = "config_data_model.py"
13+
14+
15+
def config_data_model_default_output_file() -> Path:
16+
return Path(__file__).parent / CONFIG_DATA_MODEL_FILE_NAME
17+
18+
19+
def generate_config_data_model(output_file: Path) -> Path:
20+
schema_dict = json.loads(render_template("config_schema.json"))
21+
schema_json = json.dumps(schema_dict)
22+
with TemporaryDirectory() as directory:
23+
temp_output_file = Path(directory) / CONFIG_DATA_MODEL_FILE_NAME
24+
generate(schema_json, input_file_type=InputFileType.JsonSchema, output=temp_output_file,
25+
class_name="Config", apply_default_values_for_required_fields=True)
26+
with temp_output_file.open("rt") as temp_output_file_handle:
27+
with output_file.open("wt") as output_file_handle:
28+
lines = (line for line in temp_output_file_handle)
29+
lines = filter(lambda line: "# timestamp: " not in line, lines)
30+
for line in lines:
31+
output_file_handle.write(line)
32+
return output_file
33+
34+
35+
def regenerate_config_data_model(output_file: Path):
36+
with TemporaryDirectory() as directory:
37+
temp_output_file = Path(directory) / CONFIG_DATA_MODEL_FILE_NAME
38+
generate_config_data_model(temp_output_file)
39+
with temp_output_file.open("rt") as file:
40+
new_model = file.read()
41+
if output_file.is_file():
42+
with output_file.open("rt") as file:
43+
old_model = file.read()
44+
if new_model != old_model:
45+
logger.warning(f"Regenerating config pydantic model to {output_file}")
46+
with output_file.open("wt") as file:
47+
file.write(new_model)
48+
else:
49+
logger.info(f"Generating config pydantic model to new file {output_file}")
50+
with output_file.open("wt") as file:
51+
file.write(new_model)

‎exasol_script_languages_container_ci/lib/release.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,22 @@
22
import os
33
from pathlib import Path
44

5+
from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY
56
from exasol_integration_test_docker_environment.lib.base import luigi_log_config
6-
from exasol_integration_test_docker_environment.lib.config import build_config
77

88
from exasol_script_languages_container_ci.lib.ci_build import CIBuild
99
from exasol_script_languages_container_ci.lib.ci_push import CIPush
1010
from exasol_script_languages_container_ci.lib.ci_security_scan import CISecurityScan
1111
from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest
12+
from exasol_script_languages_container_ci.lib.config.config_data_model import Config
1213
from exasol_script_languages_container_ci.lib.release_uploader import ReleaseUploader
1314

1415

1516
def release(flavor: str,
1617
docker_user: str,
1718
docker_password: str,
1819
docker_release_repository: str,
19-
config_file: str,
20+
build_config: Config,
2021
source_repo_url: str,
2122
release_id: int,
2223
is_dry_run: bool,
@@ -36,7 +37,7 @@ def release(flavor: str,
3637

3738
flavor_path = (f"flavors/{flavor}",)
3839
test_container_folder = "test_container"
39-
log_path = Path(build_config.DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log"
40+
log_path = Path(DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log"
4041
os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME] = f"{log_path.absolute()}"
4142

4243
ci_build.build(flavor_path=flavor_path, rebuild=True,

‎exasol_script_languages_container_ci/lib/render_template.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import jinja2
2+
3+
4+
def render_template(template: str, **kwargs):
5+
env = jinja2.Environment(loader=jinja2.PackageLoader("exasol_script_languages_container_ci"),
6+
autoescape=jinja2.select_autoescape(), keep_trailing_newline=True)
7+
t = env.get_template(template)
8+
return t.render(**kwargs)

‎exasol_script_languages_container_ci/templates/config_schema.json

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"type": "object",
3+
"required": [
4+
"build",
5+
"release"
6+
],
7+
"additionalProperties": false,
8+
"properties": {
9+
"build": {
10+
"type": "object",
11+
"additionalProperties": false,
12+
"required": [
13+
"ignore",
14+
"base_branch"
15+
],
16+
"properties": {
17+
"ignore": {
18+
"type": "object",
19+
"additionalProperties": false,
20+
"required": [
21+
"paths"
22+
],
23+
"properties": {
24+
"paths": {
25+
"type": "array",
26+
"items": {
27+
"type": "string"
28+
}
29+
}
30+
}
31+
},
32+
"base_branch": {
33+
"type": "string"
34+
}
35+
}
36+
},
37+
"release": {
38+
"type": "object",
39+
"required": [
40+
"timeout_in_minutes"
41+
],
42+
"additionalProperties": false,
43+
"properties": {
44+
"timeout_in_minutes": {
45+
"type": "integer"
46+
}
47+
}
48+
}
49+
}
50+
}

0 commit comments

Comments
 (0)