Skip to content

Commit b70f10c

Browse files
mkanoormkanoor
committed
feat: support PG Notify for event streams using credentials
Create a Postgres credential that can store certificates and keys or userid/password at server startup and attach it to Activations that use EventStream. This allows us to test mTLS for Postgres, a sample pg_hba file is attached to this PR. To test this you need to create certificates and keys for * Postgres Server called (server.crt and server.key) * EDA Server called (client.crt and client.key) * You also need to create the CA certificate These files have to be present in tools/docker/postgres_ssl_config/certs The docker-compose file tools/docker/docker-compose-mac-draft.yml can be used to test mTLS The docker-compose file tools/docker/docker-compose-mac.yml can be used to test userid/password auth
1 parent 7a38086 commit b70f10c

File tree

8 files changed

+532
-26
lines changed

8 files changed

+532
-26
lines changed

‎src/aap_eda/api/serializers/activation.py

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -65,24 +65,25 @@
6565
"rulebook_hash",
6666
]
6767

68+
PG_NOTIFY_DSN = (
69+
"host={{postgres_db_host}} port={{postgres_db_port}} "
70+
"dbname={{postgres_db_name}} user={{postgres_db_user}} "
71+
"password={{postgres_db_password}} sslmode={{postgres_sslmode}} "
72+
"sslcert={{eda.filename.postgres_sslcert|default(None)}} "
73+
"sslkey={{eda.filename.postgres_sslkey|default(None)}} "
74+
"sslpassword={{postgres_sslpassword|default(None)}} "
75+
"sslrootcert={{eda.filename.postgres_sslrootcert|default(None)}}"
76+
)
77+
6878

6979
@dataclass
7080
class VaultData:
7181
password: str = secrets.token_urlsafe()
7282
password_used: bool = False
7383

7484

75-
def _update_event_stream_source(
76-
validated_data: dict, vault_data: VaultData
77-
) -> str:
85+
def _update_event_stream_source(validated_data: dict) -> str:
7886
try:
79-
vault_data.password_used = True
80-
encrypted_dsn = encrypt_string(
81-
password=vault_data.password,
82-
plaintext=settings.PG_NOTIFY_DSN,
83-
vault_id=EDA_SERVER_VAULT_LABEL,
84-
)
85-
8687
source_mappings = yaml.safe_load(validated_data["source_mappings"])
8788
sources_info = {}
8889
for source_map in source_mappings:
@@ -91,7 +92,7 @@ def _update_event_stream_source(
9192

9293
sources_info[obj.name] = {
9394
"ansible.eda.pg_listener": {
94-
"dsn": encrypted_dsn,
95+
"dsn": PG_NOTIFY_DSN,
9596
"channels": [obj.channel_name],
9697
},
9798
}
@@ -329,7 +330,7 @@ def to_representation(self, activation):
329330
)
330331
eda_credentials = [
331332
EdaCredentialSerializer(credential).data
332-
for credential in activation.eda_credentials.all()
333+
for credential in activation.eda_credentials.filter(managed=False)
333334
]
334335
extra_var = (
335336
replace_vault_data(activation.extra_var)
@@ -471,8 +472,14 @@ def create(self, validated_data):
471472

472473
if validated_data.get("source_mappings", []):
473474
validated_data["rulebook_rulesets"] = _update_event_stream_source(
474-
validated_data, vault_data
475+
validated_data
475476
)
477+
eda_credentials = validated_data.get("eda_credentials", [])
478+
postgres_cred = models.EdaCredential.objects.filter(
479+
name=settings.DEFAULT_SYSTEM_PG_NOTIFY_CREDENTIAL_NAME
480+
).first()
481+
eda_credentials.append(postgres_cred.id)
482+
validated_data["eda_credentials"] = eda_credentials
476483

477484
vault = _get_vault_credential_type()
478485

@@ -642,7 +649,7 @@ def to_representation(self, activation):
642649
)
643650
eda_credentials = [
644651
EdaCredentialSerializer(credential).data
645-
for credential in activation.eda_credentials.all()
652+
for credential in activation.eda_credentials.filter(managed=False)
646653
]
647654
extra_var = (
648655
replace_vault_data(activation.extra_var)

‎src/aap_eda/core/enums.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ class DefaultCredentialType(DjangoStrEnum):
9898
SOURCE_CONTROL = "Source Control"
9999
AAP = "Red Hat Ansible Automation Platform"
100100
GPG = "GPG Public Key"
101+
POSTGRES = "Postgres"
101102

102103

103104
# TODO: rename to "RulebookProcessStatus" or "ParentProcessStatus"

‎src/aap_eda/core/management/commands/create_initial_data.py

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
from ansible_base.rbac import permission_registry
1919
from ansible_base.rbac.models import DABPermission, RoleDefinition
20+
from django.conf import settings
2021
from django.contrib.contenttypes.models import ContentType
2122
from django.core.exceptions import ImproperlyConfigured
2223
from django.core.management import BaseCommand
@@ -841,6 +842,97 @@
841842
"required": ["auth_type", "username", "password", "http_header_key"],
842843
}
843844

845+
POSTGRES_CREDENTIAL_INPUTS = {
846+
"fields": [
847+
{
848+
"id": "postgres_db_host",
849+
"label": "Postgres DB Host",
850+
"help_text": "Postgres DB Server",
851+
},
852+
{
853+
"id": "postgres_db_port",
854+
"label": "Postgres DB Port",
855+
"help_text": "Postgres DB Port",
856+
"default": "5432",
857+
},
858+
{
859+
"id": "postgres_db_name",
860+
"label": "Postgres DB Name",
861+
"help_text": "Postgres Database name",
862+
},
863+
{
864+
"id": "postgres_db_user",
865+
"label": "Postgres DB User",
866+
"help_text": "Postgres Database user",
867+
},
868+
{
869+
"id": "postgres_db_password",
870+
"label": "Postgres DB Password",
871+
"help_text": "Postgres Database password",
872+
"secret": True,
873+
},
874+
{
875+
"id": "postgres_sslmode",
876+
"label": "Postgres SSL Mode",
877+
"help_text": "Postgres SSL Mode",
878+
"choices": [
879+
"disable",
880+
"allow",
881+
"prefer",
882+
"require",
883+
"verify-ca",
884+
"verify-full",
885+
],
886+
"default": "prefer",
887+
},
888+
{
889+
"id": "postgres_sslcert",
890+
"label": "Postgres SSL Certificate",
891+
"help_text": "Postgres SSL Certificate",
892+
"multiline": True,
893+
"default": "",
894+
},
895+
{
896+
"id": "postgres_sslkey",
897+
"label": "Postgres SSL Key",
898+
"help_text": "Postgres SSL Key",
899+
"multiline": True,
900+
"secret": True,
901+
"default": "",
902+
},
903+
{
904+
"id": "postgres_sslpassword",
905+
"label": "Postgres SSL Password",
906+
"help_text": "Postgres SSL Password for key",
907+
"secret": True,
908+
"default": "",
909+
},
910+
{
911+
"id": "postgres_sslrootcert",
912+
"label": "Postgres SSL Root Certificate",
913+
"help_text": "Postgres SSL Root Certificate",
914+
"multiline": True,
915+
"default": "",
916+
},
917+
]
918+
}
919+
920+
POSTGRES_CREDENTIAL_INJECTORS = {
921+
"extra_vars": {
922+
"postgres_db_host": "{{ postgres_db_host }}",
923+
"postgres_db_port": "{{ postgres_db_port }}",
924+
"postgres_db_name": "{{ postgres_db_name }}",
925+
"postgres_db_user": "{{ postgres_db_user }}",
926+
"postgres_db_password": "{{ postgres_db_password }}",
927+
"postgres_sslpassword": "{{ postgres_sslpassword | default(None) }}",
928+
"postgres_sslmode": "{{ postgres_sslmode }}",
929+
},
930+
"file": {
931+
"template.postgres_sslcert": "{{ postgres_sslcert }}",
932+
"template.postgres_sslrootcert": "{{ postgres_sslrootcert }}",
933+
"template.postgres_sslkey": "{{ postgres_sslkey }}",
934+
},
935+
}
844936
CREDENTIAL_TYPES = [
845937
{
846938
"name": enums.DefaultCredentialType.SOURCE_CONTROL,
@@ -1014,6 +1106,14 @@
10141106
"the Basic authentication."
10151107
),
10161108
},
1109+
{
1110+
"name": enums.DefaultCredentialType.POSTGRES,
1111+
"kind": "cloud",
1112+
"namespace": "eda",
1113+
"inputs": POSTGRES_CREDENTIAL_INPUTS,
1114+
"injectors": POSTGRES_CREDENTIAL_INJECTORS,
1115+
"managed": True,
1116+
},
10171117
]
10181118

10191119

@@ -1046,6 +1146,7 @@ class Command(BaseCommand):
10461146
@transaction.atomic
10471147
def handle(self, *args, **options):
10481148
self._preload_credential_types()
1149+
self._update_postgres_credentials()
10491150
self._copy_registry_credentials()
10501151
self._copy_scm_credentials()
10511152
self._create_org_roles()
@@ -1167,6 +1268,48 @@ def _copy_scm_credentials(self):
11671268
"Control eda-credentials"
11681269
)
11691270

1271+
def _update_postgres_credentials(self):
1272+
cred_type = models.CredentialType.objects.filter(
1273+
name=enums.DefaultCredentialType.POSTGRES
1274+
).first()
1275+
inputs = {
1276+
"postgres_db_host": settings.ACTIVATION_DB_HOST,
1277+
"postgres_db_port": settings.DATABASES["default"]["PORT"],
1278+
"postgres_db_name": settings.DATABASES["default"]["NAME"],
1279+
"postgres_db_user": settings.DATABASES["default"]["USER"],
1280+
"postgres_db_password": settings.DATABASES["default"]["PASSWORD"],
1281+
"postgres_sslmode": settings.PGSSLMODE,
1282+
"postgres_sslcert": "",
1283+
"postgres_sslkey": "",
1284+
"postgres_sslrootcert": "",
1285+
}
1286+
1287+
if settings.PGSSLCERT:
1288+
inputs["postgres_sslcert"] = self._read_file(settings.PGSSLCERT)
1289+
1290+
if settings.PGSSLKEY:
1291+
inputs["postgres_sslkey"] = self._read_file(settings.PGSSLKEY)
1292+
1293+
if settings.PGSSLROOTCERT:
1294+
inputs["postgres_sslrootcert"] = self._read_file(
1295+
settings.PGSSLROOTCERT
1296+
)
1297+
1298+
models.EdaCredential.objects.update_or_create(
1299+
name=settings.DEFAULT_SYSTEM_PG_NOTIFY_CREDENTIAL_NAME,
1300+
defaults={
1301+
"description": "Default PG Notify Credentials",
1302+
"managed": True,
1303+
"credential_type": cred_type,
1304+
"inputs": inputs_to_store(inputs),
1305+
"organization": get_default_organization(),
1306+
},
1307+
)
1308+
1309+
def _read_file(self, name):
1310+
with open(name) as f:
1311+
return f.read()
1312+
11701313
def _create_org_roles(self):
11711314
org_ct = ContentType.objects.get(model="organization")
11721315
created = updated = 0

‎src/aap_eda/settings/default.py

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -781,15 +781,6 @@ def get_rulebook_process_log_level() -> RulebookProcessLogLevel:
781781
"ACTIVATION_DB_HOST", "host.containers.internal"
782782
)
783783

784-
_DEFAULT_PG_NOTIFY_DSN = (
785-
f"host={ACTIVATION_DB_HOST} "
786-
f"port={DATABASES['default']['PORT']} "
787-
f"dbname={DATABASES['default']['NAME']} "
788-
f"user={DATABASES['default']['USER']} "
789-
f"password={DATABASES['default']['PASSWORD']}"
790-
)
791-
792-
PG_NOTIFY_DSN = settings.get("PG_NOTIFY_DSN", _DEFAULT_PG_NOTIFY_DSN)
793784
PG_NOTIFY_TEMPLATE_RULEBOOK = settings.get("PG_NOTIFY_TEMPLATE_RULEBOOK", None)
794785

795786
SAFE_PLUGINS_FOR_PORT_FORWARD = settings.get(
@@ -806,7 +797,11 @@ def get_rulebook_process_log_level() -> RulebookProcessLogLevel:
806797
f"port={DATABASES['default']['PORT']} "
807798
f"dbname={DATABASES['default']['NAME']} "
808799
f"user={DATABASES['default']['USER']} "
809-
f"password={DATABASES['default']['PASSWORD']}"
800+
f"password={DATABASES['default']['PASSWORD']} "
801+
f"sslmode={settings.get('PGSSLMODE', default='prefer')} "
802+
f"sslcert={settings.get('PGSSLCERT', default='None')} "
803+
f"sslkey={settings.get('PGSSLKEY', default='None')} "
804+
f"sslrootcert={settings.get('PGSSLROOTCERT', default='None')}"
810805
)
811806
PG_NOTIFY_DSN_SERVER = settings.get(
812807
"PG_NOTIFY_DSN_SERVER", _DEFAULT_PG_NOTIFY_DSN_SERVER
@@ -823,3 +818,9 @@ def get_rulebook_process_log_level() -> RulebookProcessLogLevel:
823818
MAX_PG_NOTIFY_MESSAGE_SIZE = int(
824819
settings.get("MAX_PG_NOTIFY_MESSAGE_SIZE", 6144)
825820
)
821+
822+
DEFAULT_SYSTEM_PG_NOTIFY_CREDENTIAL_NAME = "_DEFAULT_EDA_PG_NOTIFY_CREDS"
823+
PGSSLMODE = settings.get("PGSSLMODE", default="prefer")
824+
PGSSLCERT = settings.get("PGSSLCERT", default="")
825+
PGSSLKEY = settings.get("PGSSLKEY", default="")
826+
PGSSLROOTCERT = settings.get("PGSSLROOTCERT", default="")

‎tests/integration/api/test_activation_with_event_stream.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,8 +282,11 @@ def create_activation(fks: dict):
282282

283283
@pytest.mark.django_db
284284
def test_create_activation_with_event_stream(
285-
admin_client: APIClient, preseed_credential_types
285+
admin_client: APIClient,
286+
preseed_credential_types,
287+
create_initial_data_command,
286288
):
289+
create_initial_data_command.handle()
287290
fks = create_activation_related_data(["demo"])
288291
test_activation = TEST_ACTIVATION.copy()
289292
test_activation["decision_environment_id"] = fks["decision_environment_id"]
@@ -674,12 +677,14 @@ def test_create_activation_with_duplicate_event_stream_name(
674677
def test_bad_src_activation_with_event_stream(
675678
admin_client: APIClient,
676679
preseed_credential_types,
680+
create_initial_data_command,
677681
source_tuples,
678682
rulesets,
679683
status_code,
680684
message,
681685
error_key,
682686
):
687+
create_initial_data_command.handle()
683688
names = [event_stream_name for _, event_stream_name in source_tuples]
684689
fks = create_activation_related_data(names, True, rulesets)
685690
test_activation = TEST_ACTIVATION.copy()

0 commit comments

Comments
 (0)