Skip to content

Commit

Permalink
perf: query set optimization (#1491)
Browse files Browse the repository at this point in the history 
* feat: fetch options in AutocompleteSelect instead of load function

* style: add spinner during AutocompleteSelect loading

* chore: format

* chore: remove checkConstraints in page.svelte

* feat: check constraints inside AutocompleteSelect

* feat: globalize AutocompleteSelect fetching

* chore: format

* chore: cleaning useles foreignKey fetches and old missing constraints logic

* fix: typo in ThreatForm optionsEndpoint

* fix: use optionsLabel str for risk assessment field in scenario form

* feat: add optionsLabelField auto for assets threats and feared events

* feat: sort options to put suggestions first

* chore: add missing indent

* chore: remove unused getOptions

* chore: give default values to avoid warnings in frameworks detailed view

* feat: add entity assessment folder extra label for perimeter field

* chore: nitpicking

* feat: add detailed url parameters in AutocompleteSelect fetchOptions

* feat: exclude self from options for parent assets and folder

* feat: refresh AutocompleteSelect when applied control/evidence is created inside requirement assessment

* chore: format front

* chore: remove useless foreignKeys fetch in risk assessment page

* chore: remove the latest foreignKeys fetches on server side

* Improve performance

- optimize get_accessible_object_ids by first calculating the folders that match, and only then retrieving (optimally) the objects from these folders. This should be way more scalable.
- use scope_folder_id parameter to scope the retrieved objects

* prettier/ruff

* Fix regression

special case of folder for get_accessible_object_ids

* fix published objects not visible

---------

Co-authored-by: Mohamed-Hacene <mohamedhacene.b@gmail.com>
  • Loading branch information
@eric-intuitem @Mohamed-Hacene
eric-intuitem and Mohamed-Hacene authored Feb 12, 2025
1 parent 6b42979 commit 87ffbea
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 70 deletions.
10 changes: 9 additions & 1 deletion backend/core/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
from django.middleware import csrf
from django.template.loader import render_to_string
from django.utils.functional import Promise
from django.shortcuts import get_object_or_404
from django_filters.rest_framework import DjangoFilterBackend
from iam.models import Folder, RoleAssignment, UserGroup
from rest_framework import filters, permissions, status, viewsets
Expand Down Expand Up @@ -139,6 +140,7 @@ class BaseModelViewSet(viewsets.ModelViewSet):
serializers_module = "core.serializers"

def get_queryset(self):
"""the scope_folder_id query_param allows scoping the objects to retrieve"""
if not self.model:
return None
object_ids_view = None
Expand All @@ -153,8 +155,14 @@ def get_queryset(self):
if RoleAssignment.is_object_readable(self.request.user, self.model, id):
object_ids_view = [id]
if not object_ids_view:
scope_folder_id = self.request.query_params.get("scope_folder_id")
scope_folder = (
get_object_or_404(Folder, id=scope_folder_id)
if scope_folder_id
else Folder.get_root_folder()
)
object_ids_view = RoleAssignment.get_accessible_object_ids(
Folder.get_root_folder(), self.request.user, self.model
scope_folder, self.request.user, self.model
)[0]
queryset = self.model.objects.filter(id__in=object_ids_view)
return queryset
Expand Down
134 changes: 65 additions & 69 deletions backend/iam/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -680,83 +680,79 @@ def get_accessible_object_ids(
Also retrieve published objects in view
"""
class_name = object_type.__name__.lower()
permissions = [
Permission.objects.get(codename="view_" + class_name),
Permission.objects.get(codename="change_" + class_name),
Permission.objects.get(codename="delete_" + class_name),
]
permission_view = Permission.objects.get(codename="view_" + class_name)
permission_change = Permission.objects.get(codename="change_" + class_name)
permission_delete = Permission.objects.get(codename="delete_" + class_name)
permissions = set([permission_view, permission_change, permission_delete])
result_view = set()
result_change = set()
result_delete = set()

folders_with_local_view = set()
permissions_per_object_id = defaultdict(set)
ref_permission = Permission.objects.get(codename="view_folder")
all_objects = (
object_type.objects.select_related("folder")
if hasattr(object_type, "folder")
else object_type.objects.all()
)
folder_for_object = {x: Folder.get_folder(x) for x in all_objects}
perimeter = set()
perimeter.add(folder)
perimeter.update(folder.get_sub_folders())
for ra in [
x
for x in RoleAssignment.get_role_assignments(user)
if ref_permission in x.role.permissions.all()
]:
ra_permissions = ra.role.permissions.all()
for my_folder in perimeter & set(ra.perimeter_folders.all()):
target_folders = (
[my_folder, *my_folder.get_sub_folders()]
if ra.is_recursive
else [my_folder]
perimeter = {folder} | set(folder.get_sub_folders())
# Process role assignments
role_assignments = [
ra
for ra in RoleAssignment.get_role_assignments(user)
if ref_permission in ra.role.permissions.all()
]
result_folders = defaultdict(set)
for ra in role_assignments:
ra_permissions = set(ra.role.permissions.all())
ra_perimeter = set(ra.perimeter_folders.all())
if ra.is_recursive:
ra_perimeter.update(
*[folder.get_sub_folders() for folder in ra_perimeter]
)
target_folders = perimeter & ra_perimeter
for p in permissions & ra_permissions:
for f in target_folders:
result_folders[f].add(p)
for f in result_folders:
if hasattr(object_type, "folder"):
objects_ids = object_type.objects.filter(folder=f).values_list(
"id", flat=True
)
elif hasattr(object_type, "risk_assessment"):
objects_ids = object_type.objects.filter(
risk_assessment__folder=f
).values_list("id", flat=True)
elif hasattr(object_type, "entity"):
objects_ids = object_type.objects.filter(entity__folder=f).values_list(
"id", flat=True
)
for p in [p for p in permissions if p in ra_permissions]:
if p == permissions[0]:
folders_with_local_view.add(my_folder)
for object in [
x for x in all_objects if folder_for_object[x] in target_folders
]:
# builtins objects cannot be edited or deleted
if not (
hasattr(object, "builtin")
and object.builtin
and p != permissions[0]
):
permissions_per_object_id[object.id].add(p)

if hasattr(object_type, "is_published"):
elif hasattr(object_type, "provider_entity"):
objects_ids = object_type.objects.filter(
provider_entity__folder=f
).values_list("id", flat=True)
elif hasattr(object_type, "parent_folder"):
objects_ids = [f.id]
else:
raise NotImplementedError("type not supported")
if permission_view in result_folders[f]:
result_view.update(objects_ids)
if permission_change in result_folders[f]:
result_change.update(objects_ids)
if permission_delete in result_folders[f]:
result_delete.update(objects_ids)

if hasattr(object_type, "is_published") and hasattr(object_type, "folder"):
# we assume only objects with a folder attribute are worth publishing
folders_with_local_view = [
f for f in result_folders if permission_view in result_folders[f]
]
for my_folder in folders_with_local_view:
if my_folder.content_type != Folder.ContentType.ENCLAVE:
target_folders = []
my_folder2 = my_folder
my_folder2 = my_folder.parent_folder
while my_folder2:
if my_folder2 != my_folder:
target_folders.append(my_folder2)
result_view.update(
object_type.objects.filter(
folder=my_folder2, is_published=True
).values_list("id", flat=True)
)
my_folder2 = my_folder2.parent_folder
for object in [
x
for x in all_objects
if folder_for_object[x] in target_folders and x.is_published
]:
permissions_per_object_id[object.id].add(permissions[0])

return (
[
x
for x in permissions_per_object_id
if permissions[0] in permissions_per_object_id[x]
],
[
x
for x in permissions_per_object_id
if permissions[1] in permissions_per_object_id[x]
],
[
x
for x in permissions_per_object_id
if permissions[2] in permissions_per_object_id[x]
],
)
return (list(result_view), list(result_change), list(result_delete))

def is_user_assigned(self, user) -> bool:
"""Determines if a user is assigned to the role assignment"""
Expand Down
15 changes: 15 additions & 0 deletions frontend/src/routes/(app)/(internal)/risk-scenarios/[id=uuid]/edit/+page.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,13 +172,19 @@
optionsLabelField="auto"
optionsExtraFields={[['folder', 'str']]}
field="assets"
optionsDetailedUrlParameters={[
['scope_folder_id', $page.data.scenario.perimeter.folder.id]
]}
label={m.assets()}
helpText={m.riskScenarioAssetHelpText()}
/>
<AutocompleteSelect
form={_form}
multiple
optionsEndpoint="threats"
optionsDetailedUrlParameters={[
['scope_folder_id', $page.data.scenario.perimeter.folder.id]
]}
optionsExtraFields={[['folder', 'str']]}
optionsLabelField="auto"
field="threats"
Expand All @@ -188,6 +194,9 @@
multiple
form={_form}
optionsEndpoint="vulnerabilities"
optionsDetailedUrlParameters={[
['scope_folder_id', $page.data.scenario.perimeter.folder.id]
]}
optionsExtraFields={[['folder', 'str']]}
field="vulnerabilities"
label={m.vulnerabilities()}
Expand All @@ -207,6 +216,9 @@
form={_form}
optionsEndpoint="applied-controls"
optionsExtraFields={[['folder', 'str']]}
optionsDetailedUrlParameters={[
['scope_folder_id', $page.data.scenario.perimeter.folder.id]
]}
field="existing_applied_controls"
label={m.existingControls()}
helpText={m.existingControlsHelper()}
Expand Down Expand Up @@ -280,6 +292,9 @@
form={_form}
optionsEndpoint="applied-controls"
optionsExtraFields={[['folder', 'str']]}
optionsDetailedUrlParameters={[
['scope_folder_id', $page.data.scenario.perimeter.folder.id]
]}
field="applied_controls"
label={m.extraAppliedControls()}
helpText={m.extraControlsHelper()}
Expand Down
6 changes: 6 additions & 0 deletions frontend/src/routes/(app)/(third-party)/requirement-assessments/[id=uuid]/edit/+page.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,9 @@
multiple
{form}
optionsEndpoint="applied-controls"
optionsDetailedUrlParameters={[
['scope_folder_id', $page.data.requirementAssessment.folder.id]
]}
optionsExtraFields={[['folder', 'str']]}
field="applied_controls"
/>
Expand Down Expand Up @@ -442,6 +445,9 @@
{form}
optionsEndpoint="evidences"
optionsExtraFields={[['folder', 'str']]}
optionsDetailedUrlParameters={[
['scope_folder_id', $page.data.requirementAssessment.folder.id]
]}
field="evidences"
/>
{/key}
Expand Down

0 comments on commit 87ffbea

Please sign in to comment.