Merge branch 'main' into fix-error-1

docker-compose.yml

@@ -18,7 +18,6 @@ services:
         extends: django_env
         command: >
                 bash -c "
-                    python stixify/worker/arango_view_helper.py &&
                     python manage.py collectstatic --no-input &&
                         python manage.py makemigrations &&
                                 python manage.py migrate &&

requirements.txt

@@ -1,152 +1,43 @@
-aiohappyeyeballs==2.4.0
-aiohttp==3.10.5
-aiosignal==1.3.1
 amqp==5.2.0
-annotated-types==0.7.0
-antlr4-python3-runtime==4.9.3
-anyio==4.4.0
 asgiref==3.8.1
 attrs==24.2.0
-base58==2.1.1
-beautifulsoup4==4.12.3
 billiard==4.2.0
-bs4==0.0.2
-cachetools==5.5.0
+boto3==1.35.51
+botocore==1.35.51
 celery==5.4.0
-certifi==2024.7.4
-chardet==5.2.0
-charset-normalizer==3.3.2
 click==8.1.7
 click-didyoumean==0.3.1
 click-plugins==1.1.1
 click-repl==0.3.0
-cssselect==1.2.0
-dataclasses-json==0.6.7
-Deprecated==1.2.14
-dirtyjson==1.0.8
-distro==1.9.0
 Django==5.1
 django-filter==24.3
+django-storages==1.14.4
 djangorestframework==3.15.2
 drf-spectacular==0.27.2
-filelock==3.15.4
-filetype==1.2.0
-frozenlist==1.4.1
-fsspec==2024.6.1
-google-api-core==2.19.1
-google-auth==2.34.0
-google-cloud-vision==3.7.4
-googleapis-common-protos==1.63.2
-greenlet==3.0.3
-grpcio==1.65.5
-grpcio-status==1.65.5
 gunicorn==23.0.0
-h11==0.14.0
-httpcore==1.0.5
-httpx==0.27.0
+hyperlink==21.0.0
 idna==3.7
-importlib_metadata==8.4.0
 inflection==0.5.1
-jiter==0.5.0
-joblib==1.4.2
-jsonpatch==1.33
-jsonpointer==3.0.0
+jmespath==1.0.1
 jsonschema==4.23.0
 jsonschema-specifications==2023.12.1
 kombu==5.4.0
-langchain-core==0.2.34
-langchain-openai==0.1.22
-langsmith==0.1.101
-llama-cloud==0.0.14
-llama-index==0.10.67.post1
-llama-index-agent-openai==0.2.9
-llama-index-cli==0.1.13
-llama-index-core==0.10.67
-llama-index-embeddings-openai==0.1.11
-llama-index-indices-managed-llama-cloud==0.2.7
-llama-index-legacy==0.9.48.post3
-llama-index-llms-openai==0.1.30
-llama-index-multi-modal-llms-openai==0.1.9
-llama-index-program-openai==0.1.7
-llama-index-question-gen-openai==0.1.3
-llama-index-readers-file==0.1.33
-llama-index-readers-llama-parse==0.1.6
-llama-parse==0.4.9
-lxml==5.3.0
-lxml_html_clean==0.2.0
-markdownify==0.13.1
-marshmallow==3.22.0
-multidict==6.0.5
-mypy-extensions==1.0.0
-nest-asyncio==1.6.0
-networkx==3.3
-nltk==3.9.1
-numpy==1.26.4
-openai==1.42.0
-orjson==3.10.7
+mistune==3.0.2
 packaging==24.1
-pandas==2.2.2
-pathvalidate==3.2.0
-phonenumbers==8.13.43
-pillow==10.4.0
 prompt_toolkit==3.0.47
-proto-plus==1.24.0
-protobuf==5.27.3
-psycopg2==2.9.9
-pyasn1==0.6.0
-pyasn1_modules==0.4.0
-pycountry==24.6.1
-pydantic==2.8.2
-pydantic_core==2.20.1
-PyJWT==2.9.0
-pypdf==4.3.1
-python-arango==8.1.0
+psycopg2==2.9.10
 python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
-pytz==2024.1
 PyYAML==6.0.2
-readability-lxml==0.8.1
 redis==5.0.8
 referencing==0.35.1
-regex==2024.7.24
-requests==2.32.3
-requests-file==2.1.0
-requests-toolbelt==1.0.0
 rpds-py==0.20.0
-rsa==4.9
-rstr==3.2.2
-schwifty==2024.8.1
+s3transfer==0.10.3
 setuptools==73.0.1
-simplejson==3.19.3
 six==1.16.0
-sniffio==1.3.1
-soupsieve==2.6
-SQLAlchemy==2.0.32
 sqlparse==0.5.1
-stix2==3.0.1
-stix2-patterns==2.0.0
-striprtf==0.0.26
-tabulate==0.9.0
-tenacity==8.5.0
-tiktoken==0.7.0
-tld==0.13
-tldextract==5.1.2
-tqdm==4.66.5
-typing-inspect==0.9.0
-typing_extensions==4.12.2
 tzdata==2024.1
 uritemplate==4.1.1
 urllib3==2.2.2
-validators==0.33.0
 vine==5.1.0
 wcwidth==0.2.13
-wrapt==1.16.0
-yarl==1.9.4
-zipp==3.20.0
-hyperlink==21.0.0
-mistune==3.0.2
-django-storages[s3]==1.14.4
-file2txt @ https://github.com/muchdogesec/file2txt/archive/main.zip
-stix2arango @ https://github.com/muchdogesec/stix2arango/archive/main.zip
-stix2extensions @ https://github.com/muchdogesec/stix2extensions/archive/main.zip
-txt2stix @ https://github.com/muchdogesec/txt2stix/archive/main.zip
+dogesec_commons[stixifier] @ https://github.com/muchdogesec/dogesec_commons/archive/main.zip

stixify/settings.py

@@ -50,6 +50,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'drf_spectacular',
+    'dogesec_commons.objects.app.ArangoObjectsViewApp',
     'django.contrib.postgres',
     'stixify.web',
 ]
@@ -200,6 +201,7 @@
 STIX_NAMESPACE = uuid.UUID('e92c648d-03eb-59a5-a318-9a36e6f8057c')
 
 TXT2STIX_INCLUDE_URL = "https://github.com/muchdogesec/txt2stix/blob/main/includes/"
+APP_LABEL = "web"
 
 MAXIMUM_PAGE_SIZE = int(os.getenv("MAX_PAGE_SIZE", 50))
 DEFAULT_PAGE_SIZE = int(os.getenv("DEFAULT_PAGE_SIZE", 50))
@@ -234,6 +236,7 @@
 
 ARANGODB_DATABASE   = "stixify"
 VIEW_NAME = "stixify_view"
+VIEW_NAME = "stixify_view2"
 ARANGODB_USERNAME   = os.getenv('ARANGODB_USERNAME')
 ARANGODB_PASSWORD   = os.getenv('ARANGODB_PASSWORD')
 ARANGODB_HOST_URL   = os.getenv("ARANGODB_HOST_URL")

stixify/urls.py

@@ -18,30 +18,31 @@
 from django.contrib import admin
 from django.urls import path, include
 from rest_framework import routers
-from .web.more_views import txt2stix, profile
-from .web.arango_based_views import arango_views
+from dogesec_commons.objects import views as arango_views
+from dogesec_commons.stixifier.views import ProfileView, ExtractorsView, WhitelistsView, AliasesView
 from .web.views import FileView, DossierView, JobView, ReportView
-from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView, SpectacularSwaggerView
+from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView
 from django.conf.urls.static import static
 
 API_VERSION = "v1"
 
 router = routers.SimpleRouter(use_regex_path=False)
 # profile view
-router.register('profiles', profile.ProfileView, 'profile-view')
+router.register('profiles', ProfileView, 'profile-view')
 router.register('files', FileView, 'file-view')
 router.register('dossiers', DossierView, 'dossier-view')
 router.register('jobs', JobView, 'job-view')
 router.register('reports', ReportView, 'report-view')
 # objects
+router.register("objects", arango_views.ObjectsWithReportsView, "object-view-orig")
 router.register('objects/smos', arango_views.SMOView, "object-view-smo")
 router.register('objects/scos', arango_views.SCOView, "object-view-sco")
 router.register('objects/sros', arango_views.SROView, "object-view-sro")
 router.register('objects/sdos', arango_views.SDOView, "object-view-sdo")
 # txt2stix views
-router.register('extractors', txt2stix.ExtractorsView, "extractors-view")
-router.register('whitelists', txt2stix.WhitelistsView, "whitelists-view")
-router.register('aliases', txt2stix.AliasesView, "aliases-view")
+router.register('extractors', ExtractorsView, "extractors-view")
+router.register('whitelists', WhitelistsView, "whitelists-view")
+router.register('aliases', AliasesView, "aliases-view")
 
 urlpatterns = [
     path(f'api/{API_VERSION}/', include(router.urls)),