diff --git a/conftest.py b/conftest.py
index bb5f395..2aa31a1 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,21 +1,36 @@
+import pytest
 from django.conf import global_settings, settings
+from django.test.utils import setup_test_environment
 
 pytest_plugins = ["pytester", "django_utils_lib.testing.pytest_plugin"]
 
-# Set up dummy settings for any tests that rely on Django internals
-# being able to bootstrap against settings.py
-settings.configure(default_settings=global_settings)
-settings.DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.sqlite3",
-        "NAME": ":memory:",
-    }
-}
-settings.INSTALLED_APPS = [
-    "django.contrib.admin",
-    "django.contrib.auth",
-    "django.contrib.contenttypes",
-    "django.contrib.sessions",
-    "django.contrib.messages",
-    "django.contrib.staticfiles",
-]
+
+def configure_django_settings():
+    if settings.configured:
+        return
+    settings.configure(
+        default_settings=global_settings,
+        DATABASES={
+            "default": {
+                "ENGINE": "django.db.backends.sqlite3",
+                "NAME": ":memory:",
+            }
+        },
+        INSTALLED_APPS=[
+            "django.contrib.admin",
+            "django.contrib.auth",
+            "django.contrib.contenttypes",
+            "django.contrib.sessions",
+            "django.contrib.messages",
+            "django.contrib.staticfiles",
+        ],
+    )
+
+
+def pytest_configure():
+    # TODO: This is rather hacky and it would be great to find a cleaner way to
+    # do this. However, the pytest-django fixture we are disabling here is hard
+    # to alter in any other way, especially as it uses session-level autouse
+    pytest.MonkeyPatch().setattr("pytest_django.plugin.django_test_environment", lambda: None)
+    configure_django_settings()
+    setup_test_environment(debug=False)
diff --git a/pyproject.toml b/pyproject.toml
index ee53f3c..963a323 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -43,6 +43,7 @@ testpaths = [
     "django_utils_lib/tests"
 ]
 pythonpath = ". django_utils_lib"
+django_find_project = false
 auto_debug = true
 auto_debug_wait_for_connect = false