From 40fcb79685634dd179c644854bca00906269ba58 Mon Sep 17 00:00:00 2001 From: Scott Uhlrich <43877159+suhlrich@users.noreply.github.com> Date: Tue, 3 Jun 2025 09:22:25 -0600 Subject: [PATCH 01/11] revert the 1-camera OK on n_cameras --- mcserver/views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mcserver/views.py b/mcserver/views.py index 52498cb..656c4d5 100644 --- a/mcserver/views.py +++ b/mcserver/views.py @@ -233,11 +233,11 @@ def get_n_calibrated_cameras(self, request, pk): # Nothing in calibration - assume mono # In future, we want to set a metadata parameter for isMono - this is a hack to allow us to collect data - if 'sessionWithCalibration' not in (session.meta or {}) and session.trial_set.filter(name="calibration").count() == 0: - return Response({ - 'error_message': error_message, - 'data': 1 - }) + # if 'sessionWithCalibration' not in (session.meta or {}) and session.trial_set.filter(name="calibration").count() == 0: + # return Response({ + # 'error_message': error_message, + # 'data': 1 + # }) # Check if there is a calibration trial. If not, it must be in a parent session. loop_counter = 0 From 3998e99e2ae540b1b61914e35a53e1751821ac77 Mon Sep 17 00:00:00 2001 From: carmichaelong Date: Thu, 5 Jun 2025 14:39:11 -0700 Subject: [PATCH 02/11] comment out one video checks --- mcserver/views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mcserver/views.py b/mcserver/views.py index 14eed4f..f9a2413 100644 --- a/mcserver/views.py +++ b/mcserver/views.py @@ -1475,6 +1475,8 @@ def dequeue(self, request): not_uploaded = Video.objects.filter(video='', updated_at__gte=datetime.now() + timedelta(minutes=-15)).values_list("trial__id", flat=True) + # Mono automatic check: comment out for now for performance + ''' # Trials that have only one video only_one_video = Trial.objects.annotate(video_count=Count('video')).filter(video_count=1).values_list("id", flat=True) @@ -1484,6 +1486,9 @@ def dequeue(self, request): else: # Exclude trials with not-uploaded videos and only 1 video uploaded_trials = Trial.objects.exclude(id__in=not_uploaded).exclude(id__in=only_one_video) + ''' + + uploaded_trials = Trial.objects.exclude(id__in=not_uploaded) if workerType != 'dynamic': # Priority for 'calibration' and 'neutral' From 84ec16a9167a5fdec23e1a6a21b6e2c88468d2d9 Mon Sep 17 00:00:00 2001 From: carmichaelong Date: Thu, 5 Jun 2025 14:52:35 -0700 Subject: [PATCH 03/11] let dev deploy from other branches via github. limit main redeployment only from main --- .github/workflows/ecr-dev.yml | 1 + .github/workflows/ecr.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/ecr-dev.yml b/.github/workflows/ecr-dev.yml index 3bad708..08824d3 100644 --- a/.github/workflows/ecr-dev.yml +++ b/.github/workflows/ecr-dev.yml @@ -28,6 +28,7 @@ on: push: branches: - dev + workflow_dispatch: name: DEV ECS build & deployment diff --git a/.github/workflows/ecr.yml b/.github/workflows/ecr.yml index a494487..92fcf94 100644 --- a/.github/workflows/ecr.yml +++ b/.github/workflows/ecr.yml @@ -29,6 +29,8 @@ on: branches: - main workflow_dispatch: + branches: + - main name: PRODUCTION ECS build & deployment From 8b372c96440062fc5a07f03f812461710674b1dc Mon Sep 17 00:00:00 2001 From: Seeeeeyo Date: Fri, 6 Jun 2025 10:31:27 -0600 Subject: [PATCH 04/11] Added isMono field in DB + logic behind it for dequeue and n cameras --- mcserver/models.py | 2 ++ mcserver/serializers.py | 5 +++-- mcserver/views.py | 35 ++++++++++++++++++++++++++++------- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/mcserver/models.py b/mcserver/models.py index 08b2c46..f7bcffb 100644 --- a/mcserver/models.py +++ b/mcserver/models.py @@ -78,6 +78,8 @@ class Session(models.Model): trashed = models.BooleanField(default=False) trashed_at = models.DateTimeField(blank=True, null=True) + + isMono = models.BooleanField(default=False) class Meta: ordering = ['-created_at'] diff --git a/mcserver/serializers.py b/mcserver/serializers.py index e2ef10d..437cfa7 100644 --- a/mcserver/serializers.py +++ b/mcserver/serializers.py @@ -208,7 +208,7 @@ class Meta: fields = [ 'id', 'user', 'public', 'name', 'sessionName', 'qrcode', 'meta', 'trials', 'server', - 'subject', + 'subject', 'isMono', 'created_at', 'updated_at', 'trashed', 'trashed_at', 'trials_count', 'trashed_trials_count', ] @@ -226,7 +226,7 @@ class Meta: fields = [ 'id', 'user', 'public', 'name', 'sessionName', 'qrcode', 'meta', 'trials', 'server', - 'subject', + 'subject', 'isMono', 'created_at', 'updated_at', 'trashed', 'trashed_at', 'trials_count', 'trashed_trials_count', ] @@ -431,3 +431,4 @@ class AnalysisDashboardSerializer(serializers.ModelSerializer): class Meta: model = AnalysisDashboard fields = ('id', 'title', 'function', 'template', 'layout') + diff --git a/mcserver/views.py b/mcserver/views.py index 7384a89..28f5c23 100644 --- a/mcserver/views.py +++ b/mcserver/views.py @@ -231,6 +231,12 @@ def get_n_calibrated_cameras(self, request, pk): calibration_trials = session.trial_set.filter(name="calibration") last_calibration_trial_num_videos = 0 + if session.isMono: + return Response({ + 'error_message': error_message, + 'data': 1 + }) + # Check if there is a calibration trial. If not, it must be in a parent session. loop_counter = 0 while not calibration_trials and session.meta.get('sessionWithCalibration') and loop_counter < 100: @@ -1016,6 +1022,10 @@ def set_metadata(self, request, pk): "posemodel": request.GET.get("subject_pose_model",""), } + if "isMono" in request.GET: + session.isMono = request.GET.get("isMono", "").lower() == 'true' + + if "settings_framerate" in request.GET: session.meta["settings"] = { "framerate": request.GET.get("settings_framerate",""), @@ -1460,16 +1470,24 @@ def dequeue(self, request): try: ip = get_client_ip(request) - workerType = self.request.query_params.get('workerType') + workerType = self.request.query_params.get('workerType', 'all') + isMonoQuery = self.request.query_params.get('isMono', 'False') + + # find trials with some videos not uploaded not_uploaded = Video.objects.filter(video='', updated_at__gte=datetime.now() + timedelta(minutes=-15)).values_list("trial__id", flat=True) + - print(not_uploaded) + # TODO CHANGE THIS LOGIC. + # Trials that have only one video + # only_one_video = Trial.objects.annotate(video_count=Count('video')).filter(video_count=1).values_list("id", flat=True) - uploaded_trials = Trial.objects.exclude(id__in=not_uploaded) - # uploaded_trials = Trial.objects.all() + if isMonoQuery == 'False': + uploaded_trials = Trial.objects.exclude(id__in=not_uploaded).exclude(session__isMono=False) + else: + uploaded_trials = Trial.objects.exclude(id__in=not_uploaded).filter(session__isMono=True) if workerType != 'dynamic': # Priority for 'calibration' and 'neutral' @@ -1501,7 +1519,10 @@ def dequeue(self, request): raise Http404 # prioritize admin and priority group trials (priority group doesn't exist yet, but should have same priv. as user) - trialsPrioritized = trials.filter(session__user__groups__name__in=["admin","priority"]) + trialsPrioritized = trials.filter(session__user__groups__name__in=["admin"]) + # if no admin trials, go to priority group trials + if trialsPrioritized.count() == 0: + trialsPrioritized = trials.filter(session__user__groups__name__in=["priority"]) # if not priority trials, go to normal trials if trialsPrioritized.count() == 0: trialsPrioritized = trials @@ -1532,7 +1553,7 @@ def dequeue(self, request): raise APIException(_('trial_dequeue_error')) return Response(serializer.data) - + @action(detail=False, permission_classes=[((IsAdmin | IsBackend))]) def get_trials_with_status(self, request): """ @@ -2575,4 +2596,4 @@ def data(self, request, pk): return Response(dashboard.get_available_data()) return Response(dashboard.get_available_data( - only_public=True, subject_id=request.GET.get('subject_id'), share_token=request.GET.get('share_token'))) + only_public=True, subject_id=request.GET.get('subject_id'), share_token=request.GET.get('share_token'))) \ No newline at end of file From 3582f42094f21e362798e02c8d6eade7ecdf6ccb Mon Sep 17 00:00:00 2001 From: Seeeeeyo Date: Fri, 6 Jun 2025 10:43:41 -0600 Subject: [PATCH 05/11] ckeaning --- mcserver/views.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mcserver/views.py b/mcserver/views.py index 28f5c23..25e9d7a 100644 --- a/mcserver/views.py +++ b/mcserver/views.py @@ -1480,10 +1480,6 @@ def dequeue(self, request): updated_at__gte=datetime.now() + timedelta(minutes=-15)).values_list("trial__id", flat=True) - # TODO CHANGE THIS LOGIC. - # Trials that have only one video - # only_one_video = Trial.objects.annotate(video_count=Count('video')).filter(video_count=1).values_list("id", flat=True) - if isMonoQuery == 'False': uploaded_trials = Trial.objects.exclude(id__in=not_uploaded).exclude(session__isMono=False) else: From a3fc55eb274d45975417b07d0f769308db2e962f Mon Sep 17 00:00:00 2001 From: Seeeeeyo Date: Fri, 6 Jun 2025 10:50:03 -0600 Subject: [PATCH 06/11] db index isMono --- mcserver/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcserver/models.py b/mcserver/models.py index f7bcffb..9597d48 100644 --- a/mcserver/models.py +++ b/mcserver/models.py @@ -79,7 +79,7 @@ class Session(models.Model): trashed = models.BooleanField(default=False) trashed_at = models.DateTimeField(blank=True, null=True) - isMono = models.BooleanField(default=False) + isMono = models.BooleanField(default=False, db_index=True) class Meta: ordering = ['-created_at'] From 977ecb9240a3324b4e9b9d7db0a9af1078fbbabf Mon Sep 17 00:00:00 2001 From: suhlrich <43877159+suhlrich@users.noreply.github.com> Date: Fri, 6 Jun 2025 11:07:20 -0600 Subject: [PATCH 07/11] add isMono to admin --- mcserver/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcserver/admin.py b/mcserver/admin.py index 61cd1f7..01f8cc4 100644 --- a/mcserver/admin.py +++ b/mcserver/admin.py @@ -58,7 +58,7 @@ class SessionAdmin(admin.ModelAdmin): 'public', 'created_at', 'updated_at', 'server', 'status', 'status_changed', - 'trashed', 'trashed_at', + 'trashed', 'trashed_at', 'isMono', ) raw_id_fields = ('user', 'subject') search_fields = ['id', 'user__username', "subject__name"] From 5446577326fa5f0522b5e1ea73aa6e44377fe778 Mon Sep 17 00:00:00 2001 From: suhlrich <43877159+suhlrich@users.noreply.github.com> Date: Fri, 6 Jun 2025 15:29:58 -0600 Subject: [PATCH 08/11] bug --- mcserver/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcserver/views.py b/mcserver/views.py index 25e9d7a..3fbd74f 100644 --- a/mcserver/views.py +++ b/mcserver/views.py @@ -1481,7 +1481,7 @@ def dequeue(self, request): if isMonoQuery == 'False': - uploaded_trials = Trial.objects.exclude(id__in=not_uploaded).exclude(session__isMono=False) + uploaded_trials = Trial.objects.exclude(id__in=not_uploaded).exclude(session__isMono=True) else: uploaded_trials = Trial.objects.exclude(id__in=not_uploaded).filter(session__isMono=True) From 94aa790e017bbd6dec70eede8bb248c8be567820 Mon Sep 17 00:00:00 2001 From: suhlrich <43877159+suhlrich@users.noreply.github.com> Date: Fri, 6 Jun 2025 15:51:40 -0600 Subject: [PATCH 09/11] make mono sessions show up --- mcserver/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mcserver/views.py b/mcserver/views.py index 3fbd74f..956bf95 100644 --- a/mcserver/views.py +++ b/mcserver/views.py @@ -395,9 +395,10 @@ def valid(self, request): sessions = sessions.filter(subject=subject) # A session is valid only if at least one trial is the "neutral" trial and its status is "done". + # OR if it is a monocular session, which doesn't require a neutral trial. for session in sessions: trials = Trial.objects.filter(session__exact=session, name__exact="neutral") - if trials.count() < 1: + if trials.count() < 1 and not session.isMono: sessions = sessions.exclude(id__exact=session.id) # Sort by From 005de4b9d101f5d24cffb8594f51b1b4e2de3e63 Mon Sep 17 00:00:00 2001 From: suhlrich <43877159+suhlrich@users.noreply.github.com> Date: Fri, 6 Jun 2025 16:14:17 -0600 Subject: [PATCH 10/11] bug fix for some dev sessions --- mcserver/serializers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mcserver/serializers.py b/mcserver/serializers.py index 437cfa7..ebad734 100644 --- a/mcserver/serializers.py +++ b/mcserver/serializers.py @@ -201,7 +201,9 @@ def session_name(self, session): return str(session.id).split("-")[0] def get_sessionName(self, session): - return session.meta.get("sessionName", "") if session.meta else "" + if hasattr(session, 'meta') and session.meta: + return session.meta.get("sessionName", "") + return "" class Meta: model = Session @@ -254,7 +256,9 @@ def session_name(self, session): return str(session.id).split("-")[0] def get_sessionName(self, session): - return session.meta.get("sessionName", "") + if hasattr(session, 'meta') and session.meta: + return session.meta.get("sessionName", "") + return "" class SessionStatusSerializer(serializers.ModelSerializer): From 55915d382f93a8fc860b593e5c7adc841a0b038e Mon Sep 17 00:00:00 2001 From: suhlrich <43877159+suhlrich@users.noreply.github.com> Date: Fri, 6 Jun 2025 16:30:26 -0600 Subject: [PATCH 11/11] index updated_at in the database, and only look for trials updated in the past week in the dequeue view --- ...sion_is_mono_alter_subject_sex_at_birth.py | 23 +++++++++++++++++++ .../migrations/0041_remove_session_is_mono.py | 17 ++++++++++++++ mcserver/migrations/0042_session_ismono.py | 18 +++++++++++++++ .../migrations/0043_alter_session_ismono.py | 18 +++++++++++++++ .../migrations/0044_auto_20250606_1629.py | 23 +++++++++++++++++++ mcserver/models.py | 2 +- mcserver/views.py | 6 +++-- 7 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 mcserver/migrations/0040_session_is_mono_alter_subject_sex_at_birth.py create mode 100644 mcserver/migrations/0041_remove_session_is_mono.py create mode 100644 mcserver/migrations/0042_session_ismono.py create mode 100644 mcserver/migrations/0043_alter_session_ismono.py create mode 100644 mcserver/migrations/0044_auto_20250606_1629.py diff --git a/mcserver/migrations/0040_session_is_mono_alter_subject_sex_at_birth.py b/mcserver/migrations/0040_session_is_mono_alter_subject_sex_at_birth.py new file mode 100644 index 0000000..fc7dcc3 --- /dev/null +++ b/mcserver/migrations/0040_session_is_mono_alter_subject_sex_at_birth.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.22 on 2025-06-05 21:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('mcserver', '0039_merge_20241008_1623'), + ] + + operations = [ + migrations.AddField( + model_name='session', + name='is_mono', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='subject', + name='sex_at_birth', + field=models.CharField(blank=True, choices=[('woman', 'Female'), ('man', 'Male'), ('intersect', 'Intersex'), ('not-listed', 'Not Listed'), ('prefer-not-respond', 'Prefer Not to Respond')], max_length=20, null=True), + ), + ] diff --git a/mcserver/migrations/0041_remove_session_is_mono.py b/mcserver/migrations/0041_remove_session_is_mono.py new file mode 100644 index 0000000..aeca85b --- /dev/null +++ b/mcserver/migrations/0041_remove_session_is_mono.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.22 on 2025-06-05 23:28 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('mcserver', '0040_session_is_mono_alter_subject_sex_at_birth'), + ] + + operations = [ + migrations.RemoveField( + model_name='session', + name='is_mono', + ), + ] diff --git a/mcserver/migrations/0042_session_ismono.py b/mcserver/migrations/0042_session_ismono.py new file mode 100644 index 0000000..76f813a --- /dev/null +++ b/mcserver/migrations/0042_session_ismono.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.22 on 2025-06-06 16:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('mcserver', '0041_remove_session_is_mono'), + ] + + operations = [ + migrations.AddField( + model_name='session', + name='isMono', + field=models.BooleanField(default=False), + ), + ] diff --git a/mcserver/migrations/0043_alter_session_ismono.py b/mcserver/migrations/0043_alter_session_ismono.py new file mode 100644 index 0000000..8c95d23 --- /dev/null +++ b/mcserver/migrations/0043_alter_session_ismono.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.22 on 2025-06-06 16:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('mcserver', '0042_session_ismono'), + ] + + operations = [ + migrations.AlterField( + model_name='session', + name='isMono', + field=models.BooleanField(db_index=True, default=False), + ), + ] diff --git a/mcserver/migrations/0044_auto_20250606_1629.py b/mcserver/migrations/0044_auto_20250606_1629.py new file mode 100644 index 0000000..75a9106 --- /dev/null +++ b/mcserver/migrations/0044_auto_20250606_1629.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.14 on 2025-06-06 22:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('mcserver', '0043_alter_session_ismono'), + ] + + operations = [ + migrations.AlterField( + model_name='analysisresult', + name='status', + field=models.IntegerField(choices=[(100, 'Continue'), (101, 'Switching Protocols'), (102, 'Processing'), (200, 'OK'), (201, 'Created'), (202, 'Accepted'), (203, 'Non-Authoritative Information'), (204, 'No Content'), (205, 'Reset Content'), (206, 'Partial Content'), (207, 'Multi-Status'), (208, 'Already Reported'), (226, 'IM Used'), (300, 'Multiple Choices'), (301, 'Moved Permanently'), (302, 'Found'), (303, 'See Other'), (304, 'Not Modified'), (305, 'Use Proxy'), (307, 'Temporary Redirect'), (308, 'Permanent Redirect'), (400, 'Bad Request'), (401, 'Unauthorized'), (402, 'Payment Required'), (403, 'Forbidden'), (404, 'Not Found'), (405, 'Method Not Allowed'), (406, 'Not Acceptable'), (407, 'Proxy Authentication Required'), (408, 'Request Timeout'), (409, 'Conflict'), (410, 'Gone'), (411, 'Length Required'), (412, 'Precondition Failed'), (413, 'Request Entity Too Large'), (414, 'Request-URI Too Long'), (415, 'Unsupported Media Type'), (416, 'Requested Range Not Satisfiable'), (417, 'Expectation Failed'), (421, 'Misdirected Request'), (422, 'Unprocessable Entity'), (423, 'Locked'), (424, 'Failed Dependency'), (426, 'Upgrade Required'), (428, 'Precondition Required'), (429, 'Too Many Requests'), (431, 'Request Header Fields Too Large'), (500, 'Internal Server Error'), (501, 'Not Implemented'), (502, 'Bad Gateway'), (503, 'Service Unavailable'), (504, 'Gateway Timeout'), (505, 'HTTP Version Not Supported'), (506, 'Variant Also Negotiates'), (507, 'Insufficient Storage'), (508, 'Loop Detected'), (510, 'Not Extended'), (511, 'Network Authentication Required')], default=200, help_text='Status code function responsed with.', verbose_name='Status'), + ), + migrations.AlterField( + model_name='trial', + name='updated_at', + field=models.DateTimeField(auto_now=True, db_index=True), + ), + ] diff --git a/mcserver/models.py b/mcserver/models.py index 9597d48..5ce5d9e 100644 --- a/mcserver/models.py +++ b/mcserver/models.py @@ -109,7 +109,7 @@ class Trial(models.Model): name = models.CharField(max_length=64, null=True) meta = models.JSONField(blank=True, null=True) created_at = models.DateTimeField(auto_now_add=True, db_index=True) - updated_at = models.DateTimeField(auto_now=True) + updated_at = models.DateTimeField(auto_now=True, db_index=True) server = models.GenericIPAddressField(null=True, blank=True) is_docker = models.BooleanField(null=True, blank=True) hostname = models.CharField(max_length=64, null=True, blank=True) diff --git a/mcserver/views.py b/mcserver/views.py index 956bf95..2c59cb7 100644 --- a/mcserver/views.py +++ b/mcserver/views.py @@ -1482,9 +1482,11 @@ def dequeue(self, request): if isMonoQuery == 'False': - uploaded_trials = Trial.objects.exclude(id__in=not_uploaded).exclude(session__isMono=True) + uploaded_trials = Trial.objects.filter(updated_at__gte=datetime.now() + timedelta(days=-7)).exclude( + id__in=not_uploaded).exclude(session__isMono=True) else: - uploaded_trials = Trial.objects.exclude(id__in=not_uploaded).filter(session__isMono=True) + uploaded_trials = Trial.objects.filter(updated_at__gte=datetime.now() + timedelta(days=-7)).exclude( + id__in=not_uploaded).filter(session__isMono=True) if workerType != 'dynamic': # Priority for 'calibration' and 'neutral'