Skip to content

Commit

Permalink
fix(api): Added Docket "d" and RECAPDocument "rd" search types in V4 …
Browse files Browse the repository at this point in the history 
…Search API
  • Loading branch information
@albertisfu
albertisfu committed Apr 26, 2024
1 parent 102e7fc commit 5745716
Show file tree
Hide file tree
Showing 10 changed files with 531 additions and 268 deletions.
66 changes: 66 additions & 0 deletions cl/alerts/migrations/0009_alter_alert_search_types_noop.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# Generated by Django 5.0.5 on 2024-04-25 21:44

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("alerts", "0008_add_scheduled_alert_hit_and_alert_type_to_alert"),
]

operations = [
migrations.AlterField(
model_name="alert",
name="alert_type",
field=models.CharField(
choices=[
("o", "Opinions"),
("r", "RECAP"),
("d", "RECAP Dockets"),
("rd", "RECAP Documents"),
("oa", "Oral Arguments"),
("p", "People"),
("pa", "Parenthetical"),
],
default="o",
help_text="The type of search alert this is, one of: o (Opinions), r (RECAP), d (RECAP Dockets), rd (RECAP Documents), oa (Oral Arguments), p (People), pa (Parenthetical)",
max_length=3,
),
),
migrations.AlterField(
model_name="alertevent",
name="alert_type",
field=models.CharField(
choices=[
("o", "Opinions"),
("r", "RECAP"),
("d", "RECAP Dockets"),
("rd", "RECAP Documents"),
("oa", "Oral Arguments"),
("p", "People"),
("pa", "Parenthetical"),
],
default="o",
help_text="The type of search alert this is, one of: o (Opinions), r (RECAP), d (RECAP Dockets), rd (RECAP Documents), oa (Oral Arguments), p (People), pa (Parenthetical)",
max_length=3,
),
),
migrations.AlterField(
model_name="realtimequeue",
name="item_type",
field=models.CharField(
choices=[
("o", "Opinions"),
("r", "RECAP"),
("d", "RECAP Dockets"),
("rd", "RECAP Documents"),
("oa", "Oral Arguments"),
("p", "People"),
("pa", "Parenthetical"),
],
db_index=True,
help_text="the type of item this is, one of: o (Opinions), r (RECAP), d (RECAP Dockets), rd (RECAP Documents), oa (Oral Arguments), p (People), pa (Parenthetical)",
max_length=3,
),
),
]
14 changes: 14 additions & 0 deletions cl/alerts/migrations/0009_alter_alert_search_types_noop.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
BEGIN;
--
-- Alter field alert_type on alert
--
-- (no-op)
--
-- Alter field alert_type on alertevent
--
-- (no-op)
--
-- Alter field item_type on realtimequeue
--
-- (no-op)
COMMIT;
73 changes: 63 additions & 10 deletions cl/lib/elasticsearch_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,13 +206,15 @@ def add_fields_boosting(
if cd["type"] in [
SEARCH_TYPES.RECAP,
SEARCH_TYPES.DOCKETS,
SEARCH_TYPES.RECAP_DOCUMENT,
]:
qf = BOOSTS["es"][cd["type"]].copy()

if cd["type"] in [
SEARCH_TYPES.ORAL_ARGUMENT,
SEARCH_TYPES.RECAP,
SEARCH_TYPES.DOCKETS,
SEARCH_TYPES.RECAP_DOCUMENT,
SEARCH_TYPES.OPINION,
]:
# Give a boost on the case_name field if it's obviously a case_name
Expand Down Expand Up @@ -493,6 +495,8 @@ def build_sort_results(
"name_reverse asc": {"name_reverse": {"order": "asc"}},
"docket_id asc": {"docket_id": {"order": "asc"}},
"docket_id desc": {"docket_id": {"order": "desc"}},
"id asc": {"id": {"order": "asc"}},
"id desc": {"id": {"order": "desc"}},
"dob desc,name_reverse asc": {
"dob": {"order": "desc"},
"name_reverse": {"order": "asc"},
Expand Down Expand Up @@ -534,7 +538,11 @@ def build_sort_results(
if cd["type"] == SEARCH_TYPES.PARENTHETICAL:
order_by_map["score desc"] = {"score": {"order": "desc"}}

if cd["type"] in [SEARCH_TYPES.RECAP, SEARCH_TYPES.DOCKETS]:
if cd["type"] in [
SEARCH_TYPES.RECAP,
SEARCH_TYPES.DOCKETS,
SEARCH_TYPES.RECAP_DOCUMENT,
]:
random_order_field_id = "docket_id"
elif cd["type"] in [SEARCH_TYPES.OPINION]:
random_order_field_id = "cluster_id"
Expand Down Expand Up @@ -981,7 +989,11 @@ def build_es_base_query(
parent_query_fields,
cd.get("q", ""),
)
case SEARCH_TYPES.RECAP | SEARCH_TYPES.DOCKETS:
case (
SEARCH_TYPES.RECAP
| SEARCH_TYPES.DOCKETS
| SEARCH_TYPES.RECAP_DOCUMENT
):
child_fields = SEARCH_RECAP_CHILD_QUERY_FIELDS.copy()
child_fields.extend(
add_fields_boosting(
Expand Down Expand Up @@ -1814,7 +1826,11 @@ def build_has_child_filters(
if appointer:
queries_list.extend(build_text_filter("appointer", appointer))

if cd["type"] in [SEARCH_TYPES.RECAP, SEARCH_TYPES.DOCKETS]:
if cd["type"] in [
SEARCH_TYPES.RECAP,
SEARCH_TYPES.DOCKETS,
SEARCH_TYPES.RECAP_DOCUMENT,
]:
if child_type == "recap_document":
available_only = cd.get("available_only", "")
description = cd.get("description", "")
Expand Down Expand Up @@ -1885,7 +1901,11 @@ def build_join_es_filters(cd: CleanData) -> List:
# Build position has child filter:
queries_list.extend(build_has_child_filters("position", cd))

if cd["type"] in [SEARCH_TYPES.RECAP, SEARCH_TYPES.DOCKETS]:
if cd["type"] in [
SEARCH_TYPES.RECAP,
SEARCH_TYPES.DOCKETS,
SEARCH_TYPES.RECAP_DOCUMENT,
]:
queries_list.extend(
[
*build_term_query(
Expand Down Expand Up @@ -2085,7 +2105,7 @@ def nullify_query_score(query: Query) -> Query:
def apply_custom_score_to_parent_query(
cd: CleanData, query: Query, api_version: Literal["v3", "v4"] | None = None
) -> Query:
"""Apply a custom function score to a parent document.
"""Apply a custom function score to a main document.
:param cd: The query CleanedData
:param query: The ES Query object to be modified.
Expand Down Expand Up @@ -2115,6 +2135,21 @@ def apply_custom_score_to_parent_query(
query, child_order_by, default_score=0
)

if (
child_order_by
and all(child_order_by)
and cd["type"] == SEARCH_TYPES.RECAP_DOCUMENT
):
sort_field, order = child_order_by
if sort_field == "dateFiled" and api_version:
# Applies a custom function score to sort dockets based on their
# dateFiled field. This serves as a workaround to enable the use of
# the search_after cursor for pagination on documents with a None
# dateFiled.
query = build_custom_function_score_for_date(
query, child_order_by, default_score=0
)

return query


Expand All @@ -2140,7 +2175,11 @@ def build_full_join_es_queries(

q_should = []
match cd["type"]:
case SEARCH_TYPES.RECAP | SEARCH_TYPES.DOCKETS:
case (
SEARCH_TYPES.RECAP
| SEARCH_TYPES.DOCKETS
| SEARCH_TYPES.RECAP_DOCUMENT
):
child_type = "recap_document"
case SEARCH_TYPES.OPINION:
child_type = "opinion"
Expand All @@ -2149,6 +2188,7 @@ def build_full_join_es_queries(
if cd["type"] in [
SEARCH_TYPES.RECAP,
SEARCH_TYPES.DOCKETS,
SEARCH_TYPES.RECAP_DOCUMENT,
SEARCH_TYPES.OPINION,
]:
# Build child filters.
Expand Down Expand Up @@ -2337,7 +2377,11 @@ def limit_inner_hits(
match search_type:
case SEARCH_TYPES.OPINION:
child_type = "opinion"
case SEARCH_TYPES.RECAP | SEARCH_TYPES.DOCKETS:
case (
SEARCH_TYPES.RECAP
| SEARCH_TYPES.DOCKETS
| SEARCH_TYPES.RECAP_DOCUMENT
):
child_type = "recap_document"
case _:
return
Expand Down Expand Up @@ -2649,9 +2693,18 @@ def do_es_api_query(
build_sort_results(cd, api_version=api_version)
)
else:
main_query = s
if cd["highlight"]:
main_query = add_es_highlighting(s, cd)
if cd["type"] == SEARCH_TYPES.RECAP_DOCUMENT:
# The RECAP_DOCUMENT search type returns only child documents.
s = build_child_docs_query(
join_query,
cd=cd,
)
s = apply_custom_score_to_parent_query(cd, s, api_version)
main_query = search_query.query(s)
else:
main_query = s
if cd["highlight"]:
main_query = add_es_highlighting(s, cd)

return main_query

Expand Down
96 changes: 46 additions & 50 deletions cl/lib/test_helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def midnight_pt_test(d: datetime.date) -> datetime.datetime:
}


recap_search_v4_api_keys = {
docket_v4_api_keys = {
"assignedTo": lambda x: (
x["assignedTo"]
if x.get("assignedTo")
Expand Down Expand Up @@ -261,55 +261,7 @@ def midnight_pt_test(d: datetime.date) -> datetime.datetime:
"party_id"
]
),
"recap_documents": [
{
"id": lambda x: x["result"].pk,
"docket_entry_id": lambda x: x["result"].docket_entry.pk,
"description": lambda x: (
x["description"]
if x.get("description")
else x["result"].docket_entry.description
),
"entry_number": lambda x: x["result"].docket_entry.entry_number,
"entry_date_filed": lambda x: (
x["result"].docket_entry.date_filed.isoformat()
if x["result"].docket_entry.date_filed
else None
),
"short_description": lambda x: (
x["short_description"]
if x.get("short_description")
else x["result"].description
),
"document_type": lambda x: x["result"].get_document_type_display(),
"document_number": lambda x: (
int(x["result"].document_number)
if x["result"].document_number
else None
),
"pacer_doc_id": lambda x: x["result"].pacer_doc_id or "",
"snippet": lambda x: (
x["snippet"]
if x.get("snippet")
else x["result"].plain_text or ""
),
"attachment_number": lambda x: x["result"].attachment_number
or None,
"is_available": lambda x: x["result"].is_available,
"page_count": lambda x: x["result"].page_count or None,
"filepath_local": lambda x: x["result"].filepath_local.name
or None,
"absolute_url": lambda x: x["result"].get_absolute_url(),
"cites": lambda x: list(
x["result"]
.cited_opinions.all()
.values_list("cited_opinion_id", flat=True)
),
"timestamp": lambda x: x["result"]
.date_created.isoformat()
.replace("+00:00", "Z"),
}
],
"recap_documents": [],
"referredTo": lambda x: (
x["referredTo"]
if x.get("referredTo")
Expand Down Expand Up @@ -340,6 +292,50 @@ def midnight_pt_test(d: datetime.date) -> datetime.datetime:
"more_docs": lambda x: False,
}

recap_document_v4_api_keys = {
"id": lambda x: x["result"].pk,
"docket_entry_id": lambda x: x["result"].docket_entry.pk,
"description": lambda x: (
x["description"]
if x.get("description")
else x["result"].docket_entry.description
),
"entry_number": lambda x: x["result"].docket_entry.entry_number,
"entry_date_filed": lambda x: (
x["result"].docket_entry.date_filed.isoformat()
if x["result"].docket_entry.date_filed
else None
),
"short_description": lambda x: (
x["short_description"]
if x.get("short_description")
else x["result"].description
),
"document_type": lambda x: x["result"].get_document_type_display(),
"document_number": lambda x: (
int(x["result"].document_number)
if x["result"].document_number
else None
),
"pacer_doc_id": lambda x: x["result"].pacer_doc_id or "",
"snippet": lambda x: (
x["snippet"] if x.get("snippet") else x["result"].plain_text or ""
),
"attachment_number": lambda x: x["result"].attachment_number or None,
"is_available": lambda x: x["result"].is_available,
"page_count": lambda x: x["result"].page_count or None,
"filepath_local": lambda x: x["result"].filepath_local.name or None,
"absolute_url": lambda x: x["result"].get_absolute_url(),
"cites": lambda x: list(
x["result"]
.cited_opinions.all()
.values_list("cited_opinion_id", flat=True)
),
"timestamp": lambda x: x["result"]
.date_created.isoformat()
.replace("+00:00", "Z"),
}


class CourtTestCase(SimpleTestCase):
"""Court test case factories"""
Expand Down
Loading

0 comments on commit 5745716

Please sign in to comment.