+ 2024-08-07: Added filepath_pdf_harvard field to OpinionCluster data in bulk exports. This field contains the path to the PDF file from the Harvard Caselaw Access Project for the given case.
+
+ 2024-08-02: Add new fields to the bulk data files for the Docket object: federal_dn_case_type, federal_dn_office_code, federal_dn_judge_initials_assigned, federal_dn_judge_initials_referred, federal_defendant_number, parent_docket_id. +
2023-09-26: Bulk script refactored to make it easier to maintain. Courthouse table added to bulk script. Court appeals_to through table added to bulk script. Bulk script now automatically generates a shell script to load bulk data and stream the script to S3.
diff --git a/cl/api/templates/citation-api-docs-vlatest.html b/cl/api/templates/citation-api-docs-vlatest.html index 0b529578eb..0f27031f38 100644 --- a/cl/api/templates/citation-api-docs-vlatest.html +++ b/cl/api/templates/citation-api-docs-vlatest.html @@ -80,7 +80,7 @@{% url "opinio
}
}
To understand RelatedFilters, see our filtering documentation.
- These filters allow you to filter to the opinions that an opinion cites (it's "Authorities") or the opinions that cite it.
+
These filters allow you to filter to the opinions that an opinion cites (its "Authorities" or backward citations) or the later opinions that cite it (forward citations).
For example, opinion 2812209 is the decision in Obergefell v. Hodges. To see what it cites:
curl -v \
diff --git a/cl/api/tests.py b/cl/api/tests.py
index f03422c339..f550de3b4a 100644
--- a/cl/api/tests.py
+++ b/cl/api/tests.py
@@ -59,6 +59,8 @@
SchoolViewSet,
SourceViewSet,
)
+from cl.people_db.factories import PartyFactory, PartyTypeFactory
+from cl.people_db.models import Attorney
from cl.recap.factories import ProcessingQueueFactory
from cl.recap.views import (
EmailProcessingQueueViewSet,
@@ -759,6 +761,21 @@ async def test_attorney_filters(self) -> None:
self.q = {"parties_represented__name__contains": "Honker-Nope"}
await self.assertCountInResults(0)
+ # Adds extra role to the existing attorney
+ docket = await Docket.objects.afirst()
+ attorney = await Attorney.objects.afirst()
+ await sync_to_async(PartyTypeFactory.create)(
+ party=await sync_to_async(PartyFactory)(
+ docket=docket,
+ attorneys=[attorney],
+ ),
+ docket=docket,
+ )
+ self.q = {"docket__date_created__range": "2017-04-14,2017-04-15"}
+ await self.assertCountInResults(1)
+ self.q = {"docket__date_created__range": "2017-04-15,2017-04-16"}
+ await self.assertCountInResults(0)
+
async def test_party_filters(self) -> None:
self.path = reverse("party-list", kwargs={"version": "v3"})
diff --git a/cl/api/webhooks.py b/cl/api/webhooks.py
index cf6e3f6cab..f6ca97d9e3 100644
--- a/cl/api/webhooks.py
+++ b/cl/api/webhooks.py
@@ -1,4 +1,5 @@
import json
+import random
import requests
from django.conf import settings
@@ -38,7 +39,7 @@ def send_webhook_event(
the webhook is sent.
"""
proxy_server = {
- "http": settings.EGRESS_PROXY_HOST, # type: ignore
+ "http": random.choice(settings.EGRESS_PROXY_HOSTS), # type: ignore
}
headers = {
"Content-type": "application/json",
diff --git a/cl/assets/static-global/css/override.css b/cl/assets/static-global/css/override.css
index cb62236644..7a27e9f08f 100644
--- a/cl/assets/static-global/css/override.css
+++ b/cl/assets/static-global/css/override.css
@@ -745,7 +745,6 @@ div.shown ul {
padding: 3px 0 5px 0;
}
-
#summaries ul {
padding-inline-start: 15px;
border-bottom: 1pt solid #DDD;
@@ -893,6 +892,20 @@ input.court-checkbox, input.status-checkbox {
border-bottom: 1px solid #dddddd;
}
+.description-header,
+.export-csv {
+ padding: 0px;
+}
+
+#export-csv-error {
+ padding: 5px 0px;
+}
+
+@media (min-width: 767px) {
+ .export-csv {
+ padding: 5px 10px;
+ }
+}
#docket-entry-table .recap-documents.row {
padding-top: 0px;
border-bottom: none;
@@ -1681,3 +1694,32 @@ rect.series-segment {
.ml-1 {
margin-left: 0.25rem !important;
}
+
+.htmx-indicator {
+ opacity: 0;
+ transition: opacity 200ms ease-in;
+}
+
+.htmx-hidden-indicator {
+ display:none;
+}
+.htmx-request .htmx-hidden-indicator {
+ display:inline;
+}
+.htmx-request.htmx-hidden-indicator {
+ display:inline;
+}
+
+.turbo-progress-bar {
+ position: fixed;
+ display: block;
+ top: 0;
+ left: 0;
+ height: 3px;
+ background: #B53C2C;
+ z-index: 2147483647;
+ transition:
+ width 300ms ease-out,
+ opacity 150ms 150ms ease-in;
+ transform: translate3d(0, 0, 0);
+}
diff --git a/cl/assets/static-global/js/base.js b/cl/assets/static-global/js/base.js
index ea71c270b5..99355aa207 100644
--- a/cl/assets/static-global/js/base.js
+++ b/cl/assets/static-global/js/base.js
@@ -103,6 +103,8 @@ $(document).ready(function () {
.val(el.val())
.appendTo('#search-form');
});
+ installProgressBar();
+ disableAllSubmitButtons();
document.location = '/?' + $('#search-form').serialize();
}
diff --git a/cl/assets/static-global/js/export-csv.js b/cl/assets/static-global/js/export-csv.js
new file mode 100644
index 0000000000..90fd2206a3
--- /dev/null
+++ b/cl/assets/static-global/js/export-csv.js
@@ -0,0 +1,55 @@
+document.addEventListener('htmx:beforeRequest', function () {
+ // If the mobile error message is currently visible, adds the 'hidden' class
+ // to hide it.
+ let mobileErrorMessage = document.getElementById('mobile-export-csv-error');
+ if (!mobileErrorMessage.classList.contains('hidden')) {
+ mobileErrorMessage.classList.add('hidden');
+ }
+
+ // If the desktop error message is currently visible, adds the 'hidden' class
+ // to hide it.
+ let desktopErrorMessage = document.getElementById('export-csv-error');
+ if (!desktopErrorMessage.classList.contains('hidden')) {
+ desktopErrorMessage.classList.add('hidden');
+ }
+});
+
+document.addEventListener('htmx:beforeOnLoad', function (event) {
+ // Get the XMLHttpRequest object from the event details
+ const xhr = event.detail.xhr;
+ if (xhr.status == 200) {
+ const response = xhr.response;
+ // Extract the filename from the Content-Disposition header and get
+ //the MIME type from the Content-Type header
+ const filename = xhr.getResponseHeader('Content-Disposition').split('=')[1];
+ const mimetype = xhr.getResponseHeader('Content-Type');
+
+ // Prepare a link element for a file download
+ // Create a hidden link element for download
+ const link = document.createElement('a');
+ link.style.display = 'none';
+
+ // Create a Blob object containing the response data with the correct
+ // MIME type and generate a temporary URL for it
+ const blob = new Blob([response], { type: mimetype });
+ const url = window.URL.createObjectURL(blob);
+
+ // Set the link's attributes for download
+ link.href = url;
+ link.download = filename.replaceAll('"', '');
+
+ // It needs to be added to the DOM so it can be clicked
+ document.body.appendChild(link);
+ link.click();
+
+ // Release the temporary URL after download (for memory management)
+ window.URL.revokeObjectURL(url);
+ } else {
+ // If the request failed, show the error messages
+ let mobileErrorMessage = document.getElementById('mobile-export-csv-error');
+ mobileErrorMessage.classList.remove('hidden');
+
+ let desktopErrorMessage = document.getElementById('export-csv-error');
+ desktopErrorMessage.classList.remove('hidden');
+ }
+});
diff --git a/cl/assets/static-global/js/progress-bar.js b/cl/assets/static-global/js/progress-bar.js
new file mode 100644
index 0000000000..f7115da79f
--- /dev/null
+++ b/cl/assets/static-global/js/progress-bar.js
@@ -0,0 +1,32 @@
+let trickleInterval = null;
+
+function updateProgressBar() {
+ let progressElement = document.getElementById('progress-bar');
+ let oldValue = 0;
+ if ('value' in progressElement.dataset) {
+ oldValue = parseFloat(progressElement.dataset.value);
+ }
+ let newValue = oldValue + Math.random() / 10;
+ if (newValue >= 1) newValue = 1;
+ progressElement.style.width = `${10 + newValue * 75}%`;
+ progressElement.dataset.value = newValue;
+}
+
+function installProgressBar() {
+ let progressElement = document.createElement('div');
+ progressElement.id = 'progress-bar';
+ progressElement.classList.add('turbo-progress-bar');
+ progressElement.style.width = '0';
+ progressElement.style.opacity = '1';
+ document.body.prepend(progressElement);
+ trickleInterval = window.setInterval(updateProgressBar, 300);
+}
+
+function disableAllSubmitButtons() {
+ // Get all submit buttons on the page
+ const submitButtons = document.querySelectorAll('input[type="submit"],button[type="submit"]');
+ // Disable each element
+ submitButtons.forEach((button) => {
+ button.disabled = true;
+ });
+}
diff --git a/cl/assets/templates/base.html b/cl/assets/templates/base.html
index f6ff99f3b7..33181f52e1 100644
--- a/cl/assets/templates/base.html
+++ b/cl/assets/templates/base.html
@@ -87,8 +87,6 @@ You did not supply the "private" variable to your template.
{% include 'includes/dismissible_nav_banner.html' with link="https://free.law/2024/01/18/new-recap-archive-search-is-live" text="A year in the making, today we are launching a huge new search engine for the RECAP Archive" emoji="🎁" cookie_name="no_banner"%}
{% endif %}
- {% include 'includes/dismissible_nav_banner.html' with link="https://free.law/2024/07/05/new-branding-rip-flip" text="CourtListener, RECAP, and Free Law Project are getting some new logos and new branding!" emoji="🔔" cookie_name="no_branding_banner"%}
-
{% if EMAIL_BAN_REASON %}
diff --git a/cl/assets/templates/includes/pagination.html b/cl/assets/templates/includes/pagination.html
index c84280bdc9..55067fa191 100644
--- a/cl/assets/templates/includes/pagination.html
+++ b/cl/assets/templates/includes/pagination.html
@@ -5,14 +5,14 @@
{% if page_obj.has_previous %}
{% if not hide_first %}
-
First
{% endif %}
-
@@ -25,14 +25,14 @@
{% if page_obj.has_next %}
-
Next
{% if not hide_last %}
-
Last
diff --git a/cl/citations/api_views.py b/cl/citations/api_views.py
index 93fe15c2ca..24f8a278a2 100644
--- a/cl/citations/api_views.py
+++ b/cl/citations/api_views.py
@@ -188,7 +188,9 @@ def _citation_handler(
def _get_clusters_for_canonical_list(
self, reporters: list[SafeString], volume: int, page: str
- ) -> tuple[QuerySet[OpinionCluster] | None, int, list[str]]:
+ ) -> tuple[
+ QuerySet[OpinionCluster, OpinionCluster] | None, int, list[str]
+ ]:
"""
Retrieves opinion clusters associated with a list of reporter slugs.
@@ -227,7 +229,7 @@ def _get_clusters_for_canonical_list(
def _format_cluster_response(
self,
- clusters: QuerySet[OpinionCluster],
+ clusters: QuerySet[OpinionCluster, OpinionCluster],
cluster_count: int,
) -> CitationAPIResponse:
"""
diff --git a/cl/citations/management/commands/import_citations_csv.py b/cl/citations/management/commands/import_citations_csv.py
index c709ab710e..5451a6e67a 100644
--- a/cl/citations/management/commands/import_citations_csv.py
+++ b/cl/citations/management/commands/import_citations_csv.py
@@ -10,13 +10,20 @@
How to run the command:
manage.py import_citations_csv --csv /opt/courtlistener/cl/assets/media/wl_citations_1.csv
+# Add all citations from the file and reindex existing ones
+manage.py import_citations_csv --csv /opt/courtlistener/cl/assets/media/wl_citations_1.csv --reindex
+
+# Add and index all citations from the file starting from row 2600000 and reindex existing ones
+manage.py import_citations_csv --csv /opt/courtlistener/cl/assets/media/x.csv --start-row 2600000 --delay 0.1
+
Note: If --limit is greater than --end-row, end row will be ignored
"""
+import argparse
import os.path
+import time
-import numpy as np
import pandas as pd
from django.core.management import BaseCommand
from pandas import DataFrame
@@ -34,51 +41,53 @@ def load_citations_file(options: dict) -> DataFrame | TextFileReader:
:return: loaded data
"""
- start_row = None
end_row = None
- if options["start_row"] and options["end_row"]:
- start_row = options["start_row"] if options["start_row"] > 1 else 0
- end_row = options["end_row"] - options["start_row"] + 1 # inclusive
+ dtype_mapping = {"cluster_id": "int", "citation_to_add": "str"}
- if options["limit"]:
- end_row = options["limit"]
+ if options["end_row"] or options["limit"]:
+ end_row = (
+ options["limit"]
+ if options["limit"] > options["end_row"]
+ else options["end_row"]
+ )
data = pd.read_csv(
options["csv"],
names=["cluster_id", "citation_to_add"],
+ dtype=dtype_mapping,
delimiter=",",
- skiprows=start_row,
+ skiprows=options["start_row"] - 1 if options["start_row"] else None,
nrows=end_row,
+ na_filter=False,
)
- # Replace nan in dataframe
- data = data.replace(np.nan, "", regex=True)
logger.info(f"Found {len(data.index)} rows in csv file: {options['csv']}")
return data
def process_csv_data(
- data: DataFrame | TextFileReader,
+ data: DataFrame | TextFileReader, delay_s: float, reindex: bool
) -> None:
"""Process citations from csv file
:param data: rows from csv file
+ :param delay_s: how long to wait to add each citation
+ :param reindex: force reindex of citations
:return: None
"""
for index, row in data.iterrows():
- cluster_id = int(row.get("cluster_id"))
+ cluster_id = row.get("cluster_id")
citation_to_add = row.get("citation_to_add")
if not OpinionCluster.objects.filter(id=cluster_id).exists():
- logger.info(
- f"Row: {index} - Opinion cluster doesn't exist: {cluster_id}"
- )
+ logger.info(f"Opinion cluster doesn't exist: {cluster_id}")
continue
if cluster_id and citation_to_add:
- add_citations_to_cluster([citation_to_add], cluster_id)
+ add_citations_to_cluster([citation_to_add], cluster_id, reindex)
+ time.sleep(delay_s)
class Command(BaseCommand):
@@ -87,19 +96,34 @@ class Command(BaseCommand):
def __init__(self, *args, **kwargs):
super(Command, self).__init__(*args, **kwargs)
+ def existing_path_type(self, path: str):
+ """Validate file path exists
+
+ :param path: path to validate
+ :return: valid path
+ """
+ if not os.path.exists(path):
+ raise argparse.ArgumentTypeError(
+ f"Csv file: {path} doesn't exist."
+ )
+ return path
+
def add_arguments(self, parser):
parser.add_argument(
"--csv",
+ type=self.existing_path_type,
help="Absolute path to a CSV file containing the citations to add.",
required=True,
)
parser.add_argument(
"--start-row",
+ default=0,
type=int,
help="Start row (inclusive).",
)
parser.add_argument(
"--end-row",
+ default=0,
type=int,
help="End row (inclusive).",
)
@@ -110,19 +134,29 @@ def add_arguments(self, parser):
help="Limit number of rows to process.",
required=False,
)
+ parser.add_argument(
+ "--delay",
+ type=float,
+ default=1.0,
+ help="How long to wait to add each citation (in seconds, allows floating "
+ "numbers).",
+ )
+ parser.add_argument(
+ "--reindex",
+ action="store_true",
+ default=False,
+ help="Reindex citations if they are already in the system",
+ )
def handle(self, *args, **options):
- if options["start_row"] and options["end_row"]:
- if options["start_row"] > options["end_row"]:
- print("--start-row can't be greater than --end-row")
- return
-
- if not os.path.exists(options["csv"]):
- print(f"Csv file: {options['csv']} doesn't exist.")
+ if options["end_row"] and options["start_row"] > options["end_row"]:
+ logger.info("--start-row can't be greater than --end-row")
return
data = load_citations_file(options)
- if not data.empty:
- process_csv_data(data)
- else:
- print("CSV file empty")
+
+ if data.empty:
+ logger.info("CSV file is empty or start/end row returned no rows.")
+ return
+
+ process_csv_data(data, options["delay"], options["reindex"])
diff --git a/cl/citations/match_citations.py b/cl/citations/match_citations.py
index c43d4f03ec..c0d97485b2 100644
--- a/cl/citations/match_citations.py
+++ b/cl/citations/match_citations.py
@@ -203,7 +203,9 @@ def resolve_fullcase_citation(
if waffle.switch_is_active("es_resolve_citations"):
# Revolve citations using ES; enable once all the opinions are
# indexed.
- db_search_results = es_search_db_for_full_citation(full_citation)
+ db_search_results, _ = es_search_db_for_full_citation(
+ full_citation
+ )
else:
db_search_results = search_db_for_fullcitation(full_citation)
# If there is one search result, try to return it
diff --git a/cl/citations/match_citations_queries.py b/cl/citations/match_citations_queries.py
index 8971ad5ce1..ecfab63f58 100644
--- a/cl/citations/match_citations_queries.py
+++ b/cl/citations/match_citations_queries.py
@@ -4,7 +4,9 @@
from elasticsearch_dsl import Q
from elasticsearch_dsl.query import Query
from elasticsearch_dsl.response import Hit, Response
+from eyecite import get_citations
from eyecite.models import FullCaseCitation
+from eyecite.tokenizers import HyperscanTokenizer
from cl.citations.types import SupportedCitationType
from cl.citations.utils import (
@@ -12,9 +14,12 @@
get_years_from_reporter,
make_name_param,
)
+from cl.lib.types import CleanData
from cl.search.documents import OpinionDocument
from cl.search.models import Opinion
+HYPERSCAN_TOKENIZER = HyperscanTokenizer(cache_dir=".hyperscan")
+
def fetch_citations(search_query: Search) -> list[Hit]:
"""Fetches citation matches from Elasticsearch based on the provided
@@ -27,7 +32,9 @@ def fetch_citations(search_query: Search) -> list[Hit]:
citation_hits = []
search_query = search_query.sort("id")
# Only retrieve fields required for the lookup.
- search_query = search_query.source(includes=["id", "caseName"])
+ search_query = search_query.source(
+ includes=["id", "caseName", "absolute_url", "dateFiled"]
+ )
# Citation resolution aims for a single match. Setting up a size of 2 is
# enough to determine if there is more than one match.
search_query = search_query.extra(size=2)
@@ -124,13 +131,15 @@ def es_case_name_query(
def es_search_db_for_full_citation(
- full_citation: FullCaseCitation,
-) -> list[Hit]:
+ full_citation: FullCaseCitation, query_citation: bool = False
+) -> tuple[list[Hit], bool]:
"""For a citation object, try to match it to an item in the database using
a variety of heuristics.
:param full_citation: A FullCaseCitation instance.
- return: A ElasticSearch Result object with the results, or an empty list if
- no hits
+ :param query_citation: Whether this is related to es_get_query_citation
+ resolution
+ return: A two tuple, the ElasticSearch Result object with the results, or an empty list if
+ no hits and a boolean indicating whether the citation was found.
"""
if not hasattr(full_citation, "citing_opinion"):
@@ -140,12 +149,20 @@ def es_search_db_for_full_citation(
Q(
"term", **{"status.raw": "Published"}
), # Non-precedential documents aren't cited
- Q("match", cluster_child="opinion"),
]
+
+ if query_citation:
+ # If this is related to query citation resolution, look for
+ # opinion_cluster to determine if a citation matched a single cluster.
+ filters.append(Q("match", cluster_child="opinion_cluster"))
+ else:
+ filters.append(Q("match", cluster_child="opinion"))
+
must_not = []
if full_citation.citing_opinion is not None:
# Eliminate self-cites.
must_not.append(Q("match", id=full_citation.citing_opinion.pk))
+
# Set up filter parameters
if full_citation.year:
start_year = end_year = full_citation.year
@@ -184,8 +201,9 @@ def es_search_db_for_full_citation(
query = Q("bool", must_not=must_not, filter=filters)
citations_query = search_query.query(query)
results = fetch_citations(citations_query)
+ citation_found = True if len(results) > 0 else False
if len(results) == 1:
- return results
+ return results, citation_found
if len(results) > 1:
if (
full_citation.citing_opinion is not None
@@ -196,7 +214,36 @@ def es_search_db_for_full_citation(
full_citation,
full_citation.citing_opinion,
)
- return results
-
+ return results, citation_found
# Give up.
- return []
+ return [], citation_found
+
+
+def es_get_query_citation(
+ cd: CleanData,
+) -> tuple[Hit | None, list[FullCaseCitation]]:
+ """Extract citations from the query. If it's a single citation, search for
+ it into ES, and if found, return it.
+
+ :param cd: A CleanData instance.
+ :param return: A two tuple the ES Hit object or None and the list of
+ missing citations from the query.
+ """
+ missing_citations: list[FullCaseCitation] = []
+ if not cd.get("q"):
+ return None, missing_citations
+ citations = get_citations(cd["q"], tokenizer=HYPERSCAN_TOKENIZER)
+ citations = [c for c in citations if isinstance(c, FullCaseCitation)]
+
+ matches = None
+ for citation in citations:
+ matches, citation_found = es_search_db_for_full_citation(
+ citation, query_citation=True
+ )
+ if not citation_found:
+ missing_citations.append(citation)
+
+ if len(citations) == 1 and matches and len(matches) == 1:
+ # If more than one match, don't show the tip
+ return matches[0], missing_citations
+ return matches, missing_citations
diff --git a/cl/citations/parenthetical_utils.py b/cl/citations/parenthetical_utils.py
index 9d126575b8..91bb6db132 100644
--- a/cl/citations/parenthetical_utils.py
+++ b/cl/citations/parenthetical_utils.py
@@ -8,7 +8,7 @@
async def get_or_create_parenthetical_groups(
cluster: OpinionCluster,
-) -> QuerySet[ParentheticalGroup]:
+) -> QuerySet[ParentheticalGroup, ParentheticalGroup]:
"""
Given a cluster, return its existing ParentheticalGroup's from the database
or compute and store new ones if they do not yet exist or need to be updated.
diff --git a/cl/citations/tasks.py b/cl/citations/tasks.py
index 69afeae7e9..335af6a04f 100644
--- a/cl/citations/tasks.py
+++ b/cl/citations/tasks.py
@@ -89,9 +89,9 @@ def find_citations_and_parantheticals_for_recap_documents(
:return: None
"""
- documents: QuerySet[RECAPDocument] = RECAPDocument.objects.filter(
- pk__in=doc_ids
- ).filter(
+ documents: QuerySet[
+ RECAPDocument, RECAPDocument
+ ] = RECAPDocument.objects.filter(pk__in=doc_ids).filter(
ocr_status__in=[
RECAPDocument.OCR_UNNECESSARY,
RECAPDocument.OCR_COMPLETE,
@@ -118,7 +118,9 @@ def find_citations_and_parentheticals_for_opinion_by_pks(
:param index: Whether to add the items to Solr
:return: None
"""
- opinions: QuerySet[Opinion] = Opinion.objects.filter(pk__in=opinion_pks)
+ opinions: QuerySet[Opinion, Opinion] = Opinion.objects.filter(
+ pk__in=opinion_pks
+ )
for opinion in opinions:
try:
store_opinion_citations_and_update_parentheticals(opinion, index)
diff --git a/cl/citations/types.py b/cl/citations/types.py
index a84dbfd24b..c56bce4956 100644
--- a/cl/citations/types.py
+++ b/cl/citations/types.py
@@ -23,4 +23,4 @@ class CitationAPIResponse(TypedDict):
status: int
normalized_citations: NotRequired[list[str]]
error_message: NotRequired[str]
- clusters: NotRequired[QuerySet[OpinionCluster]]
+ clusters: NotRequired[QuerySet[OpinionCluster, OpinionCluster]]
diff --git a/cl/corpus_importer/bulk_utils.py b/cl/corpus_importer/bulk_utils.py
index 730a98b61d..66e45fdc86 100644
--- a/cl/corpus_importer/bulk_utils.py
+++ b/cl/corpus_importer/bulk_utils.py
@@ -6,7 +6,7 @@
from cl.corpus_importer.tasks import get_pacer_doc_by_rd
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.lib.scorched_utils import ExtraSolrInterface
from cl.lib.search_utils import build_main_query_from_query_string
from cl.scrapers.tasks import extract_recap_pdf
@@ -75,10 +75,10 @@ def get_petitions(
)
q = options["queue"]
throttle = CeleryThrottle(queue_name=q)
- pacer_session = ProxyPacerSession(
+ session = ProxyPacerSession(
username=pacer_username, password=pacer_password
)
- pacer_session.login()
+ session.login()
for i, rd_pk in enumerate(rds):
if i < options["offset"]:
i += 1
@@ -87,17 +87,18 @@ def get_petitions(
break
if i % 1000 == 0:
- pacer_session = ProxyPacerSession(
+ session = ProxyPacerSession(
username=pacer_username, password=pacer_password
)
- pacer_session.login()
+ session.login()
logger.info(f"Sent {i} tasks to celery so far.")
logger.info("Doing row %s", i)
throttle.maybe_wait()
-
chain(
get_pacer_doc_by_rd.s(
- rd_pk, pacer_session.cookies, tag=tag_petitions
+ rd_pk,
+ SessionData(session.cookies, session.proxy_address),
+ tag=tag_petitions,
).set(queue=q),
extract_recap_pdf.si(rd_pk).set(queue=q),
add_items_to_solr.si([rd_pk], "search.RECAPDocument").set(queue=q),
diff --git a/cl/corpus_importer/import_columbia/columbia_utils.py b/cl/corpus_importer/import_columbia/columbia_utils.py
index b1a62cfd6c..8434568c7e 100644
--- a/cl/corpus_importer/import_columbia/columbia_utils.py
+++ b/cl/corpus_importer/import_columbia/columbia_utils.py
@@ -224,7 +224,7 @@ def extract_columbia_opinions(
"""
opinions: list = []
floating_content = []
- order = 0
+ order = 1
# We iterate all content to look for all possible opinions
for i, content in enumerate(outer_opinion): # type: int, Tag
@@ -363,7 +363,7 @@ def process_extracted_opinions(extracted_opinions: list) -> list:
opinions: list = []
authorless_content = []
- order = 0
+ order = 1
for i, found_content in enumerate(extracted_opinions, start=1):
byline = found_content.get("byline")
diff --git a/cl/corpus_importer/management/commands/760_project.py b/cl/corpus_importer/management/commands/760_project.py
index b4a227f0aa..b31c3a810c 100644
--- a/cl/corpus_importer/management/commands/760_project.py
+++ b/cl/corpus_importer/management/commands/760_project.py
@@ -13,7 +13,7 @@
)
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.search.models import Court, RECAPDocument
from cl.search.tasks import add_or_update_recap_docket
@@ -36,6 +36,7 @@ def get_dockets(options):
username=PACER_USERNAME, password=PACER_PASSWORD
)
session.login()
+ session_data = SessionData(session.cookies, session.proxy_address)
for i, row in enumerate(reader):
if i < options["offset"]:
continue
@@ -55,7 +56,7 @@ def get_dockets(options):
get_appellate_docket_by_docket_number.s(
docket_number=row["Cleaned case_No"],
court_id=row["fjc_court_id"],
- cookies=session.cookies,
+ session_data=session_data,
tag_names=[TAG],
**{
"show_docket_entries": True,
@@ -75,12 +76,12 @@ def get_dockets(options):
pass_through=None,
docket_number=row["Cleaned case_No"],
court_id=row["fjc_court_id"],
- cookies=session.cookies,
+ session_data=session_data,
case_name=row["Title"],
).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id=row["fjc_court_id"],
- cookies=session.cookies,
+ session_data=session_data,
tag_names=[TAG],
**{
"show_parties_and_counsel": True,
diff --git a/cl/corpus_importer/management/commands/adelman_david.py b/cl/corpus_importer/management/commands/adelman_david.py
index f24f58cae3..25aa72db2f 100644
--- a/cl/corpus_importer/management/commands/adelman_david.py
+++ b/cl/corpus_importer/management/commands/adelman_david.py
@@ -12,7 +12,7 @@
)
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import CommandUtils, VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.search.tasks import add_or_update_recap_docket
PACER_USERNAME = os.environ.get("PACER_USERNAME", settings.PACER_USERNAME)
@@ -33,6 +33,7 @@ def download_dockets(options):
username=PACER_USERNAME, password=PACER_PASSWORD
)
session.login()
+ session_data = SessionData(session.cookies, session.proxy_address)
for i, row in enumerate(reader):
if i < options["offset"]:
continue
@@ -48,7 +49,7 @@ def download_dockets(options):
get_appellate_docket_by_docket_number.s(
docket_number=row["docket_no1"],
court_id=row["cl_court"],
- cookies=session.cookies,
+ session_data=session_data,
tag_names=[PROJECT_TAG_NAME, row_tag],
# Do not get the docket entries for now. We're only
# interested in the date terminated. If it's an open case,
@@ -71,17 +72,17 @@ def download_dockets(options):
pass_through=None,
docket_number=row["docket_no1"],
court_id=row["cl_court"],
- cookies=session.cookies,
+ session_data=session_data,
case_name=row["name"],
).set(queue=q),
do_case_query_by_pacer_case_id.s(
court_id=row["cl_court"],
- cookies=session.cookies,
+ session_data=session_data,
tag_names=[PROJECT_TAG_NAME, row_tag],
).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id=row["cl_court"],
- cookies=session.cookies,
+ session_data=session_data,
tag_names=[PROJECT_TAG_NAME, row_tag],
**{
# No docket entries
diff --git a/cl/corpus_importer/management/commands/buchwald_project.py b/cl/corpus_importer/management/commands/buchwald_project.py
index 7beb4865af..ba10538152 100644
--- a/cl/corpus_importer/management/commands/buchwald_project.py
+++ b/cl/corpus_importer/management/commands/buchwald_project.py
@@ -13,7 +13,7 @@
)
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.search.models import Docket
from cl.search.tasks import add_or_update_recap_docket
@@ -59,7 +59,7 @@ def add_all_nysd_to_cl(options):
throttle.maybe_wait()
logger.info("Doing pacer_case_id: %s", pacer_case_id)
make_docket_by_iquery.apply_async(
- args=("nysd", pacer_case_id, session.cookies, [NYSD_TAG]),
+ args=("nysd", pacer_case_id, "default", [NYSD_TAG]),
queue=q,
)
@@ -104,7 +104,9 @@ def get_dockets(options):
get_docket_by_pacer_case_id.s(
data={"pacer_case_id": d.pacer_case_id},
court_id=d.court_id,
- cookies=session.cookies,
+ session_data=SessionData(
+ session.cookies, session.proxy_address
+ ),
docket_pk=d.pk,
tag_names=[BUCKWALD_TAG],
**{
diff --git a/cl/corpus_importer/management/commands/bulk_iquery_project.py b/cl/corpus_importer/management/commands/bulk_iquery_project.py
index 9781e5d9a0..79c32ba351 100644
--- a/cl/corpus_importer/management/commands/bulk_iquery_project.py
+++ b/cl/corpus_importer/management/commands/bulk_iquery_project.py
@@ -1,6 +1,5 @@
import itertools
import time
-from collections import defaultdict
from typing import List, TypedDict
from django.conf import settings
@@ -10,6 +9,7 @@
from redis.exceptions import ConnectionError
from cl.corpus_importer.tasks import make_docket_by_iquery
+from cl.corpus_importer.utils import CycleChecker
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand
from cl.lib.redis_utils import get_redis_interface
@@ -94,57 +94,6 @@ def add_all_cases_to_cl(options: OptionsType) -> None:
time.sleep(options["iteration_delay"])
-class CycleChecker:
- """Keep track of a cycling list to determine each time it starts over.
-
- We plan to iterate over dockets that are ordered by a cycling court ID, so
- imagine if we had two courts, ca1 and ca2, we'd have rows like:
-
- docket: 1, court: ca1
- docket: 14, court: ca2
- docket: 15, court: ca1
- docket: xx, court: ca2
-
- In other words, they'd just go back and forth. In reality, we have about
- 200 courts, but the idea is the same. This code lets us detect each time
- the cycle has started over, even if courts stop being part of the cycle,
- as will happen towards the end of the queryset.. For example, maybe ca1
- finishes, and now we just have:
-
- docket: x, court: ca2
- docket: y, court: ca2
- docket: z, court: ca2
-
- That's considered cycling each time we get to a new row.
-
- The way to use this is to just create an instance and then send it a
- cycling list of court_id's.
-
- Other fun requirements this hits:
- - No need to know the length of the cycle
- - No need to externally track the iteration count
- """
-
- def __init__(self) -> None:
- self.court_counts: defaultdict = defaultdict(int)
- self.current_iteration: int = 1
-
- def check_if_cycled(self, court_id: str) -> bool:
- """Check if the cycle repeated
-
- :param court_id: The ID of the court
- :return True if the cycle started over, else False
- """
- self.court_counts[court_id] += 1
- if self.court_counts[court_id] == self.current_iteration:
- return False
- else:
- # Finished cycle and court has been seen more times than the
- # iteration count. Bump the iteration count and return True.
- self.current_iteration += 1
- return True
-
-
def update_open_cases(options) -> None:
"""Update any cases that are in our system and not terminated."""
# This is a very fancy query that fetches the results while cycling over
diff --git a/cl/corpus_importer/management/commands/buried_alive_project.py b/cl/corpus_importer/management/commands/buried_alive_project.py
index 880176072e..d81a4d2185 100644
--- a/cl/corpus_importer/management/commands/buried_alive_project.py
+++ b/cl/corpus_importer/management/commands/buried_alive_project.py
@@ -7,7 +7,7 @@
from cl.corpus_importer.tasks import get_docket_by_pacer_case_id
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.lib.scorched_utils import ExtraSolrInterface
from cl.lib.search_utils import build_main_query_from_query_string
from cl.search.models import Docket
@@ -64,7 +64,10 @@ def get_pacer_dockets(options, docket_pks, tags):
get_docket_by_pacer_case_id.s(
{"pacer_case_id": d.pacer_case_id, "docket_pk": d.pk},
d.court_id,
- cookies=pacer_session.cookies,
+ session_data=SessionData(
+ pacer_session.cookies,
+ pacer_session.proxy_address,
+ ),
tag_names=tags,
**{
"show_parties_and_counsel": True,
diff --git a/cl/corpus_importer/management/commands/everything_project.py b/cl/corpus_importer/management/commands/everything_project.py
index 3ea7d27eb2..b48dd4a008 100644
--- a/cl/corpus_importer/management/commands/everything_project.py
+++ b/cl/corpus_importer/management/commands/everything_project.py
@@ -11,7 +11,7 @@
)
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.recap.constants import (
CIVIL_RIGHTS_ACCOMMODATIONS,
CIVIL_RIGHTS_ADA_EMPLOYMENT,
@@ -136,18 +136,19 @@ def get_dockets(options, items, tags, sample_size=0, doc_num_end=""):
throttle.maybe_wait()
params = make_fjc_idb_lookup_params(row)
+ session_data = SessionData(session.cookies, session.proxy_address)
chain(
get_pacer_case_id_and_title.s(
pass_through=None,
docket_number=row.docket_number,
court_id=row.district_id,
- cookies=session.cookies,
+ session_data=session_data,
**params,
).set(queue=q),
filter_docket_by_tags.s(tags, row.district_id).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id=row.district_id,
- cookies=session.cookies,
+ session_data=session_data,
tag_names=tags,
**{
"show_parties_and_counsel": True,
diff --git a/cl/corpus_importer/management/commands/export_control.py b/cl/corpus_importer/management/commands/export_control.py
index da434bd83f..4c24adff94 100644
--- a/cl/corpus_importer/management/commands/export_control.py
+++ b/cl/corpus_importer/management/commands/export_control.py
@@ -8,7 +8,7 @@
from cl.corpus_importer.tasks import save_ia_docket_to_disk
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.search.models import Court, Docket
PACER_USERNAME = os.environ.get("PACER_USERNAME", settings.PACER_USERNAME)
@@ -85,7 +85,7 @@ def get_data(options, row_transform, tags):
row["docket_number"],
row["court"],
row["case_name"],
- session.cookies,
+ SessionData(session.cookies, session.proxy_address),
tags,
q,
)
diff --git a/cl/corpus_importer/management/commands/harvard_opinions.py b/cl/corpus_importer/management/commands/harvard_opinions.py
index 9b39278c90..a3206f8639 100644
--- a/cl/corpus_importer/management/commands/harvard_opinions.py
+++ b/cl/corpus_importer/management/commands/harvard_opinions.py
@@ -498,9 +498,10 @@ def add_new_case(
docket = update_or_create_docket(
case_name,
case_name_short,
- court_id,
+ Court.objects.get(id=court_id),
docket_string,
Docket.HARVARD,
+ from_harvard=True,
case_name_full=case_name_full,
ia_needs_upload=False,
)
diff --git a/cl/corpus_importer/management/commands/import_patent.py b/cl/corpus_importer/management/commands/import_patent.py
index f207f649ab..b6956f0406 100644
--- a/cl/corpus_importer/management/commands/import_patent.py
+++ b/cl/corpus_importer/management/commands/import_patent.py
@@ -11,7 +11,7 @@
)
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.recap.constants import PATENT, PATENT_ANDA
from cl.recap.models import FjcIntegratedDatabase
from cl.search.models import Docket
@@ -44,7 +44,7 @@ def get_dockets(options: dict) -> None:
username=PACER_USERNAME, password=PACER_PASSWORD
)
session.login()
-
+ session_data = SessionData(session.cookies, session.proxy_address)
NOS_CODES = [PATENT, PATENT_ANDA]
DISTRICTS = ["ded", "txwd"]
START_DATE = "2012-01-01"
@@ -78,12 +78,12 @@ def get_dockets(options: dict) -> None:
pass_through=None,
docket_number=item.docket_number,
court_id=item.district_id,
- cookies=session.cookies,
+ session_data=session_data,
**params,
).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id=item.district_id,
- cookies=session.cookies,
+ session_data=session_data,
tag_names=PATENT_TAGS,
**{
"show_parties_and_counsel": True,
@@ -101,7 +101,7 @@ def get_dockets(options: dict) -> None:
get_docket_by_pacer_case_id.s(
data={"pacer_case_id": d.pacer_case_id},
court_id=d.court_id,
- cookies=session.cookies,
+ session_data=session_data,
docket_pk=d.pk,
tag_names=PATENT_TAGS,
**{
diff --git a/cl/corpus_importer/management/commands/invoice_project.py b/cl/corpus_importer/management/commands/invoice_project.py
index 8f3f889c34..1a48d80f25 100644
--- a/cl/corpus_importer/management/commands/invoice_project.py
+++ b/cl/corpus_importer/management/commands/invoice_project.py
@@ -14,7 +14,7 @@
)
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.lib.scorched_utils import ExtraSolrInterface
from cl.lib.search_utils import build_main_query_from_query_string
from cl.recap.tasks import process_recap_attachment
@@ -83,9 +83,10 @@ def get_attachment_pages(options):
throttle.maybe_wait()
chain(
# Query the attachment page and process it
- get_attachment_page_by_rd.s(result["id"], session.cookies).set(
- queue=q
- ),
+ get_attachment_page_by_rd.s(
+ result["id"],
+ SessionData(session.cookies, session.proxy_address),
+ ).set(queue=q),
# Take that in a new task and make a PQ object
make_attachment_pq_object.s(result["id"], recap_user.pk).set(
queue=q
@@ -150,9 +151,11 @@ def get_documents(options):
continue
chain(
- get_pacer_doc_by_rd.s(rd.pk, session.cookies, tag=TAG_PHASE_2).set(
- queue=q
- ),
+ get_pacer_doc_by_rd.s(
+ rd.pk,
+ SessionData(session.cookies, session.proxy_address),
+ tag=TAG_PHASE_2,
+ ).set(queue=q),
extract_recap_pdf.si(rd.pk).set(queue=q),
add_items_to_solr.si([rd.pk], "search.RECAPDocument").set(queue=q),
).apply_async()
diff --git a/cl/corpus_importer/management/commands/jackson_project.py b/cl/corpus_importer/management/commands/jackson_project.py
index 1e7fd98e3b..d5afc22f02 100644
--- a/cl/corpus_importer/management/commands/jackson_project.py
+++ b/cl/corpus_importer/management/commands/jackson_project.py
@@ -6,7 +6,7 @@
from cl.corpus_importer.tasks import get_docket_by_pacer_case_id
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.search.models import Docket
from cl.search.tasks import add_or_update_recap_docket
@@ -41,7 +41,9 @@ def get_dockets(options):
get_docket_by_pacer_case_id.s(
data={"pacer_case_id": d.pacer_case_id},
court_id=d.court_id,
- cookies=session.cookies,
+ session_data=SessionData(
+ session.cookies, session.proxy_address
+ ),
docket_pk=d.pk,
tag_names=[JACKSON_TAG],
**{
diff --git a/cl/corpus_importer/management/commands/kessler_ilnb.py b/cl/corpus_importer/management/commands/kessler_ilnb.py
index a3ad701b23..2c16d3c5d2 100644
--- a/cl/corpus_importer/management/commands/kessler_ilnb.py
+++ b/cl/corpus_importer/management/commands/kessler_ilnb.py
@@ -16,7 +16,7 @@
)
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.scrapers.tasks import extract_recap_pdf
from cl.search.models import DocketEntry, RECAPDocument
from cl.search.tasks import add_items_to_solr, add_or_update_recap_docket
@@ -53,6 +53,9 @@ def get_dockets(options):
logger.info(f"Sent {i} tasks to celery so far.")
logger.info("Doing row %s", i)
throttle.maybe_wait()
+ session_data = SessionData(
+ pacer_session.cookies, pacer_session.proxy_address
+ )
chain(
get_pacer_case_id_and_title.s(
pass_through=None,
@@ -60,13 +63,13 @@ def get_dockets(options):
row["docket"], row["office"]
),
court_id="ilnb",
- cookies=pacer_session.cookies,
+ session_data=session_data,
office_number=row["office"],
docket_number_letters="bk",
).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id="ilnb",
- cookies=pacer_session.cookies,
+ cookies_data=session_data,
tag_names=[TAG],
**{
"show_parties_and_counsel": True,
@@ -118,7 +121,11 @@ def get_final_docs(options):
throttle.maybe_wait()
chain(
get_pacer_doc_by_rd.s(
- rd_pk, pacer_session.cookies, tag=TAG_FINALS
+ rd_pk,
+ SessionData(
+ pacer_session.cookies, pacer_session.proxy_address
+ ),
+ tag=TAG_FINALS,
).set(queue=q),
extract_recap_pdf.si(rd_pk).set(queue=q),
add_items_to_solr.si([rd_pk], "search.RECAPDocument").set(
diff --git a/cl/corpus_importer/management/commands/legal_robot.py b/cl/corpus_importer/management/commands/legal_robot.py
index d6bc38244f..c435e5780b 100644
--- a/cl/corpus_importer/management/commands/legal_robot.py
+++ b/cl/corpus_importer/management/commands/legal_robot.py
@@ -7,7 +7,7 @@
from cl.corpus_importer.tasks import add_tags, get_pacer_doc_by_rd
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.lib.scorched_utils import ExtraSolrInterface
from cl.lib.search_utils import build_main_query_from_query_string
from cl.scrapers.tasks import extract_recap_pdf
@@ -79,9 +79,11 @@ def get_documents(options):
continue
chain(
- get_pacer_doc_by_rd.s(rd.pk, session.cookies, tag=TAG).set(
- queue=q
- ),
+ get_pacer_doc_by_rd.s(
+ rd.pk,
+ SessionData(session.cookies, session.proxy_address),
+ tag=TAG,
+ ).set(queue=q),
extract_recap_pdf.si(rd.pk).set(queue=q),
add_items_to_solr.si([rd.pk], "search.RECAPDocument").set(queue=q),
).apply_async()
diff --git a/cl/corpus_importer/management/commands/list_of_creditors_project.py b/cl/corpus_importer/management/commands/list_of_creditors_project.py
index 9783903212..e3a18d56dd 100644
--- a/cl/corpus_importer/management/commands/list_of_creditors_project.py
+++ b/cl/corpus_importer/management/commands/list_of_creditors_project.py
@@ -16,7 +16,7 @@
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
from cl.lib.pacer import map_cl_to_pacer_id
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.lib.redis_utils import create_redis_semaphore
CLIENT_PACER_USERNAME = os.environ.get("CLIENT_PACER_USERNAME", "")
@@ -139,7 +139,7 @@ def query_and_save_creditors_data(options: OptionsType) -> None:
)
throttle.maybe_wait()
query_and_save_list_of_creditors.si(
- session.cookies,
+ SessionData(session.cookies, session.proxy_address),
court_id,
d_number_file_name,
docket_number,
diff --git a/cl/corpus_importer/management/commands/make_aws_manifest_files.py b/cl/corpus_importer/management/commands/make_aws_manifest_files.py
index 479708c0d0..ebc4690312 100644
--- a/cl/corpus_importer/management/commands/make_aws_manifest_files.py
+++ b/cl/corpus_importer/management/commands/make_aws_manifest_files.py
@@ -14,45 +14,53 @@
s3_client = boto3.client("s3")
-def get_total_number_of_records(type: str, use_replica: bool = False) -> int:
+def get_total_number_of_records(type: str, options: dict[str, Any]) -> int:
"""
Retrieves the total number of records for a specific data type.
Args:
type (str): The type of data to count. Must be one of the valid values
from the `SEARCH_TYPES` class.
- use_replica (bool, optional): Whether to use the replica database
- connection (default: False).
+ options (dict[str, Any]): A dictionary containing options for filtering
+ the results.
+ - 'use_replica' (bool, optional): Whether to use the replica database
+ connection (default: False).
+ - 'random_sample_percentage' (float, optional): The percentage of
+ records to include in a random sample.
Returns:
int: The total number of records matching the specified data type.
"""
match type:
case SEARCH_TYPES.RECAP_DOCUMENT:
- query = """
- SELECT count(*) AS exact_count
- FROM search_recapdocument
+ base_query = (
+ "SELECT count(*) AS exact_count FROM search_recapdocument"
+ )
+ filter_clause = """
WHERE is_available=True AND page_count>0 AND ocr_status!=1
"""
case SEARCH_TYPES.OPINION:
- query = """
- SELECT count(*) AS exact_count
- FROM search_opinion
- WHERE extracted_by_ocr != true
- """
+ base_query = "SELECT count(*) AS exact_count FROM search_opinion"
+ filter_clause = "WHERE extracted_by_ocr != true"
case SEARCH_TYPES.ORAL_ARGUMENT:
- query = """
- SELECT count(*) AS exact_count
- FROM audio_audio
- WHERE
- local_path_mp3 != '' AND
+ base_query = "SELECT count(*) AS exact_count FROM audio_audio"
+ filter_clause = """WHERE local_path_mp3 != '' AND
download_url != 'https://www.cadc.uscourts.gov/recordings/recordings.nsf/' AND
position('Unavailable' in download_url) = 0 AND
duration > 30
"""
+ if options["random_sample_percentage"]:
+ percentage = options["random_sample_percentage"]
+ base_query = f"{base_query} TABLESAMPLE SYSTEM ({percentage})"
+
+ query = (
+ f"{base_query}\n"
+ if options["all_records"]
+ else f"{base_query}\n {filter_clause}\n"
+ )
with connections[
- "replica" if use_replica else "default"
+ "replica" if options["use_replica"] else "default"
].cursor() as cursor:
cursor.execute(query, [])
result = cursor.fetchone()
@@ -60,7 +68,9 @@ def get_total_number_of_records(type: str, use_replica: bool = False) -> int:
return int(result[0])
-def get_custom_query(type: str, last_pk: str) -> tuple[str, list[Any]]:
+def get_custom_query(
+ type: str, last_pk: str, options: dict[str, Any]
+) -> tuple[str, list[Any]]:
"""
Generates a custom SQL query based on the provided type and optional last
pk.
@@ -69,6 +79,10 @@ def get_custom_query(type: str, last_pk: str) -> tuple[str, list[Any]]:
type (str): Type of data to retrieve.
last_pk (int, optional): Last primary key retrieved in a previous
query. Defaults to None.
+ options (dict[str, Any]): A dictionary containing options for filtering
+ the results.
+ - 'random_sample_percentage' (float, optional): The percentage of
+ records to include in a random sample.
Returns:
tuple[str, list[Any]]: A tuple containing the constructed SQL
@@ -76,50 +90,48 @@ def get_custom_query(type: str, last_pk: str) -> tuple[str, list[Any]]:
the query.
"""
params = []
-
+ random_sample = options["random_sample_percentage"]
match type:
case SEARCH_TYPES.RECAP_DOCUMENT:
base_query = "SELECT id from search_recapdocument"
filter_clause = (
"WHERE is_available=True AND page_count>0 AND ocr_status!=1"
- if not last_pk
- else (
- "WHERE id > %s AND is_available = True AND page_count > 0"
- " AND ocr_status != 1"
- )
)
case SEARCH_TYPES.OPINION:
base_query = "SELECT id from search_opinion"
- filter_clause = (
- "WHERE extracted_by_ocr != true"
- if not last_pk
- else "WHERE id > %s AND extracted_by_ocr != true"
- )
+ filter_clause = "WHERE extracted_by_ocr != true"
case SEARCH_TYPES.ORAL_ARGUMENT:
base_query = "SELECT id from audio_audio"
- no_argument_where_clause = """
+ filter_clause = """
WHERE local_path_mp3 != '' AND
download_url != 'https://www.cadc.uscourts.gov/recordings/recordings.nsf/' AND
position('Unavailable' in download_url) = 0 AND
duration > 30
"""
- where_clause_with_argument = """
- WHERE id > %s AND
- local_path_mp3 != '' AND
- download_url != 'https://www.cadc.uscourts.gov/recordings/recordings.nsf/' AND
- position('Unavailable' in download_url) = 0 AND
- duration > 30
- """
- filter_clause = (
- no_argument_where_clause
- if not last_pk
- else where_clause_with_argument
- )
- if last_pk:
+ if random_sample:
+ base_query = f"{base_query} TABLESAMPLE SYSTEM ({random_sample})"
+
+ if options["all_records"]:
+ filter_clause = ""
+
+ # Using a WHERE clause with `id > last_pk` and a LIMIT clause for batch
+ # retrieval is not suitable for random sampling. The following logic
+ # removes these clauses when retrieving a random sample to ensure all rows
+ # have an equal chance of being selected.
+ if last_pk and not random_sample:
+ filter_clause = (
+ f"WHERE id > %s"
+ if not filter_clause
+ else f"{filter_clause} AND id > %s"
+ )
params.append(last_pk)
- query = f"{base_query}\n {filter_clause}\n ORDER BY id\n LIMIT %s"
+ query = (
+ f"{base_query}\n {filter_clause}"
+ if random_sample
+ else f"{base_query}\n {filter_clause}\n ORDER BY id\n LIMIT %s"
+ )
return query, params
@@ -170,6 +182,27 @@ def add_arguments(self, parser: CommandParser):
default=False,
help="Use this flag to run the queries in the replica db",
)
+ parser.add_argument(
+ "--file-name",
+ type=str,
+ default=None,
+ help="Custom name for the output files. If not provided, a default "
+ "name will be used.",
+ )
+ parser.add_argument(
+ "--random-sample-percentage",
+ type=float,
+ default=None,
+ help="Specifies the proportion of the table to be sampled (between "
+ "0.0 and 100.0). Use this flag to retrieve a random set of records.",
+ )
+ parser.add_argument(
+ "--all-records",
+ action="store_true",
+ default=False,
+ help="Use this flag to retrieve all records from the table without"
+ " applying any filters.",
+ )
def handle(self, *args, **options):
r = get_redis_interface("CACHE")
@@ -188,7 +221,7 @@ def handle(self, *args, **options):
)
if not total_number_of_records:
total_number_of_records = get_total_number_of_records(
- record_type, options["use_replica"]
+ record_type, options
)
r.hset(
f"{record_type}_import_status",
@@ -200,12 +233,17 @@ def handle(self, *args, **options):
r.hget(f"{record_type}_import_status", "next_iteration_counter")
or 0
)
+ file_name = (
+ options["file_name"]
+ if options["file_name"]
+ else f"{record_type}_filelist"
+ )
while True:
query, params = get_custom_query(
- options["record_type"],
- last_pk,
+ options["record_type"], last_pk, options
)
- params.append(options["query_batch_size"])
+ if not options["random_sample_percentage"]:
+ params.append(options["query_batch_size"])
with connections[
"replica" if options["use_replica"] else "default"
@@ -226,22 +264,37 @@ def handle(self, *args, **options):
extrasaction="ignore",
)
for row in batched(rows, options["lambda_record_size"]):
- query_dict = {
- "bucket": bucket_name,
- "file_name": (
+ if options["random_sample_percentage"]:
+ # Create an underscore-separated file name that lambda
+ # can split and use as part of batch processing.
+ ids = [str(r[0]) for r in row]
+ content = "_".join(ids)
+ else:
+ content = (
f"{row[0][0]}_{row[-1][0]}"
if len(row) > 1
else f"{row[0][0]}"
- ),
+ )
+ query_dict = {
+ "bucket": bucket_name,
+ "file_name": content,
}
writer.writerow(query_dict)
s3_client.put_object(
- Key=f"{record_type}_filelist_{counter}.csv",
+ Key=f"{file_name}_{counter}.csv",
Bucket=bucket_name,
Body=csvfile.getvalue().encode("utf-8"),
)
+ if options["random_sample_percentage"]:
+ # Due to the non-deterministic nature of random sampling,
+ # storing data to recover the query for future executions
+ # wouldn't be meaningful. Random queries are unlikely to
+ # produce the same results on subsequent runs.
+ logger.info(f"Finished processing {record_count} records")
+ break
+
counter += 1
last_pk = rows[-1][0]
records_processed = int(
diff --git a/cl/corpus_importer/management/commands/nos_700.py b/cl/corpus_importer/management/commands/nos_700.py
index 915c030eef..b95c663891 100644
--- a/cl/corpus_importer/management/commands/nos_700.py
+++ b/cl/corpus_importer/management/commands/nos_700.py
@@ -12,7 +12,7 @@
)
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.recap.constants import (
AIRPLANE_PERSONAL_INJURY,
AIRPLANE_PRODUCT_LIABILITY,
@@ -251,19 +251,20 @@ def get_dockets(options, items, tags, sample_size=0):
logger.info("Doing row %s: %s", i, row)
throttle.maybe_wait()
+ session_data = SessionData(session.cookies, session.proxy_address)
params = make_fjc_idb_lookup_params(row)
chain(
get_pacer_case_id_and_title.s(
pass_through=None,
docket_number=row.docket_number,
court_id=row.district_id,
- cookies=session.cookies,
+ session_data=session_data,
**params,
).set(queue=q),
filter_docket_by_tags.s(tags, row.district_id).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id=row.district_id,
- cookies=session.cookies,
+ session_data=session_data,
tag_names=tags,
**{
"show_parties_and_counsel": True,
diff --git a/cl/corpus_importer/management/commands/nywb_chapter_7.py b/cl/corpus_importer/management/commands/nywb_chapter_7.py
index 7efa9888fa..72aaa914c7 100644
--- a/cl/corpus_importer/management/commands/nywb_chapter_7.py
+++ b/cl/corpus_importer/management/commands/nywb_chapter_7.py
@@ -14,7 +14,7 @@
)
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
-from cl.lib.pacer_session import ProxyPacerSession
+from cl.lib.pacer_session import ProxyPacerSession, SessionData
from cl.search.tasks import add_or_update_recap_docket
PACER_USERNAME = os.environ.get("PACER_USERNAME", "UNKNOWN!")
@@ -48,6 +48,10 @@ def get_dockets(options):
logger.info(f"Sent {i} tasks to celery so far.")
logger.info("Doing row %s", i)
throttle.maybe_wait()
+ session_data = SessionData(
+ pacer_session.cookies,
+ pacer_session.proxy_address,
+ )
chain(
get_pacer_case_id_and_title.s(
pass_through=None,
@@ -55,13 +59,13 @@ def get_dockets(options):
row["DOCKET"], row["OFFICE"]
),
court_id="nywb",
- cookies=pacer_session.cookies,
+ session_data=session_data,
office_number=row["OFFICE"],
docket_number_letters="bk",
).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id="nywb",
- cookies=pacer_session.cookies,
+ session_data=session_data,
tag_names=[TAG],
**{
"doc_num_start": 1,
diff --git a/cl/corpus_importer/management/commands/scrape_pacer_free_opinions.py b/cl/corpus_importer/management/commands/scrape_pacer_free_opinions.py
index 05361ce486..ddb5cbb8c0 100644
--- a/cl/corpus_importer/management/commands/scrape_pacer_free_opinions.py
+++ b/cl/corpus_importer/management/commands/scrape_pacer_free_opinions.py
@@ -1,12 +1,16 @@
import argparse
+import datetime
+import inspect
import os
-from datetime import date, timedelta
-from typing import Callable, Dict, List, Optional, Tuple, cast
+import time
+from typing import Callable, Dict, List, Optional, cast
from celery.canvas import chain
from django.conf import settings
-from django.db.models import QuerySet
+from django.db.models import F, Q, Window
+from django.db.models.functions import RowNumber
from django.utils.timezone import now
+from juriscraper.lib.date_utils import make_date_range_tuples
from juriscraper.lib.exceptions import PacerLoginException
from juriscraper.lib.string_utils import CaseNameTweaker
from requests import RequestException
@@ -19,6 +23,8 @@
mark_court_done_on_date,
process_free_opinion_result,
)
+from cl.corpus_importer.utils import CycleChecker
+from cl.lib.argparse_types import valid_date
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.command_utils import VerboseCommand, logger
from cl.lib.pacer import map_cl_to_pacer_id, map_pacer_to_cl_id
@@ -32,11 +38,10 @@
PACER_PASSWORD = os.environ.get("PACER_PASSWORD", settings.PACER_PASSWORD)
-def get_next_date_range(
+def get_last_complete_date(
court_id: str,
- span: int = 7,
-) -> Tuple[Optional[date], Optional[date]]:
- """Get the next start and end query dates for a court.
+) -> Optional[datetime.date]:
+ """Get the next start query date for a court.
Check the DB for the last date for a court that was completed. Return the
day after that date + span days into the future as the range to query for
@@ -45,7 +50,7 @@ def get_next_date_range(
If the court is still in progress, return (None, None).
:param court_id: A PACER Court ID
- :param span: The number of days to go forward from the last completed date
+ :return: last date queried for the specified court or None if it is in progress
"""
court_id = map_pacer_to_cl_id(court_id)
try:
@@ -59,29 +64,114 @@ def get_next_date_range(
raise
if last_completion_log.status == PACERFreeDocumentLog.SCRAPE_IN_PROGRESS:
- return None, None
+ return None
# Ensure that we go back five days from the last time we had success if
# that success was in the last few days.
last_complete_date = min(
- now().date() - timedelta(days=5), last_completion_log.date_queried
+ now().date() - datetime.timedelta(days=5),
+ last_completion_log.date_queried,
)
- next_end_date = min(
- now().date(), last_complete_date + timedelta(days=span)
- )
- return last_complete_date, next_end_date
+ return last_complete_date
+
+def mark_court_in_progress(
+ court_id: str, d: datetime.date
+) -> PACERFreeDocumentLog:
+ """Create row with data of queried court
-def mark_court_in_progress(court_id: str, d: date) -> QuerySet:
- log = PACERFreeDocumentLog.objects.create(
+ Stores the pacer's court id, scraping status, and the last date queried.
+
+ :param court_id: Pacer court id
+ :param d: Last date queried
+ :return: new PACERFreeDocumentLog object
+ """
+ return PACERFreeDocumentLog.objects.create(
status=PACERFreeDocumentLog.SCRAPE_IN_PROGRESS,
date_queried=d,
court_id=map_pacer_to_cl_id(court_id),
)
- return log
-def get_and_save_free_document_reports(options: OptionsType) -> None:
+def fetch_doc_report(
+ pacer_court_id: str,
+ start: datetime.date,
+ end: datetime.date,
+) -> bool:
+ """Get free documents from pacer
+
+ Get free documents from pacer and save each using PACERFreeDocumentRow model
+
+ :param pacer_court_id: Pacer court id to fetch
+ :param start: start date to query
+ :param end: end date to query
+ :return: true if an exception occurred else false
+ """
+ exception_raised = False
+ status = PACERFreeDocumentLog.SCRAPE_FAILED
+ rows_to_create = 0
+
+ log = mark_court_in_progress(pacer_court_id, end)
+
+ logger.info(
+ "Attempting to get latest document references for "
+ "%s between %s and %s",
+ pacer_court_id,
+ start,
+ end,
+ )
+ try:
+ status, rows_to_create = get_and_save_free_document_report(pacer_court_id, start, end, log.pk) # type: ignore
+ except (
+ RequestException,
+ ReadTimeoutError,
+ IndexError,
+ TypeError,
+ PacerLoginException,
+ ValueError,
+ ) as exc:
+ if isinstance(exc, (RequestException, ReadTimeoutError)):
+ reason = "network error."
+ elif isinstance(exc, IndexError):
+ reason = "PACER 6.3 bug."
+ elif isinstance(exc, (TypeError, ValueError)):
+ reason = "failing PACER website."
+ elif isinstance(exc, PacerLoginException):
+ reason = "PACER login issue."
+ else:
+ reason = "unknown reason."
+ logger.error(
+ "Failed to get free document references for "
+ f"{pacer_court_id} between {start} and "
+ f"{end} due to {reason}.",
+ exc_info=True,
+ )
+ exception_raised = True
+
+ mark_court_done_on_date(
+ log.pk,
+ PACERFreeDocumentLog.SCRAPE_FAILED,
+ )
+
+ if not exception_raised:
+ logger.info(
+ "Got %s document references for " "%s between %s and %s",
+ rows_to_create,
+ pacer_court_id,
+ start,
+ end,
+ )
+ # Scrape successful
+ mark_court_done_on_date(log.pk, status)
+
+ return exception_raised
+
+
+def get_and_save_free_document_reports(
+ courts: list[Optional[str]],
+ date_start: Optional[datetime.date],
+ date_end: Optional[datetime.date],
+) -> None:
"""Query the Free Doc Reports on PACER and get a list of all the free
documents. Do not download those items, as that step is done later. For now
just get the list.
@@ -93,103 +183,82 @@ def get_and_save_free_document_reports(options: OptionsType) -> None:
This is a simpler version, though a slower one, but it should get the job
done.
+
+ :param courts: optionally a list of courts to scrape
+ :param date_start: optionally a start date to query all the specified courts or all
+ courts
+ :param date_end: optionally an end date to query all the specified courts or all
+ courts
"""
# Kill any *old* logs that report they're in progress. (They've failed.)
- three_hrs_ago = now() - timedelta(hours=3)
+ three_hrs_ago = now() - datetime.timedelta(hours=3)
PACERFreeDocumentLog.objects.filter(
date_started__lt=three_hrs_ago,
status=PACERFreeDocumentLog.SCRAPE_IN_PROGRESS,
).update(status=PACERFreeDocumentLog.SCRAPE_FAILED)
+ excluded_court_ids = ["casb", "gub", "ilnb", "innb", "miwb", "ohsb", "prb"]
+
+ base_filter = Q(in_use=True, end_date=None) & ~Q(pk__in=excluded_court_ids)
+ if courts:
+ base_filter &= Q(pk__in=courts)
+
cl_court_ids = (
Court.federal_courts.district_or_bankruptcy_pacer_courts()
- .filter(
- in_use=True,
- end_date=None,
- )
- .exclude(pk__in=["casb", "gub", "ilnb", "innb", "miwb", "ohsb", "prb"])
+ .filter(base_filter)
.values_list("pk", flat=True)
)
+
pacer_court_ids = [map_cl_to_pacer_id(v) for v in cl_court_ids]
- today = now()
+
+ dates = None
+ if date_start and date_end:
+ # If we pass the dates in the command then we generate the range on those dates
+ # The first date queried is 1950-05-12 from ca9, that should be the starting
+ # point for the sweep
+ dates = make_date_range_tuples(date_start, date_end, gap=7)
+
for pacer_court_id in pacer_court_ids:
- while True:
- next_start_d, next_end_d = get_next_date_range(pacer_court_id)
- if next_end_d is None:
+ court_failed = False
+ if not dates:
+ # We don't pass the dates in the command, so we generate the range based
+ # on each court
+ date_end = datetime.date.today()
+ date_start = get_last_complete_date(pacer_court_id)
+ if not date_start:
logger.warning(
f"Free opinion scraper for {pacer_court_id} still "
"in progress."
)
- break
+ continue
+ dates = make_date_range_tuples(date_start, date_end, gap=7)
- logger.info(
- "Attempting to get latest document references for "
- "%s between %s and %s",
- pacer_court_id,
- next_start_d,
- next_end_d,
+ # Iterate through the gap in dates either short or long
+ for _start, _end in dates:
+ exc = fetch_doc_report(
+ pacer_court_id, _start, _end # type: ignore
)
- mark_court_in_progress(pacer_court_id, next_end_d)
- try:
- status = get_and_save_free_document_report(
- pacer_court_id, next_start_d, next_end_d
- )
- except (
- RequestException,
- ReadTimeoutError,
- IndexError,
- TypeError,
- PacerLoginException,
- ValueError,
- ) as exc:
- if isinstance(exc, (RequestException, ReadTimeoutError)):
- reason = "network error."
- elif isinstance(exc, IndexError):
- reason = "PACER 6.3 bug."
- elif isinstance(exc, (TypeError, ValueError)):
- reason = "failing PACER website."
- elif isinstance(exc, PacerLoginException):
- reason = "PACER login issue."
- else:
- reason = "unknown reason."
- logger.error(
- "Failed to get free document references for "
- f"{pacer_court_id} between {next_start_d} and "
- f"{next_end_d} due to {reason}.",
- exc_info=True,
- )
- mark_court_done_on_date(
- PACERFreeDocumentLog.SCRAPE_FAILED,
- pacer_court_id,
- next_end_d,
- )
+ if exc:
+ # Something happened with the queried date range, abort process for
+ # that court
+ court_failed = True
break
- mark_court_done_on_date(status, pacer_court_id, next_end_d)
-
- if status == PACERFreeDocumentLog.SCRAPE_SUCCESSFUL:
- if next_end_d >= today.date():
- logger.info(
- "Got all document references for '%s'.", pacer_court_id
- )
- # Break from while loop, onwards to next court
- break
- else:
- # More dates to do; let it continue
- continue
-
- elif status == PACERFreeDocumentLog.SCRAPE_FAILED:
- logger.error(
- "Encountered critical error on %s "
- "(network error?). Marking as failed and "
- "pressing on." % pacer_court_id,
- exc_info=True,
- )
- # Break from while loop, onwards to next court
- break
+ # Wait 1s between queries to try to avoid a possible throttling/blocking
+ # from the court
+ time.sleep(1)
+ if court_failed:
+ continue
-def get_pdfs(options: OptionsType) -> None:
+
+def get_pdfs(
+ courts: list[Optional[str]],
+ date_start: datetime.date,
+ date_end: datetime.date,
+ index: bool,
+ queue: str,
+) -> None:
"""Get PDFs for the results of the Free Document Report queries.
At this stage, we have rows in the PACERFreeDocumentRow table, each of
@@ -199,21 +268,64 @@ def get_pdfs(options: OptionsType) -> None:
In this function, we iterate over the entire table of results, merge it
into our normal tables, and then download and extract the PDF.
+ :param courts: optionally a list of courts to scrape
+ :param date_start: optionally a start date to query all the specified courts or all
+ courts
+ :param date_end: optionally an end date to query all the specified courts or all
+ courts
+ :param index: true if we should index as we process the data or do it later
+ :param queue: the queue name
:return: None
"""
- q = cast(str, options["queue"])
- index = options["index"]
+ q = cast(str, queue)
cnt = CaseNameTweaker()
- rows = PACERFreeDocumentRow.objects.filter(error_msg="").only("pk")
+ base_filter = Q(error_msg="")
+
+ if courts:
+ # Download PDFs only from specified court ids
+ base_filter &= Q(court_id__in=courts)
+
+ if date_start and date_end:
+ # Download documents only from the date range passed from the command args (
+ # sweep)
+ base_filter &= Q(date_filed__gte=date_start, date_filed__lte=date_end)
+
+ # Filter rows based on the base_filter, then annotate each row with a row_number
+ # within each partition defined by 'court_id', ordering the rows by 'pk' in
+ # ascending order. Finally, order the results by 'row_number' and 'court_id' to
+ # download one item for each court until it finishes
+ rows = (
+ PACERFreeDocumentRow.objects.filter(base_filter)
+ .annotate(
+ row_number=Window(
+ expression=RowNumber(),
+ partition_by=[F("court_id")],
+ order_by=F("pk").asc(),
+ )
+ )
+ .order_by("row_number", "court_id")
+ .only("pk", "court_id")
+ )
count = rows.count()
task_name = "downloading"
if index:
task_name += " and indexing"
- logger.info(f"{task_name} {count} items from PACER.")
+ logger.info(
+ f"{task_name} {count} items from PACER from {date_start} to {date_end}."
+ )
throttle = CeleryThrottle(queue_name=q)
completed = 0
+ cycle_checker = CycleChecker()
for row in rows.iterator():
+ # Wait until the queue is short enough
throttle.maybe_wait()
+
+ if cycle_checker.check_if_cycled(row.court_id):
+ print(
+ f"Court cycle completed. Sleep 1 second before starting the next cycle."
+ )
+ time.sleep(1)
+
c = chain(
process_free_opinion_result.si(
row.pk,
@@ -233,19 +345,24 @@ def get_pdfs(options: OptionsType) -> None:
)
-def ocr_available(options: OptionsType) -> None:
- """Do the OCR for any items that need it, then save to the solr index."""
- q = cast(str, options["queue"])
+def ocr_available(queue: str, index: bool) -> None:
+ """Do the OCR for any items that need it, then save to the solr index.
+
+ :param queue: the queue name
+ :param index: true if we should index as we process the data or do it later
+ """
+ q = cast(str, queue)
rds = (
RECAPDocument.objects.filter(ocr_status=RECAPDocument.OCR_NEEDED)
.values_list("pk", flat=True)
.order_by()
)
count = rds.count()
+ logger.info(f"Total documents requiring OCR: {count}")
throttle = CeleryThrottle(queue_name=q)
for i, pk in enumerate(rds):
throttle.maybe_wait()
- if options["index"]:
+ if index:
extract_recap_pdf.si(pk, ocr_available=True).set(
queue=q
).apply_async()
@@ -258,13 +375,24 @@ def ocr_available(options: OptionsType) -> None:
logger.info(f"Sent {i + 1}/{count} tasks to celery so far.")
-def do_everything(options: OptionsType):
+def do_everything(courts, date_start, date_end, index, queue):
+ """Execute the entire process of obtaining the metadata of the free documents,
+ downloading them and ingesting them into the system
+
+ :param courts: optionally a list of courts to scrape
+ :param date_start: optionally a start date to query all the specified courts or all
+ courts
+ :param date_end: optionally an end date to query all the specified courts or all
+ courts
+ :param index: true if we should index as we process the data or do it later
+ :param queue: the queue name
+ """
logger.info("Running and compiling free document reports.")
- get_and_save_free_document_reports(options)
+ get_and_save_free_document_reports(courts, date_start, date_end)
logger.info("Getting PDFs from free document reports")
- get_pdfs(options)
+ get_pdfs(courts, date_start, date_end, index, queue)
logger.info("Doing OCR and saving items to Solr.")
- ocr_available(options)
+ ocr_available(queue, index)
class Command(VerboseCommand):
@@ -279,6 +407,38 @@ def valid_actions(self, s: str) -> Callable:
return self.VALID_ACTIONS[s]
+ def validate_date_args(self, opts):
+ """Validate dates arguments if any
+
+ :param opts: dictionary with arguments from the command
+ :return: true if the date validations are satisfied else false
+ """
+ if not opts.get("date_start") and not opts.get("date_end"):
+ return True
+ elif not opts.get("date_start") or not opts.get("date_end"):
+ logger.error(
+ "Both --date-start and --date-end must be specified together."
+ )
+ return False
+ elif opts.get("date_start") > opts.get("date_end"):
+ logger.error(
+ "--date-end must be greater than or equal to --date-start."
+ )
+ return False
+ return True
+
+ def filter_kwargs(self, func, kwargs):
+ """Keep only the params required to call the function
+
+ :param func: function to be called by the command
+ :param kwargs: dictionary with arguments from the command
+ :return: dictionary with params required for the function
+ """
+ valid_params = inspect.signature(func).parameters.keys()
+ return {
+ key: value for key, value in kwargs.items() if key in valid_params
+ }
+
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--action",
@@ -299,11 +459,37 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
default=False,
help="Do we index as we go, or leave that to be done later?",
)
+ parser.add_argument(
+ "--courts",
+ type=str,
+ default="",
+ nargs="*",
+ help="The courts that you wish to parse. Use cl ids.",
+ )
+ parser.add_argument(
+ "--date-start",
+ dest="date_start",
+ required=False,
+ type=valid_date,
+ help="Date when the query should start.",
+ )
+ parser.add_argument(
+ "--date-end",
+ dest="date_end",
+ required=False,
+ type=valid_date,
+ help="Date when the query should end.",
+ )
def handle(self, *args: List[str], **options: OptionsType) -> None:
super().handle(*args, **options)
+
+ if not self.validate_date_args(options):
+ return
+
action = cast(Callable, options["action"])
- action(options)
+ filtered_kwargs = self.filter_kwargs(action, options)
+ action(**filtered_kwargs)
VALID_ACTIONS: Dict[str, Callable] = {
"do-everything": do_everything,
diff --git a/cl/corpus_importer/management/commands/troller_bk.py b/cl/corpus_importer/management/commands/troller_bk.py
index 9c5a50cf55..6da419151a 100644
--- a/cl/corpus_importer/management/commands/troller_bk.py
+++ b/cl/corpus_importer/management/commands/troller_bk.py
@@ -280,6 +280,9 @@ async def merge_rss_data(
court_id,
docket["pacer_case_id"],
docket["docket_number"],
+ docket.get("federal_defendant_number"),
+ docket.get("federal_dn_judge_initials_assigned"),
+ docket.get("federal_dn_judge_initials_referred"),
)
docket_entry = docket["docket_entries"][0]
document_number = docket["docket_entries"][0]["document_number"]
@@ -669,7 +672,7 @@ def handle(self, *args, **options):
"The 'file' argument is required for that action."
)
- threads: list[threading.Thread] = []
+ threads = []
try:
iterate_and_import_files(options, threads)
except KeyboardInterrupt:
diff --git a/cl/corpus_importer/management/commands/update_opinions_order.py b/cl/corpus_importer/management/commands/update_opinions_order.py
new file mode 100644
index 0000000000..79ad74c038
--- /dev/null
+++ b/cl/corpus_importer/management/commands/update_opinions_order.py
@@ -0,0 +1,277 @@
+import argparse
+import re
+import time
+from typing import Any, List
+
+from bs4 import BeautifulSoup
+from django.db import transaction
+from django.db.models import Count
+
+from cl.lib.command_utils import VerboseCommand, logger
+from cl.search.models import SOURCES, Opinion, OpinionCluster
+
+
+def sort_harvard_opinions(options: dict) -> None:
+ """Sort harvard opinions
+
+ We assume that harvard data is already ordered, we just need to fill
+ the order field in each opinion
+
+ The harvard importer created the opinions in order of appearance in the file
+
+ :param options: dict of arguments passed to the command
+ :return: None
+ """
+
+ skip_until = options.get("skip_until", None)
+ limit = options.get("limit", None)
+
+ # The filepath_json_harvard field can only be filled by the harvard importer,
+ # this helps us confirm that it was imported from a Harvard json. We exclude
+ # clusters merged with columbia because those may need some extra verification
+ harvard_clusters = (
+ OpinionCluster.objects.exclude(filepath_json_harvard="")
+ .annotate(opinions_count=Count("sub_opinions"))
+ .filter(opinions_count__gt=1, source__contains=SOURCES.HARVARD_CASELAW)
+ .exclude(source__contains=SOURCES.COLUMBIA_ARCHIVE)
+ .order_by("id")
+ .values_list("id", flat=True)
+ )
+ if skip_until:
+ harvard_clusters = harvard_clusters.filter(pk__gte=skip_until)
+
+ if limit:
+ harvard_clusters = harvard_clusters[:limit]
+
+ logger.info(f"Harvard clusters to process: {harvard_clusters.count()}")
+
+ completed = 0
+ for cluster_id in harvard_clusters:
+ logger.info(f"Processing cluster id: {cluster_id}")
+ opinion_order = 1
+ any_update = False
+ with transaction.atomic():
+ opinions = Opinion.objects.filter(cluster_id=cluster_id).order_by(
+ "id"
+ )
+ # We need to make sure they are ordered by id
+ for cluster_op in opinions:
+ if cluster_op.type == Opinion.COMBINED:
+ logger.info(
+ f"Ignoring combined opinion in cluster id: {cluster_id}"
+ )
+ continue
+ cluster_op.ordering_key = opinion_order
+ cluster_op.save()
+ opinion_order = opinion_order + 1
+ any_update = True
+ if not any_update:
+ # We want to know if you found anything unexpected, like for example
+ # only having combined opinions
+ logger.info(
+ f"No sub_opinions updated for cluster id: {cluster_id}"
+ )
+ continue
+ logger.info(
+ msg=f"Harvard opinions reordered for cluster id: {cluster_id}"
+ )
+ completed += 1
+ # Wait between each processed cluster to avoid issues with redis memory
+ time.sleep(options["delay"])
+
+ logger.info(f"Processed Harvard clusters: {completed}")
+
+
+def fetch_cleaned_columbia_text(filepath: str) -> str:
+ """Get cleaned columbia content to compare opinions against
+
+ :param filepath: the filepath
+ :return: the opinion text cleaned up
+ """
+ with open(filepath, "r", encoding="utf-8") as f:
+ content = f.read()
+
+ columbia_xml = BeautifulSoup(content, "html.parser")
+ clean_columbia_text = re.sub(r"[^a-zA-Z0-9\s]+", " ", columbia_xml.text)
+ clean_columbia_text = re.sub(r"\s+", " ", clean_columbia_text)
+ return clean_columbia_text
+
+
+def generate_ngrams(words: List[str]) -> List[List[str]]:
+ """Generate n-grams based on the length of the word list.
+
+ Pass in a list of words in an opinion and divide it up into n-grams based on the
+ length of it. For small opinions look for bigrams or single unique words
+
+ Default to a 5 word n-gram unless opinion is very small
+
+ :param words: a list of words obtained splitting the opinion
+ :return: n-grams
+ """
+ width = 5
+ if len(words) <= 5:
+ width = 1
+ elif len(words) < 25:
+ width = 2
+ return [words[i : i + width] for i in range(len(words) - (width - 1))]
+
+
+def match_text(opinions: List[Any], xml_dir: str) -> List[List[Any]]:
+ """Identify a unique set of text in opinions to identify order of opinions
+
+ In a small subset of opinions, duplicate text or bad data fails and assign the
+ end of the index. These opinions are usually short dissents that we add to be the
+ back of the order.
+
+ :param opinions: Opinions to sort
+ :param xml_dir: Path to directory of the xml files
+ :return: Ordered opinion list
+ """
+ _, _, _, local_path = opinions[0]
+ filepath = local_path.replace("/home/mlissner/columbia/opinions", xml_dir)
+
+ columbia_words = fetch_cleaned_columbia_text(filepath=filepath)
+ matches = []
+ for opinion in opinions:
+ opinion_id, opinion_type, opinion_html, _ = opinion
+
+ # assign back up index to the end of the opinion
+ match_index = len(columbia_words)
+
+ soup = BeautifulSoup(opinion_html, "html.parser")
+ words = re.findall(r"\b\w+\b", soup.text)
+ ngrams_to_check = generate_ngrams(words)
+
+ # Check for unique matches in columbia_text
+ for word_group in ngrams_to_check:
+ phrase = " ".join(word_group)
+ if columbia_words.count(phrase) == 1:
+ match_index = columbia_words.find(phrase)
+ break
+ matches.append([opinion_id, opinion_type, match_index])
+ ordered_opinions = sorted(matches, key=lambda x: x[-1])
+ return ordered_opinions
+
+
+def sort_columbia_opinions(options: dict) -> None:
+ """Update opinion ordering for columbia clusters
+
+ :param options: dict of arguments passed to the command
+ :return: None
+ """
+ xml_dir = options["xml_dir"]
+ skip_until = options.get("skip_until", None)
+ limit = options.get("limit", None)
+
+ clusters = (
+ OpinionCluster.objects.filter(
+ source__contains=SOURCES.COLUMBIA_ARCHIVE
+ )
+ .order_by("id")
+ .values_list("id", flat=True)
+ )
+ if skip_until is not None:
+ clusters = clusters.filter(id__gte=skip_until)
+ if limit:
+ clusters = clusters[:limit]
+
+ completed = 0
+ logger.info(f"Columbia clusters to process: {clusters.count()}")
+ for cluster_id in clusters:
+ logger.info(f"Starting opinion cluster: {cluster_id}")
+ opinions = (
+ Opinion.objects.filter(cluster=cluster_id)
+ .exclude(local_path="")
+ .values_list("id", "type", "html_columbia", "local_path")
+ )
+ op_types = [op[1] for op in opinions]
+ if len(opinions) < 2:
+ # Only one opinion is shown, no need to order
+ logger.info(f"Skipping opinion cluster with only one opinion.")
+ continue
+ elif (
+ len(op_types) == 2
+ and Opinion.LEAD in op_types
+ and len(set(op_types)) == 2
+ ):
+ # If only two opinions and one is the lead - assign it to the number 1
+ logger.info(f"Sorting opinions with 1 Lead Opinion.")
+ opinions = [op[:2] for op in opinions]
+ ordered_opinions = sorted(opinions, key=lambda fields: fields[1])
+ else:
+ logger.info(f"Sorting order by location.")
+ ordered_opinions = match_text(opinions, xml_dir)
+
+ ordering_key = 1
+ for op in ordered_opinions:
+ opinion_obj = Opinion.objects.get(id=op[0])
+ opinion_obj.ordering_key = ordering_key
+ opinion_obj.save()
+ ordering_key += 1
+
+ completed += 1
+ logger.info(f"Opinion Cluster completed.")
+
+ # Wait between each processed cluster to avoid issues with redis memory
+ time.sleep(options["delay"])
+
+ logger.info(f"Processed Columbia clusters: {completed}")
+
+
+class Command(VerboseCommand):
+ help = "Add ordering Key for sub opinions"
+
+ def __init__(self, *args, **kwargs):
+ super(Command, self).__init__(*args, **kwargs)
+
+ def valid_actions(self, s):
+ if s.lower() not in self.VALID_ACTIONS:
+ raise argparse.ArgumentTypeError(
+ "Unable to parse action. Valid actions are: %s"
+ % (", ".join(self.VALID_ACTIONS.keys()))
+ )
+
+ return self.VALID_ACTIONS[s]
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ "--skip-until",
+ help="Specific cluster id to skip until",
+ type=int,
+ required=False,
+ )
+ parser.add_argument(
+ "--limit",
+ type=int,
+ help="Number of files to sort",
+ required=False,
+ )
+ parser.add_argument(
+ "--action",
+ type=self.valid_actions,
+ required=True,
+ help="The action you wish to take. Valid choices are: %s"
+ % (", ".join(self.VALID_ACTIONS.keys())),
+ )
+ parser.add_argument(
+ "--delay",
+ type=float,
+ default=0.1,
+ help="How long to wait to update each opinion (in seconds, allows "
+ "floating numbers).",
+ )
+ parser.add_argument(
+ "--xml-dir",
+ default="/opt/courtlistener/columbia/usb",
+ required=False,
+ help="The absolute path to the directory with columbia xml files",
+ )
+
+ def handle(self, *args, **options):
+ super().handle(*args, **options)
+ options["action"](options)
+
+ VALID_ACTIONS = {
+ "sort-harvard": sort_harvard_opinions,
+ "sort-columbia": sort_columbia_opinions,
+ }
diff --git a/cl/corpus_importer/task_canvases.py b/cl/corpus_importer/task_canvases.py
index 143c061417..01ace71b32 100644
--- a/cl/corpus_importer/task_canvases.py
+++ b/cl/corpus_importer/task_canvases.py
@@ -18,7 +18,9 @@
from cl.search.tasks import add_or_update_recap_docket
-def get_docket_and_claims(docket_number, court, case_name, cookies, tags, q):
+def get_docket_and_claims(
+ docket_number, court, case_name, cookies_data, tags, q
+):
"""Get the docket report, claims history report, and save it all to the DB
and Solr
"""
@@ -27,13 +29,13 @@ def get_docket_and_claims(docket_number, court, case_name, cookies, tags, q):
pass_through=None,
docket_number=docket_number,
court_id=court,
- cookies=cookies,
+ session_data=cookies_data,
case_name=case_name,
docket_number_letters="bk",
).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id=court,
- cookies=cookies,
+ session_data=cookies_data,
tag_names=tags,
**{
"show_parties_and_counsel": True,
@@ -41,9 +43,9 @@ def get_docket_and_claims(docket_number, court, case_name, cookies, tags, q):
"show_list_of_member_cases": False,
}
).set(queue=q),
- get_bankr_claims_registry.s(cookies=cookies, tag_names=tags).set(
- queue=q
- ),
+ get_bankr_claims_registry.s(
+ session_data=cookies_data, tag_names=tags
+ ).set(queue=q),
add_or_update_recap_docket.s().set(queue=q),
).apply_async()
@@ -72,7 +74,7 @@ def get_district_attachment_pages(options, rd_pks, tag_names, session):
break
throttle.maybe_wait()
chain(
- get_attachment_page_by_rd.s(rd_pk, session.cookies).set(queue=q),
+ get_attachment_page_by_rd.s(rd_pk, session).set(queue=q),
make_attachment_pq_object.s(rd_pk, recap_user.pk).set(queue=q),
process_recap_attachment.s(tag_names=tag_names).set(queue=q),
).apply_async()
diff --git a/cl/corpus_importer/tasks.py b/cl/corpus_importer/tasks.py
index 010239917e..e00092b791 100644
--- a/cl/corpus_importer/tasks.py
+++ b/cl/corpus_importer/tasks.py
@@ -46,7 +46,6 @@
from pyexpat import ExpatError
from redis import ConnectionError as RedisConnectionError
from requests import Response
-from requests.cookies import RequestsCookieJar
from requests.exceptions import (
ConnectionError,
HTTPError,
@@ -83,6 +82,7 @@
)
from cl.lib.pacer_session import (
ProxyPacerSession,
+ SessionData,
get_or_cache_pacer_cookies,
get_pacer_cookie_from_cache,
)
@@ -323,28 +323,27 @@ def download_recap_item(
soft_time_limit=240,
)
def get_and_save_free_document_report(
- self: Task,
- court_id: str,
- start: date,
- end: date,
-) -> int:
+ self: Task, court_id: str, start: date, end: date, log_id: int = 0
+) -> Tuple[int, int]:
"""Download the Free document report and save it to the DB.
:param self: The Celery task.
:param court_id: A pacer court id.
:param start: a date object representing the first day to get results.
:param end: a date object representing the last day to get results.
+ :param log_id: a PACERFreeDocumentLog object id
:return: The status code of the scrape
"""
- cookies = get_or_cache_pacer_cookies(
+ session_data = get_or_cache_pacer_cookies(
"pacer_scraper",
username=settings.PACER_USERNAME,
password=settings.PACER_PASSWORD,
)
s = ProxyPacerSession(
- cookies=cookies,
+ cookies=session_data.cookies,
username=settings.PACER_USERNAME,
password=settings.PACER_PASSWORD,
+ proxy=session_data.proxy_address,
)
report = FreeOpinionReport(court_id, s)
msg = ""
@@ -398,6 +397,29 @@ def get_and_save_free_document_report(
return PACERFreeDocumentLog.SCRAPE_FAILED
raise self.retry(exc=exc, countdown=5)
+ if log_id:
+ # We only save the html when the script is run automatically every day
+ log = PACERFreeDocumentLog.objects.get(pk=log_id)
+ if hasattr(report, "responses_with_params"):
+ for result in report.responses_with_params:
+ # FreeOpinionReport now also returns a list of dicts with additional
+ # data instead of a list of requests responses. We do this to verify
+ # if we have the new version of juriscraper with the new attribute.
+ if isinstance(result, dict):
+ response = result.get("response")
+ query_start = result.get("start")
+ query_end = result.get("end")
+
+ if response and query_start and query_end:
+ pacer_file = PacerHtmlFiles(
+ content_object=log,
+ upload_type=UPLOAD_TYPE.FREE_OPINIONS_REPORT,
+ )
+ pacer_file.filepath.save(
+ f"free_opinions_report_{court_id}_from_{query_start.replace('/', '-')}_to_{query_end.replace('/', '-')}.html",
+ ContentFile(response.text.encode()),
+ )
+
document_rows_to_create = []
for row in results:
document_row = PACERFreeDocumentRow(
@@ -418,7 +440,7 @@ def get_and_save_free_document_report(
# Create PACERFreeDocumentRow in bulk
PACERFreeDocumentRow.objects.bulk_create(document_rows_to_create)
- return PACERFreeDocumentLog.SCRAPE_SUCCESSFUL
+ return PACERFreeDocumentLog.SCRAPE_SUCCESSFUL, len(document_rows_to_create)
@app.task(bind=True, max_retries=5, ignore_result=True)
@@ -606,14 +628,18 @@ def get_and_process_free_pdf(
return None
raise self.retry()
- cookies = get_or_cache_pacer_cookies(
+ cookies_data = get_or_cache_pacer_cookies(
"pacer_scraper",
username=settings.PACER_USERNAME,
password=settings.PACER_PASSWORD,
)
try:
r, r_msg = download_pacer_pdf_by_rd(
- rd.pk, result.pacer_case_id, result.pacer_doc_id, cookies
+ rd.pk,
+ result.pacer_case_id,
+ result.pacer_doc_id,
+ cookies_data,
+ de_seq_num=rd.docket_entry.pacer_sequence_number,
)
except HTTPError as exc:
if exc.response and exc.response.status_code in [
@@ -867,18 +893,12 @@ def upload_to_ia(
@app.task
-def mark_court_done_on_date(
- status: int, court_id: str, d: date
-) -> Optional[int]:
- court_id = map_pacer_to_cl_id(court_id)
+def mark_court_done_on_date(log_id: int, status: int) -> Optional[int]:
try:
- doc_log = PACERFreeDocumentLog.objects.filter(
- status=PACERFreeDocumentLog.SCRAPE_IN_PROGRESS, court_id=court_id
- ).latest("date_queried")
+ doc_log = PACERFreeDocumentLog.objects.get(pk=log_id)
except PACERFreeDocumentLog.DoesNotExist:
return None
else:
- doc_log.date_queried = d
doc_log.status = status
doc_log.date_completed = now()
doc_log.save()
@@ -939,12 +959,12 @@ def get_pacer_case_id_and_title(
pass_through: Any,
docket_number: str,
court_id: str,
- cookies: Optional[RequestsCookieJar] = None,
- user_pk: Optional[int] = None,
- case_name: Optional[str] = None,
- office_number: Optional[str] = None,
- docket_number_letters: Optional[str] = None,
-) -> Optional[TaskData]:
+ session_data: SessionData | None = None,
+ user_pk: int | None = None,
+ case_name: str | None = None,
+ office_number: str | None = None,
+ docket_number_letters: str | None = None,
+) -> TaskData | None:
"""Get the pacer_case_id and title values for a district court docket. Use
heuristics to disambiguate the results.
@@ -960,8 +980,8 @@ def get_pacer_case_id_and_title(
:param docket_number: The docket number to look up. This is a flexible
field that accepts a variety of docket number styles.
:param court_id: The CourtListener court ID for the docket number
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-in PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param user_pk: The PK of a user making the request. This can be provided
instead of the cookies parameter. If so, this will get the user's cookies
from redis instead of passing them in as an argument.
@@ -989,10 +1009,19 @@ def get_pacer_case_id_and_title(
docket_number,
court_id,
)
- if not cookies:
- # Get cookies from Redis if not provided
- cookies = get_pacer_cookie_from_cache(user_pk) # type: ignore
- s = ProxyPacerSession(cookies=cookies)
+
+ if not session_data and user_pk:
+ session_data = get_pacer_cookie_from_cache(user_pk)
+ if not session_data:
+ raise Exception("Cookies not available in cache")
+ else:
+ raise Exception(
+ "user_pk is unavailable, cookies cannot be retrieved from cache"
+ )
+
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
report = PossibleCaseNumberApi(map_cl_to_pacer_id(court_id), s)
msg = ""
try:
@@ -1041,9 +1070,9 @@ def do_case_query_by_pacer_case_id(
self: Task,
data: TaskData,
court_id: str,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
tag_names: List[str] | None = None,
-) -> Optional[TaskData]:
+) -> TaskData | None:
"""Run a case query (iquery.pl) query on a case and save the data
:param self: The celery task
@@ -1051,13 +1080,15 @@ def do_case_query_by_pacer_case_id(
'pacer_case_id': The internal pacer case ID for the item.
}
:param court_id: A courtlistener court ID
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-in PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param tag_names: A list of tag names to associate with the docket when
saving it in the DB.
:return: A dict with the pacer_case_id and docket_pk values.
"""
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
if data is None:
logger.info("Empty data argument. Terminating chains and exiting.")
self.request.chain = None
@@ -1087,14 +1118,19 @@ def do_case_query_by_pacer_case_id(
# Merge the contents into CL.
if d is None:
d = async_to_sync(find_docket_object)(
- court_id, pacer_case_id, docket_data["docket_number"]
+ court_id,
+ pacer_case_id,
+ docket_data["docket_number"],
+ docket_data.get("federal_defendant_number"),
+ docket_data.get("federal_dn_judge_initials_assigned"),
+ docket_data.get("federal_dn_judge_initials_referred"),
)
d.add_recap_source()
async_to_sync(update_docket_metadata)(d, docket_data)
d.save()
- add_tags_to_objs(tag_names, [d])
+ async_to_sync(add_tags_to_objs)(tag_names, [d])
# Add the HTML to the docket in case we need it someday.
pacer_file = PacerHtmlFiles(
@@ -1158,27 +1194,28 @@ def filter_docket_by_tags(
def query_case_query_report(
court_id: str, pacer_case_id: int
-) -> dict[str, Any]:
+) -> tuple[dict[str, Any], str]:
"""Query the iquery page for a given PACER case ID.
:param court_id: A CL court ID where we'll look things up.
:param pacer_case_id: The Pacer Case ID to lookup.
- :return: The report.data.
+ :return: A two tuple, the report data and the report HTML text.
"""
- cookies = get_or_cache_pacer_cookies(
+ session_data = get_or_cache_pacer_cookies(
"pacer_scraper",
settings.PACER_USERNAME,
password=settings.PACER_PASSWORD,
)
s = ProxyPacerSession(
- cookies=cookies,
+ cookies=session_data.cookies,
username=settings.PACER_USERNAME,
password=settings.PACER_PASSWORD,
+ proxy=session_data.proxy_address,
)
report = CaseQuery(map_cl_to_pacer_id(court_id), s)
report.query(pacer_case_id)
- return report.data
+ return report.data, report.response.text
def make_docket_by_iquery_base(
@@ -1207,7 +1244,9 @@ def make_docket_by_iquery_base(
"""
try:
- report_data = query_case_query_report(court_id, pacer_case_id)
+ report_data, report_text = query_case_query_report(
+ court_id, pacer_case_id
+ )
except (requests.Timeout, requests.RequestException) as exc:
logger.warning(
"Timeout or unknown RequestException on iquery crawl. "
@@ -1237,6 +1276,9 @@ def make_docket_by_iquery_base(
court_id,
str(pacer_case_id),
report_data["docket_number"],
+ report_data.get("federal_defendant_number"),
+ report_data.get("federal_dn_judge_initials_assigned"),
+ report_data.get("federal_dn_judge_initials_referred"),
using=using,
)
@@ -1245,6 +1287,7 @@ def make_docket_by_iquery_base(
return save_iquery_to_docket(
self,
report_data,
+ report_text,
d,
tag_names,
add_to_solr=True,
@@ -1348,24 +1391,25 @@ def make_docket_by_iquery_sweep(
@retry((requests.Timeout, PacerLoginException), tries=3, delay=0.25, backoff=1)
def query_iquery_page(
court_id: str, pacer_case_id: int
-) -> bool | dict[str, Any]:
+) -> tuple[bool, None] | tuple[dict[str, Any], str]:
"""A small wrapper to query the iquery page for a given PACER case ID to
support retries via the @retry decorator in case of a failure.
:param court_id: A CL court ID where we'll look things up.
:param pacer_case_id: The Pacer Case ID to lookup.
- :return: False if not valid report, otherwise the report.data.
+ :return: A two tuple, False and None if not a valid report or the report data
+ and the report HTML text.
"""
- report_data = query_case_query_report(court_id, pacer_case_id)
+ report_data, report_text = query_case_query_report(court_id, pacer_case_id)
if not report_data:
logger.info(
"No valid data found in iquery page for %s.%s",
court_id,
pacer_case_id,
)
- return False
- return report_data
+ return False, None
+ return report_data, report_text
@app.task(
@@ -1402,7 +1446,9 @@ def probe_iquery_pages(
)
probe_iteration += 1
try:
- report_data = query_iquery_page(court_id, pacer_case_id_to_lookup)
+ report_data, report_text = query_iquery_page(
+ court_id, pacer_case_id_to_lookup
+ )
except HTTPError:
# Set expiration accordingly and value to 2 to difference from
# other waiting times.
@@ -1456,7 +1502,9 @@ def probe_iquery_pages(
if report_data:
# Find and update/store the Docket.
- reports_data.append((pacer_case_id_to_lookup, report_data))
+ reports_data.append(
+ (pacer_case_id_to_lookup, report_data, report_text)
+ )
latest_match = pacer_case_id_to_lookup
found_match = True
# Restart court_blocked_attempts and court_empty_probe_attempts.
@@ -1496,15 +1544,17 @@ def probe_iquery_pages(
# Process all the reports retrieved during the probing.
# Avoid triggering the iQuery sweep signal except for the latest hit.
avoid_trigger_signal = True
- for index, report_data in enumerate(reports_data):
+ for index, report_content in enumerate(reports_data):
+ pacer_case_id, report_data, report_text = report_content
if index == len(reports_data) - 1:
# Only trigger the sweep signal on the last hit.
avoid_trigger_signal = False
try:
process_case_query_report(
court_id,
- pacer_case_id=report_data[0],
- report_data=report_data[1],
+ pacer_case_id=pacer_case_id,
+ report_data=report_data,
+ report_text=report_text,
avoid_trigger_signal=avoid_trigger_signal,
)
except IntegrityError:
@@ -1533,11 +1583,11 @@ def get_docket_by_pacer_case_id(
self: Task,
data: TaskData,
court_id: str,
- cookies: Optional[RequestsCookieJar] = None,
+ session_data: SessionData,
docket_pk: Optional[int] = None,
tag_names: Optional[str] = None,
**kwargs,
-) -> Optional[TaskData]:
+) -> TaskData | None:
"""Get a docket by PACER case id, CL court ID, and a collection of kwargs
that can be passed to the DocketReport query.
@@ -1549,8 +1599,8 @@ def get_docket_by_pacer_case_id(
Optional: 'docket_pk': The ID of the docket to work on to avoid lookups
if it's known in advance.
:param court_id: A courtlistener court ID.
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-in PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param docket_pk: The PK of the docket to update. Can also be provided in
the data param, above.
:param tag_names: A list of tag names that should be stored with the item
@@ -1584,7 +1634,9 @@ def get_docket_by_pacer_case_id(
logging_id = f"{court_id}.{pacer_case_id}"
logger.info("Querying docket report %s", logging_id)
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
report = DocketReport(map_cl_to_pacer_id(court_id), s)
try:
report.query(pacer_case_id, **kwargs)
@@ -1606,7 +1658,12 @@ def get_docket_by_pacer_case_id(
if d is None:
d = async_to_sync(find_docket_object)(
- court_id, pacer_case_id, docket_data["docket_number"]
+ court_id,
+ pacer_case_id,
+ docket_data["docket_number"],
+ docket_data.get("federal_defendant_number"),
+ docket_data.get("federal_dn_judge_initials_assigned"),
+ docket_data.get("federal_dn_judge_initials_referred"),
)
rds_created, content_updated = merge_pacer_docket_into_cl_docket(
@@ -1635,7 +1692,7 @@ def get_appellate_docket_by_docket_number(
self: Task,
docket_number: str,
court_id: str,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
tag_names: Optional[List[str]] = None,
**kwargs,
) -> Optional[TaskData]:
@@ -1647,13 +1704,16 @@ def get_appellate_docket_by_docket_number(
:param self: The celery task
:param docket_number: The docket number of the case.
:param court_id: A courtlistener/PACER appellate court ID.
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-in PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param tag_names: The tag name that should be stored with the item in the
DB, if desired.
:param kwargs: A variety of keyword args to pass to DocketReport.query().
"""
- s = ProxyPacerSession(cookies=cookies)
+
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
report = AppellateDocketReport(court_id, s)
logging_id = f"{court_id} - {docket_number}"
logger.info("Querying docket report %s", logging_id)
@@ -1684,7 +1744,12 @@ def get_appellate_docket_by_docket_number(
if d is None:
d = async_to_sync(find_docket_object)(
- court_id, docket_number, docket_number
+ court_id,
+ docket_number,
+ docket_number,
+ docket_data.get("federal_defendant_number"),
+ docket_data.get("federal_dn_judge_initials_assigned"),
+ docket_data.get("federal_dn_judge_initials_referred"),
)
rds_created, content_updated = merge_pacer_docket_into_cl_docket(
@@ -1703,20 +1768,21 @@ def get_appellate_docket_by_docket_number(
def get_att_report_by_rd(
rd: RECAPDocument,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
) -> Optional[AttachmentPage]:
"""Method to get the attachment report for the item in PACER.
:param rd: The RECAPDocument object to use as a source.
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-on PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:return: The attachment report populated with the results
"""
-
if not rd.pacer_doc_id:
return None
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
pacer_court_id = map_cl_to_pacer_id(rd.docket_entry.docket.court_id)
att_report = AttachmentPage(pacer_court_id, s)
att_report.query(rd.pacer_doc_id)
@@ -1734,14 +1800,14 @@ def get_att_report_by_rd(
def get_attachment_page_by_rd(
self: Task,
rd_pk: int,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
) -> Optional[AttachmentPage]:
"""Get the attachment page for the item in PACER.
:param self: The celery task
:param rd_pk: The PK of a RECAPDocument object to use as a source.
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-on PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:return: The attachment report populated with the results
"""
rd = RECAPDocument.objects.get(pk=rd_pk)
@@ -1750,7 +1816,7 @@ def get_attachment_page_by_rd(
self.request.chain = None
return None
try:
- att_report = get_att_report_by_rd(rd, cookies)
+ att_report = get_att_report_by_rd(rd, session_data)
except HTTPError as exc:
if exc.response and exc.response.status_code in [
HTTPStatus.INTERNAL_SERVER_ERROR,
@@ -1788,21 +1854,24 @@ def get_attachment_page_by_rd(
def get_bankr_claims_registry(
self: Task,
data: TaskData,
- cookies: RequestsCookieJar,
- tag_names: Optional[List[str]] = None,
-) -> Optional[TaskData]:
+ session_data: SessionData,
+ tag_names: List[str] | None = None,
+) -> TaskData | None:
"""Get the bankruptcy claims registry for a docket
:param self: The celery task
:param data: A dict of data containing, primarily, a key to 'docket_pk' for
the docket for which we want to get the registry. Other keys will be
ignored.
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-in PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param tag_names: A list of tag names that should be stored with the claims
registry information in the DB.
"""
- s = ProxyPacerSession(cookies=cookies)
+
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
if data is None or data.get("docket_pk") is None:
logger.warning(
"Empty data argument or parameter. Terminating chains "
@@ -1900,8 +1969,9 @@ def download_pacer_pdf_by_rd(
rd_pk: int,
pacer_case_id: str,
pacer_doc_id: int,
- cookies: RequestsCookieJar,
- magic_number: Optional[str] = None,
+ session_data: SessionData,
+ magic_number: str | None = None,
+ de_seq_num: str | None = None,
) -> tuple[Response | None, str]:
"""Using a RECAPDocument object ID, download the PDF if it doesn't already
exist.
@@ -1909,21 +1979,24 @@ def download_pacer_pdf_by_rd(
:param rd_pk: The PK of the RECAPDocument to download
:param pacer_case_id: The internal PACER case ID number
:param pacer_doc_id: The internal PACER document ID to download
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-in PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param magic_number: The magic number to fetch PACER documents for free
this is an optional field, only used by RECAP Email documents
:return: A two-tuple of requests.Response object usually containing a PDF,
or None if that wasn't possible, and a string representing the error if
there was one.
"""
-
rd = RECAPDocument.objects.get(pk=rd_pk)
pacer_court_id = map_cl_to_pacer_id(rd.docket_entry.docket.court_id)
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
report = FreeOpinionReport(pacer_court_id, s)
- r, r_msg = report.download_pdf(pacer_case_id, pacer_doc_id, magic_number)
+ r, r_msg = report.download_pdf(
+ pacer_case_id, pacer_doc_id, magic_number, de_seq_num=de_seq_num
+ )
return r, r_msg
@@ -1932,27 +2005,32 @@ def download_pdf_by_magic_number(
court_id: str,
pacer_doc_id: str,
pacer_case_id: str,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
magic_number: str,
appellate: bool = False,
+ de_seq_num: str | None = None,
) -> tuple[Response | None, str]:
"""Small wrapper to fetch a PACER PDF document by magic number.
:param court_id: A CourtListener court ID to query the free document.
:param pacer_doc_id: The pacer_doc_id to query the free document.
:param pacer_case_id: The pacer_case_id to query the free document.
- :param cookies: The cookies of a logged in PACER session
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param magic_number: The magic number to fetch PACER documents for free.
:param appellate: Whether the download belongs to an appellate court.
+ :param de_seq_num: The sequential number assigned by the PACER system to
+ identify the docket entry within a case.
:return: A two-tuple of requests.Response object usually containing a PDF,
or None if that wasn't possible, and a string representing the error if
there was one.
"""
-
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
report = FreeOpinionReport(court_id, s)
r, r_msg = report.download_pdf(
- pacer_case_id, pacer_doc_id, magic_number, appellate
+ pacer_case_id, pacer_doc_id, magic_number, appellate, de_seq_num
)
return r, r_msg
@@ -1968,10 +2046,12 @@ def get_document_number_from_confirmation_page(
"""
recap_email_user = User.objects.get(username="recap-email")
- cookies = get_or_cache_pacer_cookies(
+ session_data = get_or_cache_pacer_cookies(
recap_email_user.pk, settings.PACER_USERNAME, settings.PACER_PASSWORD
)
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
doc_num_report = DownloadConfirmationPage(court_id, s)
doc_num_report.query(pacer_doc_id)
data = doc_num_report.data
@@ -2042,11 +2122,12 @@ def is_pacer_doc_sealed(court_id: str, pacer_doc_id: str) -> bool:
"""
recap_email_user = User.objects.get(username="recap-email")
- cookies = get_or_cache_pacer_cookies(
+ session_data = get_or_cache_pacer_cookies(
recap_email_user.pk, settings.PACER_USERNAME, settings.PACER_PASSWORD
)
-
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
receipt_report = DownloadConfirmationPage(court_id, s)
receipt_report.query(pacer_doc_id)
data = receipt_report.data
@@ -2073,11 +2154,13 @@ def is_docket_entry_sealed(
return False
recap_email_user = User.objects.get(username="recap-email")
- cookies = get_or_cache_pacer_cookies(
+ session_data = get_or_cache_pacer_cookies(
recap_email_user.pk, settings.PACER_USERNAME, settings.PACER_PASSWORD
)
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
report = BaseReport(court_id, s)
return report.is_entry_sealed(case_id, doc_id)
@@ -2180,14 +2263,15 @@ def add_tags(rd: RECAPDocument, tag_name: Optional[str]) -> None:
def get_pacer_doc_by_rd(
self: Task,
rd_pk: int,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
tag: Optional[str] = None,
) -> Optional[int]:
"""A simple method for getting the PDF associated with a RECAPDocument.
:param self: The bound celery task
:param rd_pk: The PK for the RECAPDocument object
- :param cookies: The cookies of a logged in PACER session
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param tag: The name of a tag to apply to any modified items
:return: The RECAPDocument PK
"""
@@ -2199,8 +2283,13 @@ def get_pacer_doc_by_rd(
return None
pacer_case_id = rd.docket_entry.docket.pacer_case_id
+ de_seq_num = rd.docket_entry.pacer_sequence_number
r, r_msg = download_pacer_pdf_by_rd(
- rd.pk, pacer_case_id, rd.pacer_doc_id, cookies
+ rd.pk,
+ pacer_case_id,
+ rd.pacer_doc_id,
+ session_data,
+ de_seq_num=de_seq_num,
)
court_id = rd.docket_entry.docket.court_id
@@ -2238,7 +2327,7 @@ def get_pacer_doc_by_rd_and_description(
self: Task,
rd_pk: int,
description_re: Pattern,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
fallback_to_main_doc: bool = False,
tag_name: Optional[List[str]] = None,
) -> None:
@@ -2252,15 +2341,15 @@ def get_pacer_doc_by_rd_and_description(
:param rd_pk: The PK of a RECAPDocument object to use as a source.
:param description_re: A compiled regular expression to search against the
description provided by the attachment page.
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-in PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param fallback_to_main_doc: Should we grab the main doc if none of the
attachments match the regex?
:param tag_name: A tag name to apply to any downloaded content.
:return: None
"""
rd = RECAPDocument.objects.get(pk=rd_pk)
- att_report = get_attachment_page_by_rd(self, rd_pk, cookies)
+ att_report = get_attachment_page_by_rd(self, rd_pk, session_data)
att_found = None
for attachment in att_report.data.get("attachments", []):
@@ -2308,8 +2397,13 @@ def get_pacer_doc_by_rd_and_description(
return
pacer_case_id = rd.docket_entry.docket.pacer_case_id
+ de_seq_num = rd.docket_entry.pacer_sequence_number
r, r_msg = download_pacer_pdf_by_rd(
- rd.pk, pacer_case_id, att_found["pacer_doc_id"], cookies
+ rd.pk,
+ pacer_case_id,
+ att_found["pacer_doc_id"],
+ session_data,
+ de_seq_num=de_seq_num,
)
court_id = rd.docket_entry.docket.court_id
@@ -2347,18 +2441,20 @@ def get_pacer_doc_by_rd_and_description(
def get_pacer_doc_id_with_show_case_doc_url(
self: Task,
rd_pk: int,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
) -> None:
"""use the show_case_doc URL to get pacer_doc_id values.
:param self: The celery task
:param rd_pk: The pk of the RECAPDocument you want to get.
- :param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
- logged-in PACER user.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
"""
rd = RECAPDocument.objects.get(pk=rd_pk)
d = rd.docket_entry.docket
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
pacer_court_id = map_cl_to_pacer_id(d.court_id)
report = ShowCaseDocApi(pacer_court_id, s)
last_try = self.request.retries == self.max_retries
@@ -2448,7 +2544,7 @@ def make_list_of_creditors_key(court_id: str, d_number_file_name: str) -> str:
@throttle_task("1/s", key="court_id")
def query_and_save_list_of_creditors(
self: Task,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
court_id: str,
d_number_file_name: str,
docket_number: str,
@@ -2460,7 +2556,8 @@ def query_and_save_list_of_creditors(
HTML and pipe-limited text files and convert them to CSVs.
:param self: The celery task
- :param cookies: The cookies for the current PACER session.
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param court_id: The court_id for the bankruptcy court.
:param d_number_file_name: The docket number to use as file name.
:param docket_number: The docket number of the case.
@@ -2470,8 +2567,9 @@ def query_and_save_list_of_creditors(
:return: None
"""
-
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
try:
report = ListOfCreditors(court_id, s)
except AssertionError:
diff --git a/cl/corpus_importer/tests.py b/cl/corpus_importer/tests.py
index 6d3ebd931a..7a76435ded 100644
--- a/cl/corpus_importer/tests.py
+++ b/cl/corpus_importer/tests.py
@@ -94,7 +94,7 @@
find_just_name,
)
from cl.people_db.models import Attorney, AttorneyOrganization, Party
-from cl.recap.models import UPLOAD_TYPE
+from cl.recap.models import UPLOAD_TYPE, PacerHtmlFiles
from cl.recap_rss.models import RssItemCache
from cl.scrapers.models import PACERFreeDocumentRow
from cl.search.factories import (
@@ -402,6 +402,9 @@ def test_get_appellate_court_object_from_string(self) -> None:
self.assertEqual(test["a"], got)
+@override_settings(
+ EGRESS_PROXY_HOSTS=["http://proxy_1:9090", "http://proxy_2:9090"]
+)
@pytest.mark.django_db
class PacerDocketParserTest(TestCase):
"""Can we parse RECAP dockets successfully?"""
@@ -496,10 +499,7 @@ def test_party_parsing(self) -> None:
self.assertEqual(godfrey_llp.city, "Seattle")
self.assertEqual(godfrey_llp.state, "WA")
- @patch(
- "cl.corpus_importer.tasks.get_or_cache_pacer_cookies",
- return_value=None,
- )
+ @patch("cl.corpus_importer.tasks.get_or_cache_pacer_cookies")
def test_get_and_save_free_document_report(self, mock_cookies) -> None:
"""Test the retrieval and storage of free document report data."""
@@ -3341,13 +3341,11 @@ def test_merger(self):
)
-@patch(
- "cl.corpus_importer.tasks.get_or_cache_pacer_cookies",
- return_value=None,
-)
+@patch("cl.corpus_importer.tasks.get_or_cache_pacer_cookies")
@override_settings(
IQUERY_PROBE_DAEMON_ENABLED=True,
IQUERY_SWEEP_UPLOADS_SIGNAL_ENABLED=True,
+ EGRESS_PROXY_HOSTS=["http://proxy_1:9090", "http://proxy_2:9090"],
)
class ScrapeIqueryPagesTest(TestCase):
"""Tests related to probe_iquery_pages_daemon command."""
@@ -3692,6 +3690,7 @@ def test_update_latest_case_id_and_schedule_iquery_sweep_integration(
dispatch_uid=test_dispatch_uid,
)
try:
+ pacer_files = PacerHtmlFiles.objects.all()
dockets = Docket.objects.filter(court_id=self.court_gand)
self.assertEqual(dockets.count(), 0)
@@ -3797,6 +3796,9 @@ def test_update_latest_case_id_and_schedule_iquery_sweep_integration(
self.assertEqual(
dockets.count(), 5, msg="Wrong number of dockets returned."
)
+ # Two PACER HTML files should be stored by now via iquery sweep.
+ self.assertEqual(2, pacer_files.count())
+
highest_known_pacer_case_id = r.hget(
"iquery:highest_known_pacer_case_id", self.court_gand.pk
)
@@ -3859,6 +3861,14 @@ def test_update_latest_case_id_and_schedule_iquery_sweep_integration(
self.assertEqual(
dockets.count(), 5, msg="Docket number doesn't match."
)
+ # 7 additional PACER HTML files should be stored by now, 3 added by the
+ # probing task + 4 added by the sweep task.
+ pacer_files = PacerHtmlFiles.objects.all()
+ self.assertEqual(9, pacer_files.count())
+ # Assert HTML content was properly stored in one of them.
+ self.assertEqual(
+ "Test", pacer_files[0].filepath.read().decode()
+ )
### Integration test probing task + sweep
# IQUERY_SWEEP_UPLOADS_SIGNAL_ENABLED False
diff --git a/cl/corpus_importer/utils.py b/cl/corpus_importer/utils.py
index c60696e795..e97668f56e 100644
--- a/cl/corpus_importer/utils.py
+++ b/cl/corpus_importer/utils.py
@@ -1,6 +1,7 @@
import itertools
import random
import re
+from collections import defaultdict
from datetime import date
from difflib import SequenceMatcher
from typing import Any, Iterator, Optional, Set
@@ -612,11 +613,14 @@ def merge_overlapping_data(
return data_to_update
-def add_citations_to_cluster(cites: list[str], cluster_id: int) -> None:
+def add_citations_to_cluster(
+ cites: list[str], cluster_id: int, save_again_if_exists: bool = False
+) -> None:
"""Add string citations to OpinionCluster if it has not yet been added
:param cites: citation list
:param cluster_id: cluster id related to citations
+ :param save_again_if_exists: force save citation if it already exists
:return: None
"""
for cite in cites:
@@ -636,29 +640,46 @@ def add_citations_to_cluster(cites: list[str], cluster_id: int) -> None:
cite_type_str = citation[0].all_editions[0].reporter.cite_type
reporter_type = map_reporter_db_cite_type(cite_type_str)
- if Citation.objects.filter(
- cluster_id=cluster_id, reporter=citation[0].corrected_reporter()
- ).exists():
- # Avoid adding a citation if we already have a citation from the
- # citation's reporter
- continue
-
- try:
- o, created = Citation.objects.get_or_create(
- volume=citation[0].groups["volume"],
- reporter=citation[0].corrected_reporter(),
- page=citation[0].groups["page"],
- type=reporter_type,
+ citation_params = {
+ "volume": citation[0].groups["volume"],
+ "reporter": citation[0].corrected_reporter(),
+ "page": citation[0].groups["page"],
+ "type": reporter_type,
+ "cluster_id": cluster_id,
+ }
+ citation_obj = Citation.objects.filter(**citation_params).first()
+ if citation_obj:
+ if save_again_if_exists:
+ # We already have the citation for the cluster and want to reindex it
+ citation_obj.save()
+ logger.info(
+ f"Reindexing: {cite} added to cluster id: {cluster_id}"
+ )
+ else:
+ # Ignore and go to the next citation in the list
+ continue
+ else:
+ if Citation.objects.filter(
cluster_id=cluster_id,
- )
- if created:
+ reporter=citation[0].corrected_reporter(),
+ ).exists():
+ # Avoid adding a citation if we already have a citation from the
+ # citation's reporter.
+ logger.info(
+ f"Can't add: {cite} to cluster id: {cluster_id}. There is already "
+ f"a citation from that reporter."
+ )
+ continue
+ try:
+ # We don't have the citation or any citation from the reporter
+ Citation.objects.create(**citation_params)
logger.info(
f"New citation: {cite} added to cluster id: {cluster_id}"
)
- except IntegrityError:
- logger.warning(
- f"Reporter mismatch for cluster: {cluster_id} on cite: {cite}"
- )
+ except IntegrityError:
+ logger.warning(
+ f"Reporter mismatch for cluster: {cluster_id} on cite: {cite}"
+ )
def update_cluster_panel(
@@ -1088,3 +1109,54 @@ def compute_blocked_court_wait(court_blocked_attempts: int) -> tuple[int, int]:
for i in range(court_blocked_attempts)
)
return current_wait_time, total_accumulated_time
+
+
+class CycleChecker:
+ """Keep track of a cycling list to determine each time it starts over.
+
+ We plan to iterate over dockets that are ordered by a cycling court ID, so
+ imagine if we had two courts, ca1 and ca2, we'd have rows like:
+
+ docket: 1, court: ca1
+ docket: 14, court: ca2
+ docket: 15, court: ca1
+ docket: xx, court: ca2
+
+ In other words, they'd just go back and forth. In reality, we have about
+ 200 courts, but the idea is the same. This code lets us detect each time
+ the cycle has started over, even if courts stop being part of the cycle,
+ as will happen towards the end of the queryset.. For example, maybe ca1
+ finishes, and now we just have:
+
+ docket: x, court: ca2
+ docket: y, court: ca2
+ docket: z, court: ca2
+
+ That's considered cycling each time we get to a new row.
+
+ The way to use this is to just create an instance and then send it a
+ cycling list of court_id's.
+
+ Other fun requirements this hits:
+ - No need to know the length of the cycle
+ - No need to externally track the iteration count
+ """
+
+ def __init__(self) -> None:
+ self.court_counts: defaultdict = defaultdict(int)
+ self.current_iteration: int = 1
+
+ def check_if_cycled(self, court_id: str) -> bool:
+ """Check if the cycle repeated
+
+ :param court_id: The ID of the court
+ :return True if the cycle started over, else False
+ """
+ self.court_counts[court_id] += 1
+ if self.court_counts[court_id] == self.current_iteration:
+ return False
+ else:
+ # Finished cycle and court has been seen more times than the
+ # iteration count. Bump the iteration count and return True.
+ self.current_iteration += 1
+ return True
diff --git a/cl/custom_filters/templatetags/extras.py b/cl/custom_filters/templatetags/extras.py
index 40d2813cda..0a35e6ba8e 100644
--- a/cl/custom_filters/templatetags/extras.py
+++ b/cl/custom_filters/templatetags/extras.py
@@ -1,5 +1,6 @@
import random
import re
+from datetime import date, datetime
from django import template
from django.core.exceptions import ValidationError
@@ -10,7 +11,7 @@
from django.utils.safestring import SafeString, mark_safe
from elasticsearch_dsl import AttrDict, AttrList
-from cl.search.models import Docket, DocketEntry
+from cl.search.models import Court, Docket, DocketEntry
register = template.Library()
@@ -243,3 +244,59 @@ def get_highlight(result: AttrDict | dict[str, any], field: str) -> any:
original_value = result.get(field, "")
return render_string_or_list(hl_value) if hl_value else original_value
+
+
+@register.filter
+def group_courts(courts: list[Court], num_columns: int) -> list:
+ """Divide courts in equal groupings while keeping related courts together
+
+ :param courts: Courts to group.
+ :param num_columns: Number of groups wanted
+ :return: The courts grouped together
+ """
+
+ column_len = len(courts) // num_columns
+ remainder = len(courts) % num_columns
+
+ groups = []
+ start = 0
+ for index in range(num_columns):
+ # Calculate the end index for this chunk
+ end = start + column_len + (1 if index < remainder else 0)
+
+ # Find the next COLR as a starting point (Court of last resort)
+ COLRs = [Court.TERRITORY_SUPREME, Court.STATE_SUPREME]
+ while end < len(courts) and courts[end].jurisdiction not in COLRs:
+ end += 1
+
+ # Create the column and add it to result
+ groups.append(courts[start:end])
+ start = end
+
+ return groups
+
+
+@register.filter
+def format_date(date_str: str) -> str:
+ """Formats a date string in the format 'F jS, Y'. Useful for formatting
+ ES child document results where dates are not date objects."""
+ try:
+ date_obj = datetime.strptime(date_str, "%Y-%m-%d")
+ return date_obj.strftime("%B %dth, %Y")
+ except (ValueError, TypeError):
+ return date_str
+
+
+@register.filter
+def build_docket_id_q_param(request_q: str, docket_id: str) -> str:
+ """Build a query string that includes the docket ID and any existing query
+ parameters.
+
+ :param request_q: The current query string, if present.
+ :param docket_id: The docket_id to append to the query string.
+ :return:The query string with the docket_id included.
+ """
+
+ if request_q:
+ return f"({request_q}) AND docket_id:{docket_id}"
+ return f"docket_id:{docket_id}"
diff --git a/cl/donate/api_views.py b/cl/donate/api_views.py
index b13deb9a1b..6b2503de8d 100644
--- a/cl/donate/api_views.py
+++ b/cl/donate/api_views.py
@@ -4,6 +4,7 @@
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
+from django.db.models import F
from django.http import HttpResponse
from rest_framework import mixins, serializers, viewsets
from rest_framework.request import Request
@@ -117,7 +118,7 @@ def _get_member_record(self, account_id: str) -> User:
contact_data = neon_account["primaryContact"]
users = User.objects.filter(
email__iexact=contact_data["email1"]
- ).order_by("-last_login")
+ ).order_by(F("last_login").desc(nulls_last=True))
if not users.exists():
address = self._get_address_from_neon_response(
contact_data["addresses"]
diff --git a/cl/donate/tests.py b/cl/donate/tests.py
index 0d5d950822..9fa22504d7 100644
--- a/cl/donate/tests.py
+++ b/cl/donate/tests.py
@@ -1,7 +1,9 @@
+from collections import defaultdict
from datetime import timedelta
from http import HTTPStatus
from unittest.mock import patch
+from asgiref.sync import sync_to_async
from django.core import mail
from django.test import override_settings
from django.test.client import AsyncClient, Client
@@ -17,6 +19,7 @@
from cl.lib.test_helpers import UserProfileWithParentsFactory
from cl.tests.cases import TestCase
from cl.users.models import UserProfile
+from cl.users.utils import create_stub_account
class EmailCommandTest(TestCase):
@@ -376,3 +379,59 @@ async def test_uses_insensitive_match_for_emails(
# Check the neon_account_id was updated properly
self.assertEqual(membership.user.profile.neon_account_id, "9524")
+
+ @patch(
+ "cl.lib.neon_utils.NeonClient.get_acount_by_id",
+ )
+ @patch.object(
+ MembershipWebhookViewSet, "_store_webhook_payload", return_value=None
+ )
+ async def test_updates_account_with_recent_login(
+ self, mock_store_webhook, mock_get_account
+ ) -> None:
+ # Create two profile records - one stub, one regular user,
+ _, stub_profile = await sync_to_async(create_stub_account)(
+ {
+ "email": "test_4@email.com",
+ "first_name": "test",
+ "last_name": "test",
+ },
+ defaultdict(lambda: ""),
+ )
+
+ user_profile = await sync_to_async(UserProfileWithParentsFactory)(
+ user__email="test_4@email.com"
+ )
+ user = user_profile.user
+ # Updates last login field for the regular user
+ user.last_login = now()
+ await user.asave()
+
+ # mocks the Neon API response
+ mock_get_account.return_value = {
+ "accountId": "1246",
+ "primaryContact": {
+ "email1": "test_4@email.com",
+ "firstName": "test",
+ "lastName": "test",
+ },
+ }
+
+ self.data["eventTrigger"] = "createMembership"
+ self.data["data"]["membership"]["accountId"] = "1246"
+ r = await self.async_client.post(
+ reverse("membership-webhooks-list", kwargs={"version": "v3"}),
+ data=self.data,
+ content_type="application/json",
+ )
+ self.assertEqual(r.status_code, HTTPStatus.CREATED)
+
+ # Refresh both profiles to ensure updated data
+ await stub_profile.arefresh_from_db()
+ await user_profile.arefresh_from_db()
+
+ # Verify stub account remains untouched
+ self.assertEqual(stub_profile.neon_account_id, "")
+
+ # Verify regular user account is updated with Neon data
+ self.assertEqual(user_profile.neon_account_id, "1246")
diff --git a/cl/favorites/migrations/0001_initial.py b/cl/favorites/migrations/0001_initial.py
index 4e35284e36..2ead29a919 100644
--- a/cl/favorites/migrations/0001_initial.py
+++ b/cl/favorites/migrations/0001_initial.py
@@ -40,7 +40,9 @@ class Migration(migrations.Migration):
],
options={
'unique_together': {('user', 'name')},
- 'index_together': {('user', 'name')},
+ 'indexes': [
+ models.Index(fields=['user', 'name'], name='favorites_usertag_user_id_name_54aef6fe_idx'),
+ ],
},
),
migrations.AddField(
@@ -58,7 +60,11 @@ class Migration(migrations.Migration):
('user', models.ForeignKey(help_text='The user that made the prayer', on_delete=django.db.models.deletion.CASCADE, related_name='prayers', to=settings.AUTH_USER_MODEL)),
],
options={
- 'index_together': {('recap_document', 'user'), ('recap_document', 'status'), ('date_created', 'user', 'status')},
+ 'indexes': [
+ models.Index(fields=['recap_document', 'user'], name='favorites_prayer_recap_document_id_user_id_c5d30108_idx'),
+ models.Index(fields=['recap_document', 'status'], name='favorites_prayer_recap_document_id_status_82e2dbbb_idx'),
+ models.Index(fields=['date_created', 'user', 'status'], name='favorites_prayer_date_created_user_id_status_880d7280_idx'),
+ ],
},
),
migrations.CreateModel(
diff --git a/cl/favorites/migrations/0007_rename_prayer_recap_document_status_favorites_p_recap_d_00e8c5_idx_and_more.py b/cl/favorites/migrations/0007_rename_prayer_recap_document_status_favorites_p_recap_d_00e8c5_idx_and_more.py
index 63fb07b217..ad00c057e4 100644
--- a/cl/favorites/migrations/0007_rename_prayer_recap_document_status_favorites_p_recap_d_00e8c5_idx_and_more.py
+++ b/cl/favorites/migrations/0007_rename_prayer_recap_document_status_favorites_p_recap_d_00e8c5_idx_and_more.py
@@ -15,22 +15,22 @@ class Migration(migrations.Migration):
operations = [
migrations.RenameIndex(
model_name="prayer",
- new_name="favorites_p_recap_d_00e8c5_idx",
+ new_name="favorites_prayer_recap_document_id_status_82e2dbbb_idx",
old_fields=("recap_document", "status"),
),
migrations.RenameIndex(
model_name="prayer",
- new_name="favorites_p_date_cr_8bf054_idx",
+ new_name="favorites_prayer_date_created_user_id_status_880d7280_idx",
old_fields=("date_created", "user", "status"),
),
migrations.RenameIndex(
model_name="prayer",
- new_name="favorites_p_recap_d_7c046c_idx",
+ new_name="favorites_prayer_recap_document_id_user_id_c5d30108_idx",
old_fields=("recap_document", "user"),
),
migrations.RenameIndex(
model_name="usertag",
- new_name="favorites_u_user_id_f6c9a6_idx",
+ new_name="favorites_usertag_user_id_name_54aef6fe_idx",
old_fields=("user", "name"),
),
]
diff --git a/cl/favorites/migrations/0007_rename_prayer_recap_document_status_favorites_p_recap_d_00e8c5_idx_and_more.sql b/cl/favorites/migrations/0007_rename_prayer_recap_document_status_favorites_p_recap_d_00e8c5_idx_and_more.sql
index 9a8337a7e4..1ce51c6907 100644
--- a/cl/favorites/migrations/0007_rename_prayer_recap_document_status_favorites_p_recap_d_00e8c5_idx_and_more.sql
+++ b/cl/favorites/migrations/0007_rename_prayer_recap_document_status_favorites_p_recap_d_00e8c5_idx_and_more.sql
@@ -1,18 +1,18 @@
BEGIN;
--
--- Rename unnamed index for ('recap_document', 'status') on prayer to favorites_p_recap_d_00e8c5_idx
+-- Rename unnamed index for ('recap_document', 'status') on prayer to favorites_prayer_recap_document_id_status_82e2dbbb_idx
--
-ALTER INDEX "favorites_prayer_recap_document_id_status_82e2dbbb_idx" RENAME TO "favorites_p_recap_d_00e8c5_idx";
+-- (no-op)
--
--- Rename unnamed index for ('date_created', 'user', 'status') on prayer to favorites_p_date_cr_8bf054_idx
+-- Rename unnamed index for ('date_created', 'user', 'status') on prayer to favorites_prayer_date_created_user_id_status_880d7280_idx
--
-ALTER INDEX "favorites_prayer_date_created_user_id_status_880d7280_idx" RENAME TO "favorites_p_date_cr_8bf054_idx";
+-- (no-op)
--
--- Rename unnamed index for ('recap_document', 'user') on prayer to favorites_p_recap_d_7c046c_idx
+-- Rename unnamed index for ('recap_document', 'user') on prayer to favorites_prayer_recap_document_id_user_id_c5d30108_idx
--
-ALTER INDEX "favorites_prayer_recap_document_id_user_id_c5d30108_idx" RENAME TO "favorites_p_recap_d_7c046c_idx";
+-- (no-op)
--
--- Rename unnamed index for ('user', 'name') on usertag to favorites_u_user_id_f6c9a6_idx
+-- Rename unnamed index for ('user', 'name') on usertag to favorites_usertag_user_id_name_54aef6fe_idx
--
-ALTER INDEX "favorites_usertag_user_id_name_54aef6fe_idx" RENAME TO "favorites_u_user_id_f6c9a6_idx";
+-- (no-op)
COMMIT;
diff --git a/cl/favorites/models.py b/cl/favorites/models.py
index af1d80af59..7ec08d7e6a 100644
--- a/cl/favorites/models.py
+++ b/cl/favorites/models.py
@@ -127,7 +127,12 @@ def __str__(self) -> str:
class Meta:
unique_together = (("user", "name"),)
- indexes = [models.Index(fields=["user", "name"])]
+ indexes = [
+ models.Index(
+ fields=["user", "name"],
+ name="favorites_usertag_user_id_name_54aef6fe_idx",
+ )
+ ]
@pghistory.track(AfterUpdateOrDeleteSnapshot())
@@ -167,11 +172,20 @@ class Meta:
# prayers do we have for this document?
# When loading the prayer leader board, we'll ask: Which documents
# have the most outstanding prayers?
- models.Index(fields=["recap_document", "status"]),
+ models.Index(
+ fields=["recap_document", "status"],
+ name="favorites_prayer_recap_document_id_status_82e2dbbb_idx",
+ ),
# When loading docket pages, we'll ask (hundreds of times): Did
# user ABC pray for document XYZ?
- models.Index(fields=["recap_document", "user"]),
+ models.Index(
+ fields=["recap_document", "user"],
+ name="favorites_prayer_recap_document_id_user_id_c5d30108_idx",
+ ),
# When a user votes, we'll ask: How many outstanding prayers did
# user ABC make today?
- models.Index(fields=["date_created", "user", "status"]),
+ models.Index(
+ fields=["date_created", "user", "status"],
+ name="favorites_prayer_date_created_user_id_status_880d7280_idx",
+ ),
]
diff --git a/cl/favorites/templates/tag.html b/cl/favorites/templates/tag.html
index 10c901de46..45226f9cd7 100644
--- a/cl/favorites/templates/tag.html
+++ b/cl/favorites/templates/tag.html
@@ -51,7 +51,6 @@
{% endif %}
{% include "includes/buy_pacer_modal.html" %}
- {% include "includes/buy_acms_modal.html" %}
{% include "includes/docket_li.html" %}
{% if forloop.last %}
diff --git a/cl/favorites/views.py b/cl/favorites/views.py
index 3c401c2410..ddb82076b3 100644
--- a/cl/favorites/views.py
+++ b/cl/favorites/views.py
@@ -54,9 +54,7 @@ async def get_note(request: HttpRequest) -> HttpResponse:
return note
-@sync_to_async
@login_required
-@async_to_sync
async def save_or_update_note(request: HttpRequest) -> HttpResponse:
"""Uses ajax to save or update a note.
@@ -92,9 +90,7 @@ async def save_or_update_note(request: HttpRequest) -> HttpResponse:
)
-@sync_to_async
@login_required
-@async_to_sync
async def delete_note(request: HttpRequest) -> HttpResponse:
"""Delete a user's note
diff --git a/cl/lasc/migrations/0001_initial.py b/cl/lasc/migrations/0001_initial.py
index c974af21a0..fae8246c62 100644
--- a/cl/lasc/migrations/0001_initial.py
+++ b/cl/lasc/migrations/0001_initial.py
@@ -41,7 +41,9 @@ class Migration(migrations.Migration):
('status_str', models.TextField(blank=True, help_text='The status of the case')),
],
options={
- 'index_together': {('docket_number', 'district', 'division_code')},
+ 'indexes': [
+ models.Index(fields=['docket_number', 'district', 'division_code'], name='lasc_docket_docket_number_district_division_code_07584433_idx'),
+ ],
},
),
migrations.CreateModel(
diff --git a/cl/lasc/migrations/0002_rename_docket_docket_number_district_division_code_lasc_docket_docket__4b4f04_idx.py b/cl/lasc/migrations/0002_rename_docket_docket_number_district_division_code_lasc_docket_docket__4b4f04_idx.py
index 0ae3684cf0..e80e9540cc 100644
--- a/cl/lasc/migrations/0002_rename_docket_docket_number_district_division_code_lasc_docket_docket__4b4f04_idx.py
+++ b/cl/lasc/migrations/0002_rename_docket_docket_number_district_division_code_lasc_docket_docket__4b4f04_idx.py
@@ -12,7 +12,7 @@ class Migration(migrations.Migration):
operations = [
migrations.RenameIndex(
model_name="docket",
- new_name="lasc_docket_docket__4b4f04_idx",
+ new_name="lasc_docket_docket_number_district_division_code_07584433_idx",
old_fields=("docket_number", "district", "division_code"),
),
]
diff --git a/cl/lasc/migrations/0002_rename_docket_docket_number_district_division_code_lasc_docket_docket__4b4f04_idx.sql b/cl/lasc/migrations/0002_rename_docket_docket_number_district_division_code_lasc_docket_docket__4b4f04_idx.sql
index ad6059d63c..afff17d231 100644
--- a/cl/lasc/migrations/0002_rename_docket_docket_number_district_division_code_lasc_docket_docket__4b4f04_idx.sql
+++ b/cl/lasc/migrations/0002_rename_docket_docket_number_district_division_code_lasc_docket_docket__4b4f04_idx.sql
@@ -1,6 +1,6 @@
BEGIN;
--
--- Rename unnamed index for ('docket_number', 'district', 'division_code') on docket to lasc_docket_docket__4b4f04_idx
+-- Rename unnamed index for ('docket_number', 'district', 'division_code') on docket to lasc_docket_docket_number_district_division_code_07584433_idx
--
-ALTER INDEX "lasc_docket_docket_number_district_division_code_07584433_idx" RENAME TO "lasc_docket_docket__4b4f04_idx";
+-- (no-op)
COMMIT;
diff --git a/cl/lasc/models.py b/cl/lasc/models.py
index 2cff2d37b1..7efb4c59cb 100644
--- a/cl/lasc/models.py
+++ b/cl/lasc/models.py
@@ -238,7 +238,10 @@ class Docket(AbstractDateTimeModel):
class Meta:
indexes = [
- models.Index(fields=["docket_number", "district", "division_code"])
+ models.Index(
+ fields=["docket_number", "district", "division_code"],
+ name="lasc_docket_docket_number_district_division_code_07584433_idx",
+ )
]
@property
diff --git a/cl/lib/command_utils.py b/cl/lib/command_utils.py
index d246288ac5..2c3797f9f5 100644
--- a/cl/lib/command_utils.py
+++ b/cl/lib/command_utils.py
@@ -17,6 +17,9 @@ def handle(self, *args, **options):
logger.setLevel(logging.INFO)
elif verbosity > 1:
logger.setLevel(logging.DEBUG)
+ # This will make juriscraper's logger accept most logger calls.
+ juriscraper_logger = logging.getLogger("juriscraper")
+ juriscraper_logger.setLevel(logging.DEBUG)
class CommandUtils:
diff --git a/cl/lib/elasticsearch_utils.py b/cl/lib/elasticsearch_utils.py
index 24d49257f7..1b9ca92683 100644
--- a/cl/lib/elasticsearch_utils.py
+++ b/cl/lib/elasticsearch_utils.py
@@ -13,9 +13,9 @@
from django.conf import settings
from django.core.cache import caches
from django.core.paginator import EmptyPage, Page
-from django.db.models import Case
+from django.db.models import Case, CharField
from django.db.models import Q as QObject
-from django.db.models import QuerySet, TextField, When
+from django.db.models import QuerySet, TextField, Value, When
from django.db.models.functions import Substr
from django.forms.boundfield import BoundField
from django.http import HttpRequest
@@ -70,6 +70,7 @@
SEARCH_RECAP_HL_FIELDS,
SEARCH_RECAP_PARENT_QUERY_FIELDS,
api_child_highlight_map,
+ cardinality_query_unique_ids,
)
from cl.search.exception import (
BadProximityQuery,
@@ -170,7 +171,7 @@ def build_daterange_query(
def build_more_like_this_query(related_id: list[str]):
document_list = [{"_id": f"o_{id}"} for id in related_id]
- more_like_this_fields = SEARCH_OPINION_QUERY_FIELDS
+ more_like_this_fields = SEARCH_OPINION_QUERY_FIELDS.copy()
more_like_this_fields.extend(
[
"type",
@@ -215,6 +216,7 @@ def add_fields_boosting(
SEARCH_TYPES.RECAP,
SEARCH_TYPES.DOCKETS,
SEARCH_TYPES.RECAP_DOCUMENT,
+ SEARCH_TYPES.OPINION,
]:
qf = BOOSTS["es"][cd["type"]].copy()
@@ -240,7 +242,7 @@ def add_fields_boosting(
matter_of_query = query.lower().startswith("matter of ")
ex_parte_query = query.lower().startswith("ex parte ")
if any([vs_query, in_re_query, matter_of_query, ex_parte_query]):
- qf.update({"caseName": 50})
+ qf.update({"caseName.exact": 50})
if fields:
qf = {key: value for key, value in qf.items() if key in fields}
@@ -727,7 +729,7 @@ def build_es_plain_filters(cd: CleanData) -> List:
)
# Build caseName terms filter
queries_list.extend(
- build_text_filter("caseName", cd.get("case_name", ""))
+ build_text_filter("caseName.exact", cd.get("case_name", ""))
)
# Build judge terms filter
queries_list.extend(build_text_filter("judge", cd.get("judge", "")))
@@ -1147,7 +1149,7 @@ def build_es_base_query(
"description",
# Docket Fields
"docketNumber",
- "caseName",
+ "caseName.exact",
],
)
)
@@ -1158,7 +1160,7 @@ def build_es_base_query(
cd,
[
"docketNumber",
- "caseName",
+ "caseName.exact",
],
)
)
@@ -1184,7 +1186,7 @@ def build_es_base_query(
[
"type",
"text",
- "caseName",
+ "caseName.exact",
"docketNumber",
],
),
@@ -1195,7 +1197,7 @@ def build_es_base_query(
add_fields_boosting(
cd,
[
- "caseName",
+ "caseName.exact",
"docketNumber",
],
)
@@ -1662,7 +1664,7 @@ def merge_courts_from_db(results: Page, search_type: str) -> None:
def fill_position_mapping(
- positions: QuerySet[Position],
+ positions: QuerySet[Position, Position],
request_type: Literal["frontend", "v3", "v4"] = "frontend",
) -> BasePositionMapping | ApiPositionMapping:
"""Extract all the data from the position queryset and
@@ -1788,6 +1790,95 @@ def merge_unavailable_fields_on_parent_document(
result["id"], ""
)
+ case (
+ SEARCH_TYPES.RECAP | SEARCH_TYPES.DOCKETS
+ ) if request_type == "frontend":
+ # Merge initial document button to the frontend search results.
+ docket_ids = {doc["docket_id"] for doc in results}
+ # This query retrieves initial documents considering two
+ # possibilities:
+ # 1. For district, bankruptcy, and appellate entries where we don't know
+ # if the entry contains attachments, it considers:
+ # document_number=1 and attachment_number=None and document_type=PACER_DOCUMENT
+ # This represents the main document with document_number 1.
+ # 2. For appellate entries where the attachment page has already been
+ # merged, it considers:
+ # document_number=1 and attachment_number=1 and document_type=ATTACHMENT
+ # This represents document_number 1 that has been converted to an attachment.
+
+ appellate_court_ids = (
+ Court.federal_courts.appellate_pacer_courts().values_list(
+ "pk", flat=True
+ )
+ )
+ initial_documents = (
+ RECAPDocument.objects.filter(
+ QObject(
+ QObject(
+ attachment_number=None,
+ document_type=RECAPDocument.PACER_DOCUMENT,
+ )
+ | QObject(
+ attachment_number=1,
+ document_type=RECAPDocument.ATTACHMENT,
+ docket_entry__docket__court_id__in=appellate_court_ids,
+ )
+ ),
+ docket_entry__docket_id__in=docket_ids,
+ document_number="1",
+ )
+ .select_related(
+ "docket_entry",
+ "docket_entry__docket",
+ "docket_entry__docket__court",
+ )
+ .only(
+ "pk",
+ "document_type",
+ "document_number",
+ "attachment_number",
+ "pacer_doc_id",
+ "is_available",
+ "filepath_local",
+ "docket_entry__docket_id",
+ "docket_entry__docket__slug",
+ "docket_entry__docket__pacer_case_id",
+ "docket_entry__docket__court__jurisdiction",
+ "docket_entry__docket__court_id",
+ )
+ )
+
+ initial_documents_in_page = {}
+ for initial_document in initial_documents:
+ if initial_document.has_valid_pdf:
+ # Initial Document available
+ initial_documents_in_page[
+ initial_document.docket_entry.docket_id
+ ] = (
+ initial_document.get_absolute_url(),
+ None,
+ "Initial Document",
+ )
+ else:
+ # Initial Document not available. Buy button.
+ initial_documents_in_page[
+ initial_document.docket_entry.docket_id
+ ] = (
+ None,
+ initial_document.pacer_url,
+ "Buy Initial Document",
+ )
+
+ for result in results:
+ document_url, buy_document_url, text_button = (
+ initial_documents_in_page.get(
+ result.docket_id, (None, None, "")
+ )
+ )
+ result["initial_document_url"] = document_url
+ result["buy_initial_document_url"] = buy_document_url
+ result["initial_document_text"] = text_button
+
case SEARCH_TYPES.OPINION if request_type == "v4" and not highlight:
# Retrieves the Opinion plain_text from the DB to fill the snippet
# when highlighting is disabled. Considering the same prioritization
@@ -1933,17 +2024,24 @@ def fetch_es_results(
es_from = (page - 1) * rows_per_page
error = True
try:
- main_query = search_query.extra(from_=es_from, size=rows_per_page)
+ # Set track_total_hits False to avoid retrieving the hit count in the main query.
+ main_query = search_query.extra(
+ from_=es_from, size=rows_per_page, track_total_hits=False
+ )
main_doc_count_query = clean_count_query(search_query)
- # Set size to 0 to avoid retrieving documents in the count queries for
- # better performance. Set track_total_hits to True to consider all the
- # documents.
- main_doc_count_query = main_doc_count_query.extra(
- size=0, track_total_hits=True
+
+ search_type = get_params.get("type", SEARCH_TYPES.OPINION)
+ parent_unique_field = cardinality_query_unique_ids[search_type]
+ main_doc_count_query = build_cardinality_count(
+ main_doc_count_query, parent_unique_field
)
+
if child_docs_count_query:
- child_total_query = child_docs_count_query.extra(
- size=0, track_total_hits=True
+ child_unique_field = cardinality_query_unique_ids[
+ SEARCH_TYPES.RECAP_DOCUMENT
+ ]
+ child_total_query = build_cardinality_count(
+ child_docs_count_query, child_unique_field
)
# Execute the ES main query + count queries in a single request.
@@ -1955,10 +2053,14 @@ def fetch_es_results(
main_response = responses[0]
main_doc_count_response = responses[1]
- parent_total = main_doc_count_response.hits.total.value
+ parent_total = simplify_estimated_count(
+ main_doc_count_response.aggregations.unique_documents.value
+ )
if child_total_query:
child_doc_count_response = responses[2]
- child_total = child_doc_count_response.hits.total.value
+ child_total = simplify_estimated_count(
+ child_doc_count_response.aggregations.unique_documents.value
+ )
query_time = main_response.took
search_type = get_params.get("type", SEARCH_TYPES.OPINION)
@@ -2081,7 +2183,7 @@ def build_join_es_filters(cd: CleanData) -> List:
cd.get("court", "").split()
),
),
- *build_text_filter("caseName", cd.get("case_name", "")),
+ *build_text_filter("caseName.exact", cd.get("case_name", "")),
*build_term_query(
"docketNumber",
cd.get("docket_number", ""),
@@ -2120,7 +2222,7 @@ def build_join_es_filters(cd: CleanData) -> List:
cd.get("court", "").split()
),
),
- *build_text_filter("caseName", cd.get("case_name", "")),
+ *build_text_filter("caseName.exact", cd.get("case_name", "")),
*build_daterange_query(
"dateFiled",
cd.get("filed_before", ""),
@@ -2927,31 +3029,30 @@ def do_es_api_query(
return main_query, child_docs_query
-def build_cardinality_count(
- base_query: Search, query: Query, unique_field: str
-) -> Search:
+def build_cardinality_count(count_query: Search, unique_field: str) -> Search:
"""Build an Elasticsearch cardinality aggregation.
This aggregation estimates the count of unique documents based on the
specified unique field. The precision_threshold, set by
ELASTICSEARCH_CARDINALITY_PRECISION, determines the point at which the
- count begins to trade accuracy for performance.
+ count begins to trade accuracy for performance. The error in the
+ approximation count using this method ranges from 1% to 6%.
+ https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-cardinality-aggregation.html#_counts_are_approximate
- :param base_query: The Elasticsearch DSL Search object.
- :param query: The ES Query object to perform the count query.
+ :param count_query: The Elasticsearch DSL Search object containing the
+ count query.
:param unique_field: The field name on which the cardinality aggregation
will be based to estimate uniqueness.
:return: The ES cardinality aggregation query.
"""
- search_query = base_query.query(query)
- search_query.aggs.bucket(
+ count_query.aggs.bucket(
"unique_documents",
"cardinality",
field=unique_field,
precision_threshold=settings.ELASTICSEARCH_CARDINALITY_PRECISION,
)
- return search_query.extra(size=0, track_total_hits=True)
+ return count_query.extra(size=0, track_total_hits=False)
def do_collapse_count_query(
@@ -2970,7 +3071,8 @@ def do_collapse_count_query(
unique_field = (
"cluster_id" if search_type == SEARCH_TYPES.OPINION else "docket_id"
)
- search_query = build_cardinality_count(main_query, query, unique_field)
+ count_query = main_query.query(query)
+ search_query = build_cardinality_count(count_query, unique_field)
try:
total_results = (
search_query.execute().aggregations.unique_documents.value
@@ -3014,3 +3116,30 @@ def do_es_alert_estimation_query(
estimation_query, _ = build_es_base_query(search_query, cd)
return estimation_query.count()
+
+
+def compute_lowest_possible_estimate(precision_threshold: int) -> int:
+ """Estimates can be below reality by as much as 6%. Round numbers below that threshold.
+ :return: The lowest possible estimate.
+ """
+ return int(precision_threshold * 0.94)
+
+
+def simplify_estimated_count(search_count: int) -> int:
+ """Simplify the estimated search count to the nearest rounded figure.
+ It only applies this rounding if the search_count exceeds the
+ ELASTICSEARCH_CARDINALITY_PRECISION threshold.
+
+ :param search_count: The original search count.
+ :return: The simplified search_count, rounded to the nearest significant
+ figure or the original search_count if below the threshold.
+ """
+
+ if search_count >= compute_lowest_possible_estimate(
+ settings.ELASTICSEARCH_CARDINALITY_PRECISION
+ ):
+ search_count_str = str(search_count)
+ first_two = search_count_str[:2]
+ zeroes = (len(search_count_str) - 2) * "0"
+ return int(first_two + zeroes)
+ return search_count
diff --git a/cl/lib/juriscraper_utils.py b/cl/lib/juriscraper_utils.py
index f74f8165cb..ae8c090f41 100644
--- a/cl/lib/juriscraper_utils.py
+++ b/cl/lib/juriscraper_utils.py
@@ -1,5 +1,6 @@
import importlib
import pkgutil
+import re
import juriscraper
@@ -16,6 +17,12 @@ def get_scraper_object_by_name(court_id: str, juriscraper_module: str = ""):
:rtype: juriscraper.AbstractSite.Site
"""
if juriscraper_module:
+ if re.search(r"\.del$", juriscraper_module):
+ # edge case where the module name is not the same as the court id
+ juriscraper_module = juriscraper_module.replace(
+ ".del", ".delaware"
+ )
+
return importlib.import_module(juriscraper_module).Site()
for _, full_module_path, _ in pkgutil.walk_packages(
diff --git a/cl/lib/model_helpers.py b/cl/lib/model_helpers.py
index de1c81a8dd..a08cfdb899 100644
--- a/cl/lib/model_helpers.py
+++ b/cl/lib/model_helpers.py
@@ -1,7 +1,7 @@
import contextlib
import os
import re
-from typing import Optional
+from typing import Callable, Optional
from django.core.exceptions import ValidationError
from django.utils.text import get_valid_filename, slugify
@@ -207,12 +207,14 @@ def make_upload_path(instance, filename):
d = instance.file_with_date
except AttributeError:
from cl.audio.models import Audio
- from cl.search.models import Opinion
+ from cl.search.models import Opinion, OpinionCluster
if type(instance) == Audio:
d = instance.docket.date_argued
elif type(instance) == Opinion:
d = instance.cluster.date_filed
+ elif type(instance) == OpinionCluster:
+ d = instance.date_filed
return "%s/%s/%02d/%02d/%s" % (
filename.split(".")[-1],
@@ -487,3 +489,119 @@ def suppress_autotime(model, fields):
field.auto_now_add = _original_values[field.name][
"auto_now_add"
]
+
+
+def linkify_orig_docket_number(agency: str, og_docket_number: str) -> str:
+ """Make an originating docket number for an appellate case into a link (MVP version)
+
+ **NOTE: These links are presented to users and should be subject to strict security checks.**
+
+ For example, each regex should be carefully written so it accepts only the narrowest of
+ matches. The risk is that:
+
+ - Mallory uploads a bad document via the RECAP APIs (these are open APIs).
+ - The code here parses that upload in a way to create a redirect on the federalregister.gov
+ website.
+ - federalregister.gov has an open redirect vulnerability (these are common).
+ - The user clicks a link on our site that goes to federalregister.gov, which redirects the
+ user to evilsite.com (b/c evilsite.com got through our checks here).
+ - The user is tricked on that site into doing something bad.
+
+ This is all quite unlikely, but we can ensure it doesn't happen by being strict about
+ the inputs our regular expressions capture.
+
+ :param agency: The administrative agency the case originated from
+ :param og_docket_number: The docket number where the case was originally heard.
+ :returns: A linkified version of the docket number for the user to click on, or the original if no link can be made.
+ """
+ # Simple pattern for Federal Register citations
+ fr_match = re.search(
+ r"(\d{1,3})\s*(?:FR|Fed\.?\s*Reg\.?)\s*(\d{1,5}(?:,\d{3})*)",
+ og_docket_number,
+ )
+
+ if fr_match:
+ volume, page = fr_match.groups()
+ return f"https://www.federalregister.gov/citation/{volume}-FR-{page}"
+
+ # NLRB pattern
+ if agency == "National Labor Relations Board":
+ match = re.match(
+ r"^(?:NLRB-)?(\d{1,2})-?([A-Z]{2})-?(\d{1,6})$", og_docket_number
+ )
+ if match:
+ region, case_type, number = match.groups()
+ formatted_number = (
+ f"{region.zfill(2)}-{case_type}-{number.zfill(6)}"
+ )
+ return f"https://www.nlrb.gov/case/{formatted_number}"
+
+ # US Tax Court pattern
+ if any(x in agency for x in ("Tax", "Internal Revenue")):
+ match = re.match(
+ r"^(?:USTC-)?(\d{1,5})-(\d{2})([A-Z])?$", og_docket_number
+ )
+ if match:
+ number, year, letter_suffix = match.groups()
+ formatted_number = f"{number.zfill(5)}-{year}"
+ if letter_suffix:
+ formatted_number += letter_suffix
+ return (
+ f"https://dawson.ustaxcourt.gov/case-detail/{formatted_number}"
+ )
+
+ # EPA non-Federal Register pattern
+ if "Environmental Protection" in agency:
+ match = re.match(
+ r"^EPA-(HQ|R\d{2})-[A-Z]{2,5}-\d{4}-\d{4}$", og_docket_number
+ )
+ if match:
+ return f"https://www.regulations.gov/docket/{match.group(0)}"
+
+ """Add other agencies as feasible. Note that the Federal Register link should cover multiple agencies.
+ """
+ # If no match is found, return empty str
+ return ""
+
+
+class CSVExportMixin:
+
+ def get_csv_columns(self, get_column_name: bool = False) -> list[str]:
+ """Get list of column names required in a csv file.
+ If get column name is True. It will add class name to attribute
+
+ :param: get_column_name: bool. Whether add class name to attr name
+
+ :return: list of attrs of class to get into csv file"""
+ raise NotImplementedError(
+ "Subclass must implement get_csv_columns method"
+ )
+
+ def get_column_function(self) -> dict[str, Callable[[str], str]]:
+ """Get dict of attrs: function to apply on field value if it needs
+ to be pre-processed before being add to csv
+
+ returns: dict -- > {attr1: function}"""
+ raise NotImplementedError(
+ "Subclass must implement get_column_fuction method"
+ )
+
+ def to_csv_row(self) -> list[str]:
+ """Get fields in model based on attrs column names.
+ Apply function to attr value if required.
+ Return list of modified values for csv row"""
+ row = []
+ functions = self.get_column_function()
+ columns = self.get_csv_columns(get_column_name=False)
+ for field in columns:
+ attr = getattr(self, field)
+ if not attr:
+ attr = ""
+ function = functions.get(field)
+ if function:
+ attr = function(field)
+ row.append(attr)
+ return row
+
+ def add_class_name(self, attribute_name: str) -> str:
+ return f"{self.__class__.__name__.lower()}_{attribute_name}"
diff --git a/cl/lib/neon_utils.py b/cl/lib/neon_utils.py
index fcd9bef2fe..4e35c1ea39 100644
--- a/cl/lib/neon_utils.py
+++ b/cl/lib/neon_utils.py
@@ -71,7 +71,12 @@ def search_account_by_email(self, email: str) -> list[dict[str, str]]:
"""
search_payload = {
"searchFields": [
- {"field": "Email", "operator": "EQUAL", "value": email}
+ {"field": "Email", "operator": "EQUAL", "value": email},
+ {
+ "field": "Account Type",
+ "operator": "EQUAL",
+ "value": "Individual",
+ },
],
"outputFields": ["Account ID"],
"pagination": {"pageSize": 10},
diff --git a/cl/lib/pacer_session.py b/cl/lib/pacer_session.py
index 7c993556cd..c37c8cd11a 100644
--- a/cl/lib/pacer_session.py
+++ b/cl/lib/pacer_session.py
@@ -1,4 +1,6 @@
import pickle
+import random
+from dataclasses import dataclass
from typing import Union
from urllib.parse import urlparse
@@ -12,6 +14,27 @@
session_key = "session:pacer:cookies:user.%s"
+@dataclass
+class SessionData:
+ """
+ The goal of this class is to encapsulate data required for PACER requests.
+
+ This class serves as a lightweight container for PACER session data,
+ excluding authentication details for efficient caching.
+
+ Handles default values for the `proxy` attribute when not explicitly
+ provided, indicating session data was not generated using the
+ `ProxyPacerSession` class.
+ """
+
+ cookies: RequestsCookieJar
+ proxy_address: str = ""
+
+ def __post_init__(self):
+ if not self.proxy_address:
+ self.proxy_address = settings.EGRESS_PROXY_HOSTS[0]
+
+
class ProxyPacerSession(PacerSession):
"""
This class overrides the _prepare_login_request and post methods of the
@@ -28,14 +51,32 @@ class ProxyPacerSession(PacerSession):
"""
def __init__(
- self, cookies=None, username=None, password=None, client_code=None
+ self,
+ cookies=None,
+ username=None,
+ password=None,
+ client_code=None,
+ proxy=None,
):
super().__init__(cookies, username, password, client_code)
+ self.proxy_address = proxy if proxy else self._pick_proxy_connection()
self.proxies = {
- "http": settings.EGRESS_PROXY_HOST,
+ "http": self.proxy_address,
}
self.headers["X-WhSentry-TLS"] = "true"
+ def _pick_proxy_connection(self) -> str:
+ """
+ Picks a proxy connection string from available options.
+
+ this function randomly chooses a string from the
+ `settings.EGRESS_PROXY_HOSTS` list and returns it.
+
+ Returns:
+ str: The chosen proxy connection string.
+ """
+ return random.choice(settings.EGRESS_PROXY_HOSTS)
+
def _change_protocol(self, url: str) -> str:
"""Converts a URL from HTTPS to HTTP protocol.
@@ -75,13 +116,14 @@ def log_into_pacer(
username: str,
password: str,
client_code: str | None = None,
-) -> RequestsCookieJar:
- """Log into PACER and return the cookie jar
+) -> SessionData:
+ """Log into PACER and returns a SessionData object containing the session's
+ cookies and proxy information.
:param username: A PACER username
:param password: A PACER password
:param client_code: A PACER client_code
- :return: Request.CookieJar
+ :return: A SessionData object containing the session's cookies and proxy.
"""
s = ProxyPacerSession(
username=username,
@@ -89,7 +131,7 @@ def log_into_pacer(
client_code=client_code,
)
s.login()
- return s.cookies
+ return SessionData(s.cookies, s.proxy_address)
def get_or_cache_pacer_cookies(
@@ -98,7 +140,7 @@ def get_or_cache_pacer_cookies(
password: str,
client_code: str | None = None,
refresh: bool = False,
-) -> RequestsCookieJar:
+) -> SessionData:
"""Get PACER cookies for a user or create and cache fresh ones
For the PACER Fetch API, we store users' PACER cookies in Redis with a
@@ -107,7 +149,7 @@ def get_or_cache_pacer_cookies(
This function attempts to get cookies for a user from Redis. If it finds
them, it returns them. If not, it attempts to log the user in and then
- returns the fresh cookies (after caching them).
+ returns the fresh cookies and the proxy used to login(after caching them).
:param user_pk: The PK of the user attempting to store their credentials.
Needed to create the key in Redis.
@@ -115,21 +157,25 @@ def get_or_cache_pacer_cookies(
:param password: The PACER password of the user
:param client_code: The PACER client code of the user
:param refresh: If True, refresh the cookies even if they're already cached
- :return: Cookies for the PACER user
+ :return: A SessionData object containing the session's cookies and proxy.
"""
r = get_redis_interface("CACHE", decode_responses=False)
- cookies = get_pacer_cookie_from_cache(user_pk, r=r)
+ cookies_data = get_pacer_cookie_from_cache(user_pk, r=r)
ttl_seconds = r.ttl(session_key % user_pk)
- if cookies and ttl_seconds >= 300 and not refresh:
+ if cookies_data and ttl_seconds >= 300 and not refresh:
# cookies were found in cache and ttl >= 5 minutes, return them
- return cookies
+ return cookies_data
# Unable to find cookies in cache, are about to expire or refresh needed
# Login and cache new values.
- cookies = log_into_pacer(username, password, client_code)
+ session_data = log_into_pacer(username, password, client_code)
cookie_expiration = 60 * 60
- r.set(session_key % user_pk, pickle.dumps(cookies), ex=cookie_expiration)
- return cookies
+ r.set(
+ session_key % user_pk,
+ pickle.dumps(session_data),
+ ex=cookie_expiration,
+ )
+ return session_data
def get_pacer_cookie_from_cache(
diff --git a/cl/lib/paginators.py b/cl/lib/paginators.py
index 0c37aea9c1..aa962f75e6 100644
--- a/cl/lib/paginators.py
+++ b/cl/lib/paginators.py
@@ -11,8 +11,6 @@ def __init__(self, total_query_results: int | None, *args, **kwargs):
super().__init__(*args, **kwargs)
if total_query_results:
self._count = total_query_results
- elif hasattr(self.object_list, "hits"):
- self._count = self.object_list.hits.total.value
else:
self._count = len(self.object_list)
self._aggregations = (
diff --git a/cl/lib/ratelimiter.py b/cl/lib/ratelimiter.py
index 07e4bccdab..6ac26b0a06 100644
--- a/cl/lib/ratelimiter.py
+++ b/cl/lib/ratelimiter.py
@@ -67,6 +67,7 @@ def get_path_to_make_key(group: str, request: HttpRequest) -> str:
ratelimiter_all_2_per_m = lambda func: func
ratelimiter_unsafe_3_per_m = lambda func: func
ratelimiter_unsafe_10_per_m = lambda func: func
+ ratelimiter_all_10_per_h = lambda func: func
ratelimiter_unsafe_2000_per_h = lambda func: func
else:
ratelimiter_all_2_per_m = ratelimit(
@@ -83,6 +84,10 @@ def get_path_to_make_key(group: str, request: HttpRequest) -> str:
rate="10/m",
method=UNSAFE,
)
+ ratelimiter_all_10_per_h = ratelimit(
+ key=get_path_to_make_key,
+ rate="10/h",
+ )
ratelimiter_unsafe_2000_per_h = ratelimit(
key=get_path_to_make_key,
rate="2000/h",
diff --git a/cl/lib/search_index_utils.py b/cl/lib/search_index_utils.py
index 3551d4c98e..8e4653cb5b 100644
--- a/cl/lib/search_index_utils.py
+++ b/cl/lib/search_index_utils.py
@@ -52,3 +52,24 @@ def normalize_search_dicts(d):
else:
new_dict[k] = v
return new_dict
+
+
+def get_parties_from_case_name(case_name: str) -> list[str]:
+ """Extracts the parties from case_name by splitting on common case_name
+ separators.
+
+ :param case_name: The case_name to be split.
+ :return: A list of parties. If no valid separator is found, returns an
+ empty list.
+ """
+
+ valid_case_name_separators = [
+ " v ",
+ " v. ",
+ " vs. ",
+ " vs ",
+ ]
+ for separator in valid_case_name_separators:
+ if separator in case_name:
+ return case_name.split(separator, 1)
+ return []
diff --git a/cl/lib/search_utils.py b/cl/lib/search_utils.py
index 5a3fdb6afb..3923cf0f4a 100644
--- a/cl/lib/search_utils.py
+++ b/cl/lib/search_utils.py
@@ -230,11 +230,13 @@ def merge_form_with_courts(
"district": [],
"state": [],
"special": [],
+ "military": [],
+ "tribal": [],
}
bap_bundle = []
b_bundle = []
- state_bundle: List = []
- state_bundles = []
+ states = []
+ territories = []
for court in courts:
if court.jurisdiction == Court.FEDERAL_APPELLATE:
court_tabs["federal"].append(court)
@@ -247,36 +249,25 @@ def merge_form_with_courts(
else:
b_bundle.append(court)
elif court.jurisdiction in Court.STATE_JURISDICTIONS:
- # State courts get bundled by supreme courts
- if court.jurisdiction == Court.STATE_SUPREME:
- # Whenever we hit a state supreme court, we append the
- # previous bundle and start a new one.
- if state_bundle:
- state_bundles.append(state_bundle)
- state_bundle = [court]
- else:
- state_bundle.append(court)
+ states.append(court)
+ elif court.jurisdiction in Court.TERRITORY_JURISDICTIONS:
+ territories.append(court)
elif court.jurisdiction in [
Court.FEDERAL_SPECIAL,
Court.COMMITTEE,
Court.INTERNATIONAL,
- Court.MILITARY_APPELLATE,
- Court.MILITARY_TRIAL,
]:
court_tabs["special"].append(court)
-
- # append the final state bundle after the loop ends. Hack?
- state_bundles.append(state_bundle)
+ elif court.jurisdiction in Court.MILITARY_JURISDICTIONS:
+ court_tabs["military"].append(court)
+ elif court.jurisdiction in Court.TRIBAL_JURISDICTIONS:
+ court_tabs["tribal"].append(court)
# Put the bankruptcy bundles in the courts dict
if bap_bundle:
court_tabs["bankruptcy_panel"] = [bap_bundle]
court_tabs["bankruptcy"] = [b_bundle]
-
- # Divide the state bundles into the correct partitions
- court_tabs["state"].append(state_bundles[:17])
- court_tabs["state"].append(state_bundles[17:34])
- court_tabs["state"].append(state_bundles[34:])
+ court_tabs["state"] = [states, territories]
return court_tabs, court_count_human, court_count
diff --git a/cl/lib/test_helpers.py b/cl/lib/test_helpers.py
index 69976430f1..22c6b9d96c 100644
--- a/cl/lib/test_helpers.py
+++ b/cl/lib/test_helpers.py
@@ -1533,7 +1533,7 @@ def setUpTestData(cls):
sha1="a49ada009774496ac01fb49818837e2296705c92",
)
cls.audio_3 = AudioFactory.create(
- case_name="Hong Liu Yang v. Lynch-Loretta E.",
+ case_name="Hong Liu Yang v. Lynch-Loretta E. Howell",
docket_id=cls.docket_3.pk,
duration=653,
judges="Joseph Information Deposition H Administrative magazine",
@@ -1552,10 +1552,10 @@ def setUpTestData(cls):
)
cls.audio_4.panel.add(cls.author)
cls.audio_5 = AudioFactory.create(
- case_name="Freedom of Inform Wikileaks",
+ case_name="Freedom of Inform Wikileaks Howells",
docket_id=cls.docket_4.pk,
duration=400,
- judges="Wallace to Friedland ⚖️ Deposit xx-xxxx apa magistrate",
+ judges="Wallace to Friedland ⚖️ Deposit xx-xxxx apa magistrate Freedom of Inform Wikileaks",
sha1="a49ada009774496ac01fb49818837e2296705c95",
)
cls.audio_1.panel.add(cls.author)
diff --git a/cl/lib/tests.py b/cl/lib/tests.py
index 6507fab826..b913787656 100644
--- a/cl/lib/tests.py
+++ b/cl/lib/tests.py
@@ -1,8 +1,13 @@
import datetime
+import pickle
from typing import Tuple, TypedDict, cast
+from unittest.mock import patch
from asgiref.sync import async_to_sync
+from django.conf import settings
from django.core.files.base import ContentFile
+from django.test import override_settings
+from requests.cookies import RequestsCookieJar
from cl.lib.date_time import midnight_pt
from cl.lib.elasticsearch_utils import append_query_conjunctions
@@ -11,6 +16,7 @@
from cl.lib.model_helpers import (
clean_docket_number,
is_docket_number,
+ linkify_orig_docket_number,
make_docket_number_core,
make_upload_path,
)
@@ -21,6 +27,12 @@
normalize_attorney_role,
normalize_us_state,
)
+from cl.lib.pacer_session import (
+ ProxyPacerSession,
+ SessionData,
+ get_or_cache_pacer_cookies,
+ session_key,
+)
from cl.lib.privacy_tools import anonymize
from cl.lib.ratelimiter import parse_rate
from cl.lib.redis_utils import (
@@ -80,6 +92,87 @@ def test_auto_blocking_small_bankr_docket(self) -> None:
)
+@override_settings(
+ EGRESS_PROXY_HOSTS=["http://proxy_1:9090", "http://proxy_2:9090"]
+)
+class TestPacerSessionUtils(TestCase):
+
+ def setUp(self) -> None:
+ r = get_redis_interface("CACHE", decode_responses=False)
+ # Clear cached session keys to prevent data inconsistencies.
+ key = r.keys(session_key % "test_user_new_cookie")
+ if key:
+ r.delete(*key)
+ self.test_cookies = RequestsCookieJar()
+ self.test_cookies.set("PacerSession", "this-is-a-test")
+ r.set(
+ session_key % "test_user_new_format",
+ pickle.dumps(
+ SessionData(self.test_cookies, "http://proxy_1:9090")
+ ),
+ ex=60 * 60,
+ )
+ r.set(
+ session_key % "test_new_format_almost_expired",
+ pickle.dumps(
+ SessionData(self.test_cookies, "http://proxy_1:9090")
+ ),
+ ex=60,
+ )
+
+ def test_pick_random_proxy_when_list_is_available(self):
+ """Does ProxyPacerSession choose a random proxy from the available list?"""
+ session = ProxyPacerSession(username="test", password="password")
+ self.assertIn(
+ session.proxy_address,
+ ["http://proxy_1:9090", "http://proxy_2:9090"],
+ )
+
+ @patch("cl.lib.pacer_session.log_into_pacer")
+ def test_compute_new_cookies_with_new_format(self, mock_log_into_pacer):
+ """Are we using the dataclass for new cookies?"""
+ mock_log_into_pacer.return_value = SessionData(
+ self.test_cookies,
+ "http://proxy_1:9090",
+ )
+ session_data = get_or_cache_pacer_cookies(
+ "test_user_new_cookie", username="test", password="password"
+ )
+ self.assertEqual(mock_log_into_pacer.call_count, 1)
+ self.assertIsInstance(session_data, SessionData)
+ self.assertEqual(session_data.proxy_address, "http://proxy_1:9090")
+
+ @patch("cl.lib.pacer_session.log_into_pacer")
+ def test_parse_cookie_proxy_pair_properly(self, mock_log_into_pacer):
+ """Can we parse the dataclass from cache properly?"""
+ session_data = get_or_cache_pacer_cookies(
+ "test_user_new_format", username="test", password="password"
+ )
+ self.assertEqual(mock_log_into_pacer.call_count, 0)
+ self.assertIsInstance(session_data, SessionData)
+ self.assertEqual(session_data.proxy_address, "http://proxy_1:9090")
+
+ @patch("cl.lib.pacer_session.log_into_pacer")
+ def test_compute_cookies_for_almost_expired_data(
+ self, mock_log_into_pacer
+ ):
+ """Are we using the dataclass when re-computing session?"""
+ mock_log_into_pacer.return_value = SessionData(
+ self.test_cookies, "http://proxy_2:9090"
+ )
+
+ # Attempts to get almost expired cookies with the new format from cache
+ # Expects refresh.
+ session_data = get_or_cache_pacer_cookies(
+ "test_new_format_almost_expired",
+ username="test",
+ password="password",
+ )
+ self.assertIsInstance(session_data, SessionData)
+ self.assertEqual(mock_log_into_pacer.call_count, 1)
+ self.assertEqual(session_data.proxy_address, "http://proxy_2:9090")
+
+
class TestStringUtils(SimpleTestCase):
def test_trunc(self) -> None:
"""Does trunc give us the results we expect?"""
@@ -1131,3 +1224,89 @@ def test_redis_lock(self) -> None:
result = release_redis_lock(r, lock_key, identifier)
self.assertEqual(result, 1)
+
+
+class TestLinkifyOrigDocketNumber(SimpleTestCase):
+ def test_linkify_orig_docket_number(self):
+ test_pairs = [
+ (
+ "National Labor Relations Board",
+ "19-CA-289275",
+ "https://www.nlrb.gov/case/19-CA-289275",
+ ),
+ (
+ "National Labor Relations Board",
+ "NLRB-09CA110508",
+ "https://www.nlrb.gov/case/09-CA-110508",
+ ),
+ (
+ "EPA",
+ "85 FR 20688",
+ "https://www.federalregister.gov/citation/85-FR-20688",
+ ),
+ (
+ "Other Agency",
+ "85 Fed. Reg. 12345",
+ "https://www.federalregister.gov/citation/85-FR-12345",
+ ),
+ (
+ "National Labor Relations Board",
+ "85 Fed. Reg. 12345",
+ "https://www.federalregister.gov/citation/85-FR-12345",
+ ),
+ (
+ "Bureau of Land Management",
+ "88FR20688",
+ "https://www.federalregister.gov/citation/88-FR-20688",
+ ),
+ (
+ "Bureau of Land Management",
+ "88 Fed Reg 34523",
+ "https://www.federalregister.gov/citation/88-FR-34523",
+ ),
+ (
+ "Department of Transportation",
+ "89 Fed. Reg. 34,620",
+ "https://www.federalregister.gov/citation/89-FR-34,620",
+ ),
+ (
+ "Environmental Protection Agency",
+ "EPA-HQ-OW-2020-0005",
+ "https://www.regulations.gov/docket/EPA-HQ-OW-2020-0005",
+ ),
+ (
+ "United States Tax Court",
+ "USTC-2451-13",
+ "https://dawson.ustaxcourt.gov/case-detail/02451-13",
+ ),
+ (
+ "United States Tax Court",
+ "6837-20",
+ "https://dawson.ustaxcourt.gov/case-detail/06837-20",
+ ),
+ (
+ "United States Tax Court",
+ "USTC-5903-19W",
+ "https://dawson.ustaxcourt.gov/case-detail/05903-19W",
+ ),
+ ("Federal Communications Commission", "19-CA-289275", ""),
+ (
+ "National Labor Relations Board",
+ "This is not an NLRB case",
+ "",
+ ),
+ ("Other Agency", "This is not a Federal Register citation", ""),
+ ]
+
+ for i, (agency, docket_number, expected_output) in enumerate(
+ test_pairs
+ ):
+ with self.subTest(
+ f"Testing description text cleaning for {agency, docket_number}...",
+ i=i,
+ ):
+ self.assertEqual(
+ linkify_orig_docket_number(agency, docket_number),
+ expected_output,
+ f"Got incorrect result from clean_parenthetical_text for text: {agency, docket_number}",
+ )
diff --git a/cl/opinion_page/forms.py b/cl/opinion_page/forms.py
index ecc96b0b40..c4eefd75c1 100644
--- a/cl/opinion_page/forms.py
+++ b/cl/opinion_page/forms.py
@@ -651,13 +651,13 @@ class TennWorkCompAppUploadForm(BaseCourtUploadForm):
"""Form for Tennessee Workers' Compensation Appeals Board (tennworkcompapp)
Upload Portal"""
- second_judge = forms.ModelChoiceField(
+ second_judge: forms.ModelChoiceField = forms.ModelChoiceField(
queryset=Person.objects.none(),
required=False,
label="Second Panelist",
widget=forms.Select(attrs={"class": "form-control"}),
)
- third_judge = forms.ModelChoiceField(
+ third_judge: forms.ModelChoiceField = forms.ModelChoiceField(
queryset=Person.objects.none(),
required=False,
label="Third Panelist",
diff --git a/cl/opinion_page/static/js/buy_pacer_modal.js b/cl/opinion_page/static/js/buy_pacer_modal.js
index 0bf8ecda1e..982d76dbea 100644
--- a/cl/opinion_page/static/js/buy_pacer_modal.js
+++ b/cl/opinion_page/static/js/buy_pacer_modal.js
@@ -12,20 +12,6 @@ $(document).ready(function () {
}
});
-
- $('.open_buy_acms_modal').on('click', function (e) {
- //Modal clicked
- //check if ctrl or shift key pressed
- if (e.metaKey || e.shiftKey) {
- //prevent modal from opening, go directly to href link
- e.stopPropagation();
- }else {
- //otherwise open modal and concatenate pacer URL to button
- let pacer_url = $(this).attr('href');
- $('#acms_url').attr('href', pacer_url);
- }
- });
-
//////////////////////////
// Modal Cookie Handling//
//////////////////////////
@@ -39,9 +25,4 @@ $(document).ready(function () {
///Close Modal
$('#modal-buy-pacer').modal('toggle');
});
-
- $('#acms_url').on('click', function (e) {
- ///Close Modal
- $('#modal-buy-acms').modal('toggle');
- });
});
diff --git a/cl/opinion_page/templates/docket_tabs.html b/cl/opinion_page/templates/docket_tabs.html
index 22d274cb68..073d0739ad 100644
--- a/cl/opinion_page/templates/docket_tabs.html
+++ b/cl/opinion_page/templates/docket_tabs.html
@@ -19,6 +19,13 @@
+ {% if DEBUG %}
+
+
+ {% else %}
+
+ {% endif %}
+
{% if request.user.is_authenticated %}
@@ -39,7 +46,6 @@
src="{% static "js/buy_pacer_modal.js" %}">
{% include "includes/buy_pacer_modal.html" %}
- {% include "includes/buy_acms_modal.html" %}
{% include "includes/date_picker.html" %}
+
{% endblock %}
{% block nav %}
@@ -349,6 +356,8 @@ Originating Court Information
data-toggle="tooltip"
data-placement="right"
title="Search for this docket number in the RECAP Archive.">{{ og_info.docket_number }})
+ {% elif og_info.administrative_link %}
+ ({{ og_info.docket_number }})
{% else %}
({{ og_info.docket_number }})
{% endif %}
@@ -436,7 +445,7 @@ Oral Argument Recordings
{% for af in docket.audio_files.all %}
-
-
+
{{ af|best_case_name|safe|v_wrapper }}
{% if perms.audio.change_audio %}
diff --git a/cl/opinion_page/templates/includes/authorities_list.html b/cl/opinion_page/templates/includes/authorities_list.html
index a914a9fced..279cd34df2 100644
--- a/cl/opinion_page/templates/includes/authorities_list.html
+++ b/cl/opinion_page/templates/includes/authorities_list.html
@@ -4,7 +4,7 @@
{% for authority in authorities %}
-
{{ authority.depth }} reference{{ authority.depth|pluralize }} to
-
+
{{ authority.cited_opinion.cluster.caption|safe|v_wrapper }}
diff --git a/cl/opinion_page/templates/includes/buy_acms_modal.html b/cl/opinion_page/templates/includes/buy_acms_modal.html
deleted file mode 100644
index 237623db62..0000000000
--- a/cl/opinion_page/templates/includes/buy_acms_modal.html
+++ /dev/null
@@ -1,26 +0,0 @@
-{% load humanize %}
-
-
-
-
-
- ACMS Document URL Unavailable
-
-
- This document is filed in the new Appellate Case Management System (ACMS), which does not have direct document access. You must buy the docket sheet from the system and then buy the documents from there.
-
-
-
-
-
-
-
diff --git a/cl/opinion_page/templates/includes/de_filter.html b/cl/opinion_page/templates/includes/de_filter.html
index a1946b23a2..eda1135fca 100644
--- a/cl/opinion_page/templates/includes/de_filter.html
+++ b/cl/opinion_page/templates/includes/de_filter.html
@@ -85,7 +85,7 @@
{% if docket_entries.has_previous %}
-
+
Prev.
{% else %}
@@ -94,7 +94,7 @@
{% endif %}
{% if docket_entries.has_next %}
-
+
Next
{% else %}
diff --git a/cl/opinion_page/templates/includes/de_list.html b/cl/opinion_page/templates/includes/de_list.html
index 1f43612f7e..0c0776ef9f 100644
--- a/cl/opinion_page/templates/includes/de_list.html
+++ b/cl/opinion_page/templates/includes/de_list.html
@@ -8,7 +8,50 @@
Document Number
Date Filed
- Description
+
+
+
+ There was a problem. Try again later.
+
+
+
+
+
+
+ Export CSV
+
+
+
+
+
+
+
+
{% for de in docket_entries %}
-
- {% if '-' in rd.pacer_doc_id %}
- Direct Link Unavailable
- {% else %}
- {% if rd.is_free_on_pacer %}From PACER{% else %}Buy on PACER{% endif %} {% if rd.page_count %}(${{ rd|price }}){% endif %}
- {% endif %}
+ {% if rd.is_free_on_pacer %}From PACER{% else %}Buy on PACER{% endif %} {% if rd.page_count %}(${{ rd|price }}){% endif %}
-
{% endif %}
{% else %}
@@ -128,23 +162,14 @@
{% else %}
{% if rd.pacer_url %}
- {% if '-' in rd.pacer_doc_id %}
- Direct Link Unavailable
- {% else %}
- Buy on PACER {% if rd.page_count %}(${{ rd|price }}){% endif %}
- {% endif %}
+ rel="nofollow">Buy on PACER {% if rd.page_count %}(${{ rd|price }}){% endif %}
{% endif %}
{% endif %}
@@ -168,12 +193,7 @@
{% else %}
{% if rd.pacer_url %}
+ title="Buy on PACER {% if rd.page_count %}(${{ rd|price }}){% endif %}">
{% endif %}
{% endif %}
diff --git a/cl/opinion_page/templates/includes/opinions_sidebar.html b/cl/opinion_page/templates/includes/opinions_sidebar.html
index b15dd44002..8ca4a0881c 100644
--- a/cl/opinion_page/templates/includes/opinions_sidebar.html
+++ b/cl/opinion_page/templates/includes/opinions_sidebar.html
@@ -4,7 +4,7 @@
{% flag "o-es-active" %}
{% for opinion in opinions.object_list %}
-
-
+
{% with opinion.title as title %}
{{ opinion.caseName|default:title|default_if_none:"N/A"|safe|truncatewords:10|v_wrapper }}
{% endwith %}
@@ -14,7 +14,7 @@
{% else %}
{% for opinion in opinions %}
-
-
+
{% with opinion.title as title %}
{{ opinion.caseName|default:title|default_if_none:"N/A"|safe|truncatewords:10|v_wrapper }}
{% endwith %}
diff --git a/cl/opinion_page/templates/includes/rd_download_button.html b/cl/opinion_page/templates/includes/rd_download_button.html
index 0157f53fec..73f2ba88a4 100644
--- a/cl/opinion_page/templates/includes/rd_download_button.html
+++ b/cl/opinion_page/templates/includes/rd_download_button.html
@@ -25,20 +25,15 @@
{% endif %}
- {% if rd.pacer_url or '-' in rd.pacer_doc_id%}
+ {% if rd.pacer_url %}
-
- {% if '-' in rd.pacer_doc_id %}Direct Link Unavailable{% else %}Buy on PACER{% endif %}
-
+ rel="nofollow">Buy on PACER
{% endif %}
@@ -47,13 +42,9 @@
{% if rd.is_sealed %}
This Item is Sealed
{% else %}
- {% if rd.pacer_url or '-' in rd.pacer_doc_id%}
+ {% if rd.pacer_url %}
-
- {% if '-' in rd.pacer_doc_id %}Direct Link Unavailable{% else %}Buy on PACER{% endif %}
-
+ Buy on PACER
{% endif %}
{% endif %}
{% endif %}
diff --git a/cl/opinion_page/templates/opinion.html b/cl/opinion_page/templates/opinion.html
index a5992f8189..290e837ad8 100644
--- a/cl/opinion_page/templates/opinion.html
+++ b/cl/opinion_page/templates/opinion.html
@@ -100,7 +100,7 @@ Summaries ({{ summaries_count|intcomma }})
{% endfor %}
-
View All Summaries
@@ -123,7 +123,7 @@
{% for citing_cluster in citing_clusters %}
-
- {{ citing_cluster.caseName|safe|truncatewords:12|v_wrapper }} ({{ citing_cluster.dateFiled|date:"Y" }})
+ {{ citing_cluster.caseName|safe|truncatewords:12|v_wrapper }} ({{ citing_cluster.dateFiled|date:"Y" }})
{% endfor %}
diff --git a/cl/opinion_page/templates/opinion_authorities.html b/cl/opinion_page/templates/opinion_authorities.html
index a146143ba2..50ed524a89 100644
--- a/cl/opinion_page/templates/opinion_authorities.html
+++ b/cl/opinion_page/templates/opinion_authorities.html
@@ -18,7 +18,7 @@
@@ -28,7 +28,7 @@
{% block content %}
- {{ caption|safe|v_wrapper }}
@@ -41,7 +41,7 @@ This opinion cites {{ authorities_with_data|length|intcomma }} opinion{{ aut
{% for authority in authorities_with_data %}
-
{{ authority.citation_depth }} reference{{ authority.citation_depth|pluralize }} to
-
{{ authority.caption|safe|v_wrapper }}
diff --git a/cl/opinion_page/templates/opinion_summaries.html b/cl/opinion_page/templates/opinion_summaries.html
index eb8c8e7b87..1cafb08765 100644
--- a/cl/opinion_page/templates/opinion_summaries.html
+++ b/cl/opinion_page/templates/opinion_summaries.html
@@ -19,7 +19,7 @@
@@ -29,7 +29,7 @@
{% block content %}
- {{ caption|safe|v_wrapper }}
@@ -53,7 +53,7 @@ {{ summaries_count|intcomma }} judge-written summar{{ summaries_count|plural
{{ representative_cluster.date_filed }}
-
+
{{ representative_cluster|best_case_name|safe }}
@@ -78,7 +78,7 @@ {{ summaries_count|intcomma }} judge-written summar{{ summaries_count|plural
{{ describing_cluster.date_filed }}
-
+
{{ describing_cluster|best_case_name|safe }}
diff --git a/cl/opinion_page/templates/opinion_visualizations.html b/cl/opinion_page/templates/opinion_visualizations.html
index 0d1811c6dc..5b531096e9 100644
--- a/cl/opinion_page/templates/opinion_visualizations.html
+++ b/cl/opinion_page/templates/opinion_visualizations.html
@@ -19,7 +19,7 @@
diff --git a/cl/opinion_page/templates/recap_document.html b/cl/opinion_page/templates/recap_document.html
index 32600cbb41..8c1cb5fbae 100644
--- a/cl/opinion_page/templates/recap_document.html
+++ b/cl/opinion_page/templates/recap_document.html
@@ -73,7 +73,6 @@ {{ rd.docket_entry.docket|best_case_name|safe|v_wrapper }}
{% include "includes/notes_modal.html" %}
{% include "includes/buy_pacer_modal.html" %}
- {% include "includes/buy_acms_modal.html" %}
{% if redirect_to_pacer_modal %}
{% include "includes/redirect_to_pacer_modal.html" %}
{% endif %}
diff --git a/cl/opinion_page/tests.py b/cl/opinion_page/tests.py
index d09b0d8d93..d5ac1475d6 100644
--- a/cl/opinion_page/tests.py
+++ b/cl/opinion_page/tests.py
@@ -1,19 +1,20 @@
# mypy: disable-error-code=attr-defined
import datetime
import os
+import re
import shutil
from datetime import date
from http import HTTPStatus
from unittest import mock
-from unittest.mock import MagicMock, PropertyMock
+from unittest.mock import AsyncMock, MagicMock, PropertyMock
from asgiref.sync import async_to_sync, sync_to_async
from django.conf import settings
from django.contrib.auth.hashers import make_password
-from django.contrib.auth.models import Group
+from django.contrib.auth.models import Group, User
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.management import call_command
-from django.test import override_settings
+from django.test import RequestFactory, override_settings
from django.test.client import AsyncClient
from django.urls import reverse
from django.utils.text import slugify
@@ -38,9 +39,14 @@
)
from cl.opinion_page.utils import (
es_get_citing_clusters_with_cache,
+ generate_docket_entries_csv_data,
make_docket_title,
)
-from cl.opinion_page.views import get_prev_next_volumes
+from cl.opinion_page.views import (
+ download_docket_entries_csv,
+ fetch_docket_entries,
+ get_prev_next_volumes,
+)
from cl.people_db.factories import (
PersonFactory,
PersonWithChildrenFactory,
@@ -50,6 +56,7 @@
from cl.recap.factories import (
AppellateAttachmentFactory,
AppellateAttachmentPageFactory,
+ DocketDataFactory,
DocketEntriesDataFactory,
DocketEntryDataFactory,
)
@@ -57,17 +64,20 @@
from cl.search.factories import (
CitationWithParentsFactory,
CourtFactory,
+ DocketEntryFactory,
DocketFactory,
OpinionClusterFactoryWithChildrenAndParents,
OpinionClusterWithParentsFactory,
OpinionFactory,
OpinionsCitedWithParentsFactory,
+ RECAPDocumentFactory,
)
from cl.search.models import (
PRECEDENTIAL_STATUS,
SEARCH_TYPES,
Citation,
Docket,
+ DocketEntry,
Opinion,
OpinionCluster,
RECAPDocument,
@@ -1523,3 +1533,179 @@ async def test_block_cluster_and_docket_via_ajax_view(self) -> None:
await self.cluster.arefresh_from_db()
self.assertTrue(self.cluster.blocked)
+
+
+class DocketEntryFileDownload(TestCase):
+ """Test Docket entries File Download and required functions."""
+
+ def setUp(self):
+ court = CourtFactory(id="ca5", jurisdiction="F")
+ # Main docket to test
+ docket = DocketFactory(
+ court=court,
+ case_name="Foo v. Bar",
+ docket_number="12-11111",
+ pacer_case_id="12345",
+ )
+
+ de1 = DocketEntryFactory(
+ docket=docket,
+ entry_number=506581111,
+ )
+ RECAPDocumentFactory(
+ docket_entry=de1,
+ pacer_doc_id="00506581111",
+ document_number="00506581111",
+ document_type=RECAPDocument.PACER_DOCUMENT,
+ )
+ de1_2 = DocketEntryFactory(
+ docket=docket,
+ entry_number=1,
+ )
+ RECAPDocumentFactory(
+ docket_entry=de1_2,
+ pacer_doc_id="00506581111",
+ document_number="1",
+ document_type=RECAPDocument.PACER_DOCUMENT,
+ )
+
+ de2 = DocketEntryFactory(
+ docket=docket,
+ entry_number=2,
+ description="Lorem ipsum dolor sit amet",
+ )
+ RECAPDocumentFactory(
+ docket_entry=de2,
+ pacer_doc_id="",
+ document_number="2",
+ document_type=RECAPDocument.PACER_DOCUMENT,
+ )
+
+ de3 = DocketEntryFactory(
+ docket=docket,
+ entry_number=506582222,
+ )
+ RECAPDocumentFactory(
+ docket_entry=de3,
+ pacer_doc_id="00506582222",
+ document_number="3",
+ document_type=RECAPDocument.ATTACHMENT,
+ attachment_number=1,
+ )
+ RECAPDocumentFactory(
+ docket_entry=de3,
+ description="Document attachment",
+ document_type=RECAPDocument.ATTACHMENT,
+ document_number="3",
+ attachment_number=2,
+ )
+ # Create extra docket and docket entries to make sure it only fetch
+ # required docket_entries
+ docket1 = DocketFactory(
+ court=court,
+ case_name="Test v. Test1",
+ docket_number="12-222222",
+ pacer_case_id="12345",
+ )
+ de4 = DocketEntryFactory(
+ docket=docket1,
+ entry_number=506582222,
+ )
+ RECAPDocumentFactory(
+ docket_entry=de4,
+ pacer_doc_id="00506582222",
+ document_number="005506582222",
+ document_type=RECAPDocument.PACER_DOCUMENT,
+ )
+ self.mocked_docket = docket
+ self.mocked_extra_docket = docket1
+ self.mocked_docket_entries = [de1, de1_2, de2, de3]
+ self.mocked_extra_docket_entries = [de4]
+
+ request_factory = RequestFactory()
+ self.request = request_factory.get("/mock-url/")
+ self.user = UserFactory.create(
+ username="learned",
+ email="learnedhand@scotus.gov",
+ )
+ self.request.auser = AsyncMock(return_value=self.user)
+
+ def tearDown(self):
+ # Clear all test data
+ Docket.objects.all().delete()
+ DocketEntry.objects.all().delete()
+ RECAPDocument.objects.all().delete()
+ User.objects.all().delete()
+
+ async def test_fetch_docket_entries(self) -> None:
+ """Verify that fetch entries function returns right docket_entries"""
+ res = await fetch_docket_entries(self.mocked_docket)
+ self.assertEqual(await res.acount(), len(self.mocked_docket_entries))
+ self.assertTrue(await res.acontains(self.mocked_docket_entries[0]))
+ self.assertFalse(
+ await res.acontains(self.mocked_extra_docket_entries[0])
+ )
+
+ def test_generate_docket_entries_csv_data(self) -> None:
+ """Verify str with csv data is created. Check column and data entry"""
+ res = generate_docket_entries_csv_data(self.mocked_docket_entries)
+ res_lines = res.split("\r\n")
+ res_line_data = res_lines[1].split(",")
+ self.assertEqual(res[:16], '"docketentry_id"')
+ self.assertEqual(res_line_data[1], '"506581111"')
+
+ # Checks if the number of values in each CSV row matches the expected
+ # number of columns.
+
+ # Compute the expected number of columns by combining the columns from
+ # the docket entry and recap documents
+ docket_entry = self.mocked_docket_entries[0]
+ de_columns = docket_entry.get_csv_columns(get_column_name=True)
+ rd_columns = docket_entry.recap_documents.first().get_csv_columns(
+ get_column_name=True
+ )
+ column_count = len(de_columns + rd_columns)
+
+ # Iterate over each line in the generated CSV data and count the number
+ # of values.
+ rows = [
+ len(re.findall('"([^"]*)"', line)) == column_count
+ for line in res_lines
+ if line
+ ]
+ # Assert that all rows have the expected number of values.
+ self.assertTrue(
+ all(rows),
+ "One or more rows of the CSV file has more values than expected",
+ )
+
+ @mock.patch("cl.opinion_page.utils.user_has_alert")
+ @mock.patch("cl.opinion_page.utils.core_docket_data")
+ @mock.patch("cl.opinion_page.utils.generate_docket_entries_csv_data")
+ def test_view_download_docket_entries_csv(
+ self,
+ mock_download_function,
+ mock_core_docket_data,
+ mock_user_has_alert,
+ ) -> None:
+ """Test download_docket_entries_csv returns csv content"""
+
+ mock_download_function.return_value = (
+ '"col1","col2","col3"\r\n"value1","value2","value3"'
+ )
+ mock_user_has_alert.return_value = False
+ mock_core_docket_data.return_value = (
+ self.mocked_docket,
+ {
+ "docket": self.mocked_docket,
+ "title": "title",
+ "note_form": "note_form",
+ "has_alert": mock_user_has_alert.return_value,
+ "timezone": "EST",
+ "private": True,
+ },
+ )
+ response = download_docket_entries_csv(
+ self.request, self.mocked_docket.id
+ )
+ self.assertEqual(response["Content-Type"], "text/csv")
diff --git a/cl/opinion_page/urls.py b/cl/opinion_page/urls.py
index 28b45319c6..5e7a9e1a54 100644
--- a/cl/opinion_page/urls.py
+++ b/cl/opinion_page/urls.py
@@ -9,6 +9,7 @@
court_publish_page,
docket_authorities,
docket_idb_data,
+ download_docket_entries_csv,
redirect_docket_recap,
redirect_og_lookup,
view_authorities,
@@ -52,7 +53,15 @@
),
path("opinion///", view_opinion, name="view_case"), # type: ignore[arg-type]
path(
- "docket///", view_docket, name="view_docket" # type: ignore[arg-type]
+ "docket//download/",
+ download_docket_entries_csv, # type: ignore[arg-type]
+ name="view_download_docket",
+ ),
+ path(
+ "docket///",
+ view_docket,
+ name="view_docket",
+ # type: ignore[arg-type]
),
path(
"recap/gov.uscourts../",
diff --git a/cl/opinion_page/utils.py b/cl/opinion_page/utils.py
index bc74cfb5e3..2e5f7f1c99 100644
--- a/cl/opinion_page/utils.py
+++ b/cl/opinion_page/utils.py
@@ -1,3 +1,5 @@
+import csv
+from io import StringIO
from typing import Dict, Tuple, Union
from asgiref.sync import sync_to_async
@@ -42,7 +44,7 @@ async def core_docket_data(
pk: int,
) -> Tuple[Docket, Dict[str, Union[bool, str, Docket, NoteForm]]]:
"""Gather the core data for a docket, party, or IDB page."""
- docket = await aget_object_or_404(Docket, pk=pk)
+ docket: Docket = await aget_object_or_404(Docket, pk=pk)
title = make_docket_title(docket)
try:
@@ -60,7 +62,7 @@ async def core_docket_data(
else:
note_form = NoteForm(instance=note)
- has_alert = await user_has_alert(await request.auser(), docket) # type: ignore[attr-defined]
+ has_alert = await user_has_alert(await request.auser(), docket) # type: ignore[arg-type]
return (
docket,
@@ -131,3 +133,32 @@ async def es_get_citing_clusters_with_cache(
cache_key, (citing_clusters, citing_cluster_count), a_week
)
return citing_clusters, citing_cluster_count
+
+
+def generate_docket_entries_csv_data(docket_entries):
+ """Get str representing in memory file from docket_entries.
+
+ :param docket_entries: List of DocketEntry that implements CSVExportMixin.
+ :returns str with csv in memory content
+ """
+ output: StringIO = StringIO()
+ csvwriter = csv.writer(output, quotechar='"', quoting=csv.QUOTE_ALL)
+ columns = []
+
+ columns = docket_entries[0].get_csv_columns(get_column_name=True)
+ columns += (
+ docket_entries[0]
+ .recap_documents.first()
+ .get_csv_columns(get_column_name=True)
+ )
+ csvwriter.writerow(columns)
+
+ for docket_entry in docket_entries:
+ for recap_doc in docket_entry.recap_documents.all():
+ csvwriter.writerow(
+ docket_entry.to_csv_row() + recap_doc.to_csv_row()
+ )
+
+ csv_content: str = output.getvalue()
+ output.close()
+ return csv_content
diff --git a/cl/opinion_page/views.py b/cl/opinion_page/views.py
index 75295d223b..0c292920a2 100644
--- a/cl/opinion_page/views.py
+++ b/cl/opinion_page/views.py
@@ -50,6 +50,7 @@
from cl.lib.http import is_ajax
from cl.lib.model_helpers import choices_to_csv
from cl.lib.models import THUMBNAIL_STATUSES
+from cl.lib.ratelimiter import ratelimiter_all_10_per_h
from cl.lib.search_utils import (
get_citing_clusters_with_cache,
get_related_clusters_with_cache,
@@ -73,6 +74,7 @@
from cl.opinion_page.utils import (
core_docket_data,
es_get_citing_clusters_with_cache,
+ generate_docket_entries_csv_data,
get_case_title,
)
from cl.people_db.models import AttorneyOrganization, CriminalCount, Role
@@ -182,7 +184,6 @@ async def court_homepage(request: HttpRequest, pk: str) -> HttpResponse:
return TemplateResponse(request, template, render_dict)
-@sync_to_async
@group_required(
"tenn_work_uploaders",
"uploaders_tennworkcompcl",
@@ -195,7 +196,6 @@ async def court_homepage(request: HttpRequest, pk: str) -> HttpResponse:
"uploaders_miss",
"uploaders_missctapp",
)
-@async_to_sync
async def court_publish_page(request: HttpRequest, pk: str) -> HttpResponse:
"""Display upload form and intake Opinions for partner courts
@@ -222,8 +222,8 @@ async def court_publish_page(request: HttpRequest, pk: str) -> HttpResponse:
"Mississippi Supreme Court and Mississippi Court of Appeals."
)
# Validate the user has permission
- user = await request.auser() # type: ignore[attr-defined]
- if not user.is_staff and not user.is_superuser:
+ user = await request.auser()
+ if not user.is_staff and not user.is_superuser: # type: ignore[union-attr]
if not await user.groups.filter( # type: ignore
name__in=[f"uploaders_{pk}"]
).aexists():
@@ -328,7 +328,7 @@ async def redirect_docket_recap(
court: Court,
pacer_case_id: str,
) -> HttpResponseRedirect:
- docket = await aget_object_or_404(
+ docket: Docket = await aget_object_or_404(
Docket, pacer_case_id=pacer_case_id, court=court
)
return HttpResponseRedirect(
@@ -336,19 +336,32 @@ async def redirect_docket_recap(
)
+async def fetch_docket_entries(docket):
+ """Fetch docket entries asociated to docket
+
+ param docket: docket.id to get related docket_entries.
+ returns: DocketEntry Queryset.
+ """
+ de_list = docket.docket_entries.all().prefetch_related(
+ Prefetch(
+ "recap_documents",
+ queryset=RECAPDocument.objects.defer("plain_text"),
+ )
+ )
+ return de_list
+
+
async def view_docket(
request: HttpRequest, pk: int, slug: str
) -> HttpResponse:
+
+ sort_order_asc = True
+ form = DocketEntryFilterForm(request.GET, request=request)
docket, context = await core_docket_data(request, pk)
await increment_view_count(docket, request)
- sort_order_asc = True
- page = request.GET.get("page", 1)
- rd_queryset = RECAPDocument.objects.defer("plain_text")
- de_list = docket.docket_entries.all().prefetch_related(
- Prefetch("recap_documents", queryset=rd_queryset)
- )
- form = DocketEntryFilterForm(request.GET, request=request)
+ de_list = await fetch_docket_entries(docket)
+
if await sync_to_async(form.is_valid)():
cd = form.cleaned_data
@@ -366,6 +379,8 @@ async def view_docket(
"-recap_sequence_number", "-entry_number"
)
+ page = request.GET.get("page", 1)
+
@sync_to_async
def paginate_docket_entries(docket_entries, docket_page):
paginator = Paginator(docket_entries, 200, orphans=10)
@@ -563,6 +578,27 @@ async def make_thumb_if_needed(
return rd
+@ratelimiter_all_10_per_h
+def download_docket_entries_csv(
+ request: HttpRequest, docket_id: int
+) -> HttpResponse:
+ """Download csv file containing list of DocketEntry for specific Docket"""
+
+ docket, _ = async_to_sync(core_docket_data)(request, docket_id)
+ de_list = async_to_sync(fetch_docket_entries)(docket)
+ court_id = docket.court_id
+ case_name = docket.slug
+
+ date_str = datetime.datetime.now().strftime("%Y-%m-%d")
+ filename = f"{case_name}.{court_id}.{docket_id}.{date_str}.csv"
+
+ # TODO check if for large files we'll cache or send file by email
+ csv_content = generate_docket_entries_csv_data(de_list)
+ response: HttpResponse = HttpResponse(csv_content, content_type="text/csv")
+ response["Content-Disposition"] = f'attachment; filename="{filename}"'
+ return response
+
+
async def view_recap_document(
request: HttpRequest,
docket_id: int | None = None,
@@ -745,7 +781,7 @@ async def view_opinion(request: HttpRequest, pk: int, _: str) -> HttpResponse:
unbound form.
"""
# Look up the court, cluster, title and note information
- cluster = await aget_object_or_404(OpinionCluster, pk=pk)
+ cluster: OpinionCluster = await aget_object_or_404(OpinionCluster, pk=pk)
title = ", ".join(
[
s
@@ -866,7 +902,7 @@ async def view_opinion(request: HttpRequest, pk: int, _: str) -> HttpResponse:
async def view_summaries(
request: HttpRequest, pk: int, slug: str
) -> HttpResponse:
- cluster = await aget_object_or_404(OpinionCluster, pk=pk)
+ cluster: OpinionCluster = await aget_object_or_404(OpinionCluster, pk=pk)
parenthetical_groups_qs = await get_or_create_parenthetical_groups(cluster)
parenthetical_groups = [
parenthetical_group
@@ -899,7 +935,7 @@ async def view_summaries(
async def view_authorities(
request: HttpRequest, pk: int, slug: str, doc_type=0
) -> HttpResponse:
- cluster = await aget_object_or_404(OpinionCluster, pk=pk)
+ cluster: OpinionCluster = await aget_object_or_404(OpinionCluster, pk=pk)
return TemplateResponse(
request,
@@ -918,7 +954,7 @@ async def view_authorities(
async def cluster_visualizations(
request: HttpRequest, pk: int, slug: str
) -> HttpResponse:
- cluster = await aget_object_or_404(OpinionCluster, pk=pk)
+ cluster: OpinionCluster = await aget_object_or_404(OpinionCluster, pk=pk)
return TemplateResponse(
request,
"opinion_visualizations.html",
@@ -1322,7 +1358,7 @@ async def citation_homepage(request: HttpRequest) -> HttpResponse:
async def block_item(request: HttpRequest) -> HttpResponse:
"""Block an item from search results using AJAX"""
user = await request.auser() # type: ignore[attr-defined]
- if is_ajax(request) and user.is_superuser:
+ if is_ajax(request) and user.is_superuser: # type: ignore[union-attr]
obj_type = request.POST["type"]
pk = request.POST["id"]
@@ -1331,13 +1367,14 @@ async def block_item(request: HttpRequest) -> HttpResponse:
"This view can not handle the provided type"
)
- cluster = None
+ cluster: OpinionCluster | None = None
if obj_type == "cluster":
# Block the cluster
cluster = await aget_object_or_404(OpinionCluster, pk=pk)
- cluster.blocked = True
- cluster.date_blocked = now()
- await cluster.asave(index=False)
+ if cluster is not None:
+ cluster.blocked = True
+ cluster.date_blocked = now()
+ await cluster.asave(index=False)
docket_pk = (
pk
@@ -1347,7 +1384,7 @@ async def block_item(request: HttpRequest) -> HttpResponse:
if not docket_pk:
return HttpResponse("It worked")
- d = await aget_object_or_404(Docket, pk=docket_pk)
+ d: Docket = await aget_object_or_404(Docket, pk=docket_pk)
d.blocked = True
d.date_blocked = now()
await d.asave()
diff --git a/cl/package-lock.json b/cl/package-lock.json
index a759a6b0f1..22c32aa3ec 100644
--- a/cl/package-lock.json
+++ b/cl/package-lock.json
@@ -51,7 +51,7 @@
"react-virtual": "^2.2.1",
"terser-webpack-plugin": "^5.3.6",
"typescript": "^4.2.4",
- "webpack": "^5.76.0",
+ "webpack": "^5.94.0",
"webpack-bundle-analyzer": "^4.4.1",
"webpack-cli": "^4.10.0",
"webpack-dev-server": "^4.10.0"
@@ -1917,13 +1917,13 @@
"dev": true
},
"node_modules/@jridgewell/gen-mapping": {
- "version": "0.3.2",
- "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz",
- "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==",
+ "version": "0.3.5",
+ "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz",
+ "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==",
"dependencies": {
- "@jridgewell/set-array": "^1.0.1",
+ "@jridgewell/set-array": "^1.2.1",
"@jridgewell/sourcemap-codec": "^1.4.10",
- "@jridgewell/trace-mapping": "^0.3.9"
+ "@jridgewell/trace-mapping": "^0.3.24"
},
"engines": {
"node": ">=6.0.0"
@@ -1938,20 +1938,20 @@
}
},
"node_modules/@jridgewell/set-array": {
- "version": "1.1.2",
- "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz",
- "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==",
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz",
+ "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==",
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/@jridgewell/source-map": {
- "version": "0.3.2",
- "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz",
- "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==",
+ "version": "0.3.6",
+ "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz",
+ "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==",
"dependencies": {
- "@jridgewell/gen-mapping": "^0.3.0",
- "@jridgewell/trace-mapping": "^0.3.9"
+ "@jridgewell/gen-mapping": "^0.3.5",
+ "@jridgewell/trace-mapping": "^0.3.25"
}
},
"node_modules/@jridgewell/sourcemap-codec": {
@@ -1960,9 +1960,9 @@
"integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw=="
},
"node_modules/@jridgewell/trace-mapping": {
- "version": "0.3.19",
- "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz",
- "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==",
+ "version": "0.3.25",
+ "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz",
+ "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==",
"dependencies": {
"@jridgewell/resolve-uri": "^3.1.0",
"@jridgewell/sourcemap-codec": "^1.4.14"
@@ -2024,24 +2024,6 @@
"@types/node": "*"
}
},
- "node_modules/@types/eslint": {
- "version": "8.4.6",
- "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.6.tgz",
- "integrity": "sha512-/fqTbjxyFUaYNO7VcW5g+4npmqVACz1bB7RTHYuLj+PRjw9hrCwrUXVQFpChUS0JsyEFvMZ7U/PfmvWgxJhI9g==",
- "dependencies": {
- "@types/estree": "*",
- "@types/json-schema": "*"
- }
- },
- "node_modules/@types/eslint-scope": {
- "version": "3.7.4",
- "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.4.tgz",
- "integrity": "sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==",
- "dependencies": {
- "@types/eslint": "*",
- "@types/estree": "*"
- }
- },
"node_modules/@types/eslint-visitor-keys": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz",
@@ -2049,9 +2031,9 @@
"dev": true
},
"node_modules/@types/estree": {
- "version": "0.0.51",
- "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.51.tgz",
- "integrity": "sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ=="
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz",
+ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw=="
},
"node_modules/@types/express": {
"version": "4.17.13",
@@ -2436,133 +2418,133 @@
}
},
"node_modules/@webassemblyjs/ast": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz",
- "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz",
+ "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==",
"dependencies": {
- "@webassemblyjs/helper-numbers": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1"
+ "@webassemblyjs/helper-numbers": "1.11.6",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6"
}
},
"node_modules/@webassemblyjs/floating-point-hex-parser": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz",
- "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ=="
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz",
+ "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw=="
},
"node_modules/@webassemblyjs/helper-api-error": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz",
- "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg=="
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz",
+ "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q=="
},
"node_modules/@webassemblyjs/helper-buffer": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz",
- "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA=="
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz",
+ "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw=="
},
"node_modules/@webassemblyjs/helper-numbers": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz",
- "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==",
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz",
+ "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==",
"dependencies": {
- "@webassemblyjs/floating-point-hex-parser": "1.11.1",
- "@webassemblyjs/helper-api-error": "1.11.1",
+ "@webassemblyjs/floating-point-hex-parser": "1.11.6",
+ "@webassemblyjs/helper-api-error": "1.11.6",
"@xtuc/long": "4.2.2"
}
},
"node_modules/@webassemblyjs/helper-wasm-bytecode": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz",
- "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q=="
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz",
+ "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA=="
},
"node_modules/@webassemblyjs/helper-wasm-section": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz",
- "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz",
+ "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==",
"dependencies": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-buffer": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
- "@webassemblyjs/wasm-gen": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-buffer": "1.12.1",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+ "@webassemblyjs/wasm-gen": "1.12.1"
}
},
"node_modules/@webassemblyjs/ieee754": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz",
- "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==",
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz",
+ "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==",
"dependencies": {
"@xtuc/ieee754": "^1.2.0"
}
},
"node_modules/@webassemblyjs/leb128": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz",
- "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==",
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz",
+ "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==",
"dependencies": {
"@xtuc/long": "4.2.2"
}
},
"node_modules/@webassemblyjs/utf8": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz",
- "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ=="
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz",
+ "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA=="
},
"node_modules/@webassemblyjs/wasm-edit": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz",
- "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz",
+ "integrity": "sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g==",
"dependencies": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-buffer": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
- "@webassemblyjs/helper-wasm-section": "1.11.1",
- "@webassemblyjs/wasm-gen": "1.11.1",
- "@webassemblyjs/wasm-opt": "1.11.1",
- "@webassemblyjs/wasm-parser": "1.11.1",
- "@webassemblyjs/wast-printer": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-buffer": "1.12.1",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+ "@webassemblyjs/helper-wasm-section": "1.12.1",
+ "@webassemblyjs/wasm-gen": "1.12.1",
+ "@webassemblyjs/wasm-opt": "1.12.1",
+ "@webassemblyjs/wasm-parser": "1.12.1",
+ "@webassemblyjs/wast-printer": "1.12.1"
}
},
"node_modules/@webassemblyjs/wasm-gen": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz",
- "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz",
+ "integrity": "sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w==",
"dependencies": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
- "@webassemblyjs/ieee754": "1.11.1",
- "@webassemblyjs/leb128": "1.11.1",
- "@webassemblyjs/utf8": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+ "@webassemblyjs/ieee754": "1.11.6",
+ "@webassemblyjs/leb128": "1.11.6",
+ "@webassemblyjs/utf8": "1.11.6"
}
},
"node_modules/@webassemblyjs/wasm-opt": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz",
- "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz",
+ "integrity": "sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg==",
"dependencies": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-buffer": "1.11.1",
- "@webassemblyjs/wasm-gen": "1.11.1",
- "@webassemblyjs/wasm-parser": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-buffer": "1.12.1",
+ "@webassemblyjs/wasm-gen": "1.12.1",
+ "@webassemblyjs/wasm-parser": "1.12.1"
}
},
"node_modules/@webassemblyjs/wasm-parser": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz",
- "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz",
+ "integrity": "sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ==",
"dependencies": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-api-error": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
- "@webassemblyjs/ieee754": "1.11.1",
- "@webassemblyjs/leb128": "1.11.1",
- "@webassemblyjs/utf8": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-api-error": "1.11.6",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+ "@webassemblyjs/ieee754": "1.11.6",
+ "@webassemblyjs/leb128": "1.11.6",
+ "@webassemblyjs/utf8": "1.11.6"
}
},
"node_modules/@webassemblyjs/wast-printer": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz",
- "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz",
+ "integrity": "sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA==",
"dependencies": {
- "@webassemblyjs/ast": "1.11.1",
+ "@webassemblyjs/ast": "1.12.1",
"@xtuc/long": "4.2.2"
}
},
@@ -2981,21 +2963,21 @@
}
},
"node_modules/body-parser": {
- "version": "1.20.0",
- "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
- "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==",
+ "version": "1.20.3",
+ "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
+ "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
"dev": true,
"dependencies": {
"bytes": "3.1.2",
- "content-type": "~1.0.4",
+ "content-type": "~1.0.5",
"debug": "2.6.9",
"depd": "2.0.0",
"destroy": "1.2.0",
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"on-finished": "2.4.1",
- "qs": "6.10.3",
- "raw-body": "2.5.1",
+ "qs": "6.13.0",
+ "raw-body": "2.5.2",
"type-is": "~1.6.18",
"unpipe": "1.0.0"
},
@@ -3059,10 +3041,22 @@
"concat-map": "0.0.1"
}
},
+ "node_modules/braces": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+ "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
+ "dev": true,
+ "dependencies": {
+ "fill-range": "^7.1.1"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/browserslist": {
- "version": "4.21.9",
- "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.9.tgz",
- "integrity": "sha512-M0MFoZzbUrRU4KNfCrDLnvyE7gub+peetoTid3TBIqtunaDJyXlwhakT+/VkvSXcfIzFfK/nkCs4nmyTmxdNSg==",
+ "version": "4.23.3",
+ "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.3.tgz",
+ "integrity": "sha512-btwCFJVjI4YWDNfau8RhZ+B1Q/VLoUITrm3RlP6y1tYGWIOa+InuYiRGXUBXo8nA1qKmHMyLB/iVQg5TT4eFoA==",
"funding": [
{
"type": "opencollective",
@@ -3078,10 +3072,10 @@
}
],
"dependencies": {
- "caniuse-lite": "^1.0.30001503",
- "electron-to-chromium": "^1.4.431",
- "node-releases": "^2.0.12",
- "update-browserslist-db": "^1.0.11"
+ "caniuse-lite": "^1.0.30001646",
+ "electron-to-chromium": "^1.5.4",
+ "node-releases": "^2.0.18",
+ "update-browserslist-db": "^1.1.0"
},
"bin": {
"browserslist": "cli.js"
@@ -3105,13 +3099,19 @@
}
},
"node_modules/call-bind": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
- "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz",
+ "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==",
"dev": true,
"dependencies": {
- "function-bind": "^1.1.1",
- "get-intrinsic": "^1.0.2"
+ "es-define-property": "^1.0.0",
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2",
+ "get-intrinsic": "^1.2.4",
+ "set-function-length": "^1.2.1"
+ },
+ "engines": {
+ "node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -3134,9 +3134,9 @@
}
},
"node_modules/caniuse-lite": {
- "version": "1.0.30001517",
- "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001517.tgz",
- "integrity": "sha512-Vdhm5S11DaFVLlyiKu4hiUTkpZu+y1KA/rZZqVQfOD5YdDT/eQKlkt7NaE0WGOFgX32diqt9MiP9CAiFeRklaA==",
+ "version": "1.0.30001653",
+ "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001653.tgz",
+ "integrity": "sha512-XGWQVB8wFQ2+9NZwZ10GxTYC5hk0Fa+q8cSkr0tgvMhYhMHP/QC+WTgrePMDBWiWc/pV+1ik82Al20XOK25Gcw==",
"funding": [
{
"type": "opencollective",
@@ -3192,51 +3192,6 @@
"fsevents": "~2.3.2"
}
},
- "node_modules/chokidar/node_modules/braces": {
- "version": "3.0.2",
- "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
- "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
- "dev": true,
- "dependencies": {
- "fill-range": "^7.0.1"
- },
- "engines": {
- "node": ">=8"
- }
- },
- "node_modules/chokidar/node_modules/fill-range": {
- "version": "7.0.1",
- "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
- "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
- "dev": true,
- "dependencies": {
- "to-regex-range": "^5.0.1"
- },
- "engines": {
- "node": ">=8"
- }
- },
- "node_modules/chokidar/node_modules/is-number": {
- "version": "7.0.0",
- "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
- "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
- "dev": true,
- "engines": {
- "node": ">=0.12.0"
- }
- },
- "node_modules/chokidar/node_modules/to-regex-range": {
- "version": "5.0.1",
- "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
- "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
- "dev": true,
- "dependencies": {
- "is-number": "^7.0.0"
- },
- "engines": {
- "node": ">=8.0"
- }
- },
"node_modules/chrome-trace-event": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz",
@@ -3463,9 +3418,9 @@
]
},
"node_modules/content-type": {
- "version": "1.0.4",
- "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
- "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==",
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
+ "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
"dev": true,
"engines": {
"node": ">= 0.6"
@@ -3480,9 +3435,9 @@
}
},
"node_modules/cookie": {
- "version": "0.5.0",
- "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
- "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
+ "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
"dev": true,
"engines": {
"node": ">= 0.6"
@@ -3691,6 +3646,23 @@
"node": ">= 10"
}
},
+ "node_modules/define-data-property": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
+ "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==",
+ "dev": true,
+ "dependencies": {
+ "es-define-property": "^1.0.0",
+ "es-errors": "^1.3.0",
+ "gopd": "^1.0.1"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/define-lazy-prop": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
@@ -3894,9 +3866,9 @@
"dev": true
},
"node_modules/electron-to-chromium": {
- "version": "1.4.467",
- "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.467.tgz",
- "integrity": "sha512-2qI70O+rR4poYeF2grcuS/bCps5KJh6y1jtZMDDEteyKJQrzLOEhFyXCLcHW6DTBjKjWkk26JhWoAi+Ux9A0fg=="
+ "version": "1.5.13",
+ "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.13.tgz",
+ "integrity": "sha512-lbBcvtIJ4J6sS4tb5TLp1b4LyfCdMkwStzXPyAgVgTRAsep4bvrAGaBOP7ZJtQMNJpSQ9SqG4brWOroNaQtm7Q=="
},
"node_modules/emoji-regex": {
"version": "8.0.0",
@@ -3913,14 +3885,26 @@
}
},
"node_modules/encodeurl": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
- "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
+ "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"dev": true,
"engines": {
"node": ">= 0.8"
}
},
+ "node_modules/enhanced-resolve": {
+ "version": "5.17.1",
+ "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz",
+ "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==",
+ "dependencies": {
+ "graceful-fs": "^4.2.4",
+ "tapable": "^2.2.0"
+ },
+ "engines": {
+ "node": ">=10.13.0"
+ }
+ },
"node_modules/enquirer": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
@@ -3995,10 +3979,31 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/es-define-property": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
+ "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
+ "dev": true,
+ "dependencies": {
+ "get-intrinsic": "^1.2.4"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
+ "node_modules/es-errors": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+ "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+ "dev": true,
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/es-module-lexer": {
- "version": "0.9.3",
- "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.9.3.tgz",
- "integrity": "sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ=="
+ "version": "1.5.4",
+ "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
+ "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw=="
},
"node_modules/es-to-primitive": {
"version": "1.2.1",
@@ -4018,9 +4023,9 @@
}
},
"node_modules/escalade": {
- "version": "3.1.1",
- "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
- "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+ "version": "3.1.2",
+ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz",
+ "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==",
"engines": {
"node": ">=6"
}
@@ -4472,37 +4477,37 @@
}
},
"node_modules/express": {
- "version": "4.18.1",
- "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz",
- "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==",
+ "version": "4.21.0",
+ "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz",
+ "integrity": "sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==",
"dev": true,
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
- "body-parser": "1.20.0",
+ "body-parser": "1.20.3",
"content-disposition": "0.5.4",
"content-type": "~1.0.4",
- "cookie": "0.5.0",
+ "cookie": "0.6.0",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "2.0.0",
- "encodeurl": "~1.0.2",
+ "encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
- "finalhandler": "1.2.0",
+ "finalhandler": "1.3.1",
"fresh": "0.5.2",
"http-errors": "2.0.0",
- "merge-descriptors": "1.0.1",
+ "merge-descriptors": "1.0.3",
"methods": "~1.1.2",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
- "path-to-regexp": "0.1.7",
+ "path-to-regexp": "0.1.10",
"proxy-addr": "~2.0.7",
- "qs": "6.10.3",
+ "qs": "6.13.0",
"range-parser": "~1.2.1",
"safe-buffer": "5.2.1",
- "send": "0.18.0",
- "serve-static": "1.15.0",
+ "send": "0.19.0",
+ "serve-static": "1.16.2",
"setprototypeof": "1.2.0",
"statuses": "2.0.1",
"type-is": "~1.6.18",
@@ -4627,14 +4632,26 @@
"node": "^10.12.0 || >=12.0.0"
}
},
+ "node_modules/fill-range": {
+ "version": "7.1.1",
+ "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+ "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
+ "dev": true,
+ "dependencies": {
+ "to-regex-range": "^5.0.1"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/finalhandler": {
- "version": "1.2.0",
- "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
- "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==",
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz",
+ "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==",
"dev": true,
"dependencies": {
"debug": "2.6.9",
- "encodeurl": "~1.0.2",
+ "encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
@@ -4788,9 +4805,12 @@
}
},
"node_modules/function-bind": {
- "version": "1.1.1",
- "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
- "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+ "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
},
"node_modules/functional-red-black-tree": {
"version": "1.0.1",
@@ -4816,14 +4836,19 @@
}
},
"node_modules/get-intrinsic": {
- "version": "1.1.1",
- "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz",
- "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==",
+ "version": "1.2.4",
+ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
+ "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
"dev": true,
"dependencies": {
- "function-bind": "^1.1.1",
- "has": "^1.0.3",
- "has-symbols": "^1.0.1"
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2",
+ "has-proto": "^1.0.1",
+ "has-symbols": "^1.0.3",
+ "hasown": "^2.0.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -4937,10 +4962,22 @@
"node": ">=0.10.0"
}
},
+ "node_modules/gopd": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+ "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+ "dev": true,
+ "dependencies": {
+ "get-intrinsic": "^1.1.3"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/graceful-fs": {
- "version": "4.2.10",
- "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz",
- "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA=="
+ "version": "4.2.11",
+ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
+ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="
},
"node_modules/gzip-size": {
"version": "6.0.0",
@@ -4991,10 +5028,34 @@
"node": ">=4"
}
},
- "node_modules/has-symbols": {
+ "node_modules/has-property-descriptors": {
"version": "1.0.2",
- "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
- "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
+ "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
+ "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==",
+ "dev": true,
+ "dependencies": {
+ "es-define-property": "^1.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/has-proto": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz",
+ "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==",
+ "dev": true,
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/has-symbols": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
+ "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
"dev": true,
"engines": {
"node": ">= 0.4"
@@ -5018,6 +5079,18 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/hasown": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+ "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+ "dev": true,
+ "dependencies": {
+ "function-bind": "^1.1.2"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/history": {
"version": "4.10.1",
"resolved": "https://registry.npmjs.org/history/-/history-4.10.1.tgz",
@@ -5167,39 +5240,6 @@
}
}
},
- "node_modules/http-proxy-middleware/node_modules/braces": {
- "version": "3.0.2",
- "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
- "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
- "dev": true,
- "dependencies": {
- "fill-range": "^7.0.1"
- },
- "engines": {
- "node": ">=8"
- }
- },
- "node_modules/http-proxy-middleware/node_modules/fill-range": {
- "version": "7.0.1",
- "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
- "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
- "dev": true,
- "dependencies": {
- "to-regex-range": "^5.0.1"
- },
- "engines": {
- "node": ">=8"
- }
- },
- "node_modules/http-proxy-middleware/node_modules/is-number": {
- "version": "7.0.0",
- "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
- "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
- "dev": true,
- "engines": {
- "node": ">=0.12.0"
- }
- },
"node_modules/http-proxy-middleware/node_modules/micromatch": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
@@ -5213,18 +5253,6 @@
"node": ">=8.6"
}
},
- "node_modules/http-proxy-middleware/node_modules/to-regex-range": {
- "version": "5.0.1",
- "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
- "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
- "dev": true,
- "dependencies": {
- "is-number": "^7.0.0"
- },
- "engines": {
- "node": ">=8.0"
- }
- },
"node_modules/human-signals": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
@@ -5505,6 +5533,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/is-number": {
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.12.0"
+ }
+ },
"node_modules/is-number-object": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.6.tgz",
@@ -5954,10 +5991,13 @@
}
},
"node_modules/merge-descriptors": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
- "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==",
- "dev": true
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
+ "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==",
+ "dev": true,
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
},
"node_modules/merge-stream": {
"version": "2.0.0",
@@ -6103,9 +6143,9 @@
}
},
"node_modules/node-releases": {
- "version": "2.0.13",
- "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.13.tgz",
- "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ=="
+ "version": "2.0.18",
+ "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz",
+ "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g=="
},
"node_modules/normalize-path": {
"version": "3.0.0",
@@ -6137,10 +6177,13 @@
}
},
"node_modules/object-inspect": {
- "version": "1.11.0",
- "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz",
- "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==",
+ "version": "1.13.2",
+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz",
+ "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==",
"dev": true,
+ "engines": {
+ "node": ">= 0.4"
+ },
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
@@ -6462,9 +6505,9 @@
"integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="
},
"node_modules/path-to-regexp": {
- "version": "0.1.7",
- "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
- "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==",
+ "version": "0.1.10",
+ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz",
+ "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==",
"dev": true
},
"node_modules/path-type": {
@@ -6476,9 +6519,9 @@
}
},
"node_modules/picocolors": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz",
- "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ=="
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz",
+ "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew=="
},
"node_modules/picomatch": {
"version": "2.3.1",
@@ -6727,12 +6770,12 @@
}
},
"node_modules/qs": {
- "version": "6.10.3",
- "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
- "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
+ "version": "6.13.0",
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
+ "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"dev": true,
"dependencies": {
- "side-channel": "^1.0.4"
+ "side-channel": "^1.0.6"
},
"engines": {
"node": ">=0.6"
@@ -6759,9 +6802,9 @@
}
},
"node_modules/raw-body": {
- "version": "2.5.1",
- "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
- "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
+ "version": "2.5.2",
+ "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz",
+ "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==",
"dev": true,
"dependencies": {
"bytes": "3.1.2",
@@ -7398,9 +7441,9 @@
}
},
"node_modules/send": {
- "version": "0.18.0",
- "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
- "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==",
+ "version": "0.19.0",
+ "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
+ "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==",
"dev": true,
"dependencies": {
"debug": "2.6.9",
@@ -7445,6 +7488,15 @@
"node": ">= 0.8"
}
},
+ "node_modules/send/node_modules/encodeurl": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
+ "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+ "dev": true,
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
"node_modules/send/node_modules/mime": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
@@ -7472,6 +7524,14 @@
"node": ">= 0.8"
}
},
+ "node_modules/serialize-javascript": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz",
+ "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==",
+ "dependencies": {
+ "randombytes": "^2.1.0"
+ }
+ },
"node_modules/serve-index": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz",
@@ -7533,15 +7593,15 @@
"dev": true
},
"node_modules/serve-static": {
- "version": "1.15.0",
- "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
- "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==",
+ "version": "1.16.2",
+ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
+ "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==",
"dev": true,
"dependencies": {
- "encodeurl": "~1.0.2",
+ "encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"parseurl": "~1.3.3",
- "send": "0.18.0"
+ "send": "0.19.0"
},
"engines": {
"node": ">= 0.8.0"
@@ -7552,6 +7612,23 @@
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
"integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
},
+ "node_modules/set-function-length": {
+ "version": "1.2.2",
+ "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
+ "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==",
+ "dev": true,
+ "dependencies": {
+ "define-data-property": "^1.1.4",
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2",
+ "get-intrinsic": "^1.2.4",
+ "gopd": "^1.0.1",
+ "has-property-descriptors": "^1.0.2"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/setprototypeof": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
@@ -7603,14 +7680,18 @@
}
},
"node_modules/side-channel": {
- "version": "1.0.4",
- "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
- "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz",
+ "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==",
"dev": true,
"dependencies": {
- "call-bind": "^1.0.0",
- "get-intrinsic": "^1.0.2",
- "object-inspect": "^1.9.0"
+ "call-bind": "^1.0.7",
+ "es-errors": "^1.3.0",
+ "get-intrinsic": "^1.2.4",
+ "object-inspect": "^1.13.1"
+ },
+ "engines": {
+ "node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -8009,13 +8090,21 @@
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
"dev": true
},
+ "node_modules/tapable": {
+ "version": "2.2.1",
+ "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
+ "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==",
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/terser": {
- "version": "5.15.0",
- "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.0.tgz",
- "integrity": "sha512-L1BJiXVmheAQQy+as0oF3Pwtlo4s3Wi1X2zNZ2NxOB4wx9bdS9Vk67XQENLFdLYGCK/Z2di53mTj/hBafR+dTA==",
+ "version": "5.31.6",
+ "resolved": "https://registry.npmjs.org/terser/-/terser-5.31.6.tgz",
+ "integrity": "sha512-PQ4DAriWzKj+qgehQ7LK5bQqCFNMmlhjR2PFFLuqGCpuCAauxemVBWwWOxo3UIwWQx8+Pr61Df++r76wDmkQBg==",
"dependencies": {
- "@jridgewell/source-map": "^0.3.2",
- "acorn": "^8.5.0",
+ "@jridgewell/source-map": "^0.3.3",
+ "acorn": "^8.8.2",
"commander": "^2.20.0",
"source-map-support": "~0.5.20"
},
@@ -8027,15 +8116,15 @@
}
},
"node_modules/terser-webpack-plugin": {
- "version": "5.3.6",
- "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.6.tgz",
- "integrity": "sha512-kfLFk+PoLUQIbLmB1+PZDMRSZS99Mp+/MHqDNmMA6tOItzRt+Npe3E+fsMs5mfcM0wCtrrdU387UnV+vnSffXQ==",
+ "version": "5.3.10",
+ "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz",
+ "integrity": "sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==",
"dependencies": {
- "@jridgewell/trace-mapping": "^0.3.14",
+ "@jridgewell/trace-mapping": "^0.3.20",
"jest-worker": "^27.4.5",
"schema-utils": "^3.1.1",
- "serialize-javascript": "^6.0.0",
- "terser": "^5.14.1"
+ "serialize-javascript": "^6.0.1",
+ "terser": "^5.26.0"
},
"engines": {
"node": ">= 10.13.0"
@@ -8076,18 +8165,10 @@
"url": "https://opencollective.com/webpack"
}
},
- "node_modules/terser-webpack-plugin/node_modules/serialize-javascript": {
- "version": "6.0.0",
- "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz",
- "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==",
- "dependencies": {
- "randombytes": "^2.1.0"
- }
- },
"node_modules/terser/node_modules/acorn": {
- "version": "8.8.0",
- "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz",
- "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==",
+ "version": "8.12.1",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
+ "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==",
"bin": {
"acorn": "bin/acorn"
},
@@ -8127,6 +8208,18 @@
"node": ">=4"
}
},
+ "node_modules/to-regex-range": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+ "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+ "dev": true,
+ "dependencies": {
+ "is-number": "^7.0.0"
+ },
+ "engines": {
+ "node": ">=8.0"
+ }
+ },
"node_modules/toidentifier": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
@@ -8287,9 +8380,9 @@
}
},
"node_modules/update-browserslist-db": {
- "version": "1.0.11",
- "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz",
- "integrity": "sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==",
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz",
+ "integrity": "sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==",
"funding": [
{
"type": "opencollective",
@@ -8305,8 +8398,8 @@
}
],
"dependencies": {
- "escalade": "^3.1.1",
- "picocolors": "^1.0.0"
+ "escalade": "^3.1.2",
+ "picocolors": "^1.0.1"
},
"bin": {
"update-browserslist-db": "cli.js"
@@ -8375,9 +8468,9 @@
}
},
"node_modules/watchpack": {
- "version": "2.4.0",
- "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz",
- "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==",
+ "version": "2.4.2",
+ "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz",
+ "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==",
"dependencies": {
"glob-to-regexp": "^0.4.1",
"graceful-fs": "^4.1.2"
@@ -8396,33 +8489,32 @@
}
},
"node_modules/webpack": {
- "version": "5.76.0",
- "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.0.tgz",
- "integrity": "sha512-l5sOdYBDunyf72HW8dF23rFtWq/7Zgvt/9ftMof71E/yUb1YLOBmTgA2K4vQthB3kotMrSj609txVE0dnr2fjA==",
- "dependencies": {
- "@types/eslint-scope": "^3.7.3",
- "@types/estree": "^0.0.51",
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/wasm-edit": "1.11.1",
- "@webassemblyjs/wasm-parser": "1.11.1",
+ "version": "5.94.0",
+ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.94.0.tgz",
+ "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==",
+ "dependencies": {
+ "@types/estree": "^1.0.5",
+ "@webassemblyjs/ast": "^1.12.1",
+ "@webassemblyjs/wasm-edit": "^1.12.1",
+ "@webassemblyjs/wasm-parser": "^1.12.1",
"acorn": "^8.7.1",
- "acorn-import-assertions": "^1.7.6",
- "browserslist": "^4.14.5",
+ "acorn-import-attributes": "^1.9.5",
+ "browserslist": "^4.21.10",
"chrome-trace-event": "^1.0.2",
- "enhanced-resolve": "^5.10.0",
- "es-module-lexer": "^0.9.0",
+ "enhanced-resolve": "^5.17.1",
+ "es-module-lexer": "^1.2.1",
"eslint-scope": "5.1.1",
"events": "^3.2.0",
"glob-to-regexp": "^0.4.1",
- "graceful-fs": "^4.2.9",
+ "graceful-fs": "^4.2.11",
"json-parse-even-better-errors": "^2.3.1",
"loader-runner": "^4.2.0",
"mime-types": "^2.1.27",
"neo-async": "^2.6.2",
- "schema-utils": "^3.1.0",
+ "schema-utils": "^3.2.0",
"tapable": "^2.1.1",
- "terser-webpack-plugin": "^5.1.3",
- "watchpack": "^2.4.0",
+ "terser-webpack-plugin": "^5.3.10",
+ "watchpack": "^2.4.1",
"webpack-sources": "^3.2.3"
},
"bin": {
@@ -8830,9 +8922,9 @@
}
},
"node_modules/webpack/node_modules/acorn": {
- "version": "8.8.0",
- "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz",
- "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==",
+ "version": "8.12.1",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
+ "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==",
"bin": {
"acorn": "bin/acorn"
},
@@ -8840,30 +8932,18 @@
"node": ">=0.4.0"
}
},
- "node_modules/webpack/node_modules/acorn-import-assertions": {
- "version": "1.8.0",
- "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz",
- "integrity": "sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw==",
+ "node_modules/webpack/node_modules/acorn-import-attributes": {
+ "version": "1.9.5",
+ "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz",
+ "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==",
"peerDependencies": {
"acorn": "^8"
}
},
- "node_modules/webpack/node_modules/enhanced-resolve": {
- "version": "5.10.0",
- "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz",
- "integrity": "sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ==",
- "dependencies": {
- "graceful-fs": "^4.2.4",
- "tapable": "^2.2.0"
- },
- "engines": {
- "node": ">=10.13.0"
- }
- },
"node_modules/webpack/node_modules/schema-utils": {
- "version": "3.1.1",
- "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz",
- "integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==",
+ "version": "3.3.0",
+ "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz",
+ "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==",
"dependencies": {
"@types/json-schema": "^7.0.8",
"ajv": "^6.12.5",
@@ -8877,14 +8957,6 @@
"url": "https://opencollective.com/webpack"
}
},
- "node_modules/webpack/node_modules/tapable": {
- "version": "2.2.1",
- "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
- "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==",
- "engines": {
- "node": ">=6"
- }
- },
"node_modules/webpack/node_modules/webpack-sources": {
"version": "3.2.3",
"resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz",
@@ -10500,13 +10572,13 @@
"dev": true
},
"@jridgewell/gen-mapping": {
- "version": "0.3.2",
- "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz",
- "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==",
+ "version": "0.3.5",
+ "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz",
+ "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==",
"requires": {
- "@jridgewell/set-array": "^1.0.1",
+ "@jridgewell/set-array": "^1.2.1",
"@jridgewell/sourcemap-codec": "^1.4.10",
- "@jridgewell/trace-mapping": "^0.3.9"
+ "@jridgewell/trace-mapping": "^0.3.24"
}
},
"@jridgewell/resolve-uri": {
@@ -10515,17 +10587,17 @@
"integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w=="
},
"@jridgewell/set-array": {
- "version": "1.1.2",
- "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz",
- "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw=="
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz",
+ "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A=="
},
"@jridgewell/source-map": {
- "version": "0.3.2",
- "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz",
- "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==",
+ "version": "0.3.6",
+ "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz",
+ "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==",
"requires": {
- "@jridgewell/gen-mapping": "^0.3.0",
- "@jridgewell/trace-mapping": "^0.3.9"
+ "@jridgewell/gen-mapping": "^0.3.5",
+ "@jridgewell/trace-mapping": "^0.3.25"
}
},
"@jridgewell/sourcemap-codec": {
@@ -10534,9 +10606,9 @@
"integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw=="
},
"@jridgewell/trace-mapping": {
- "version": "0.3.19",
- "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz",
- "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==",
+ "version": "0.3.25",
+ "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz",
+ "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==",
"requires": {
"@jridgewell/resolve-uri": "^3.1.0",
"@jridgewell/sourcemap-codec": "^1.4.14"
@@ -10598,24 +10670,6 @@
"@types/node": "*"
}
},
- "@types/eslint": {
- "version": "8.4.6",
- "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.6.tgz",
- "integrity": "sha512-/fqTbjxyFUaYNO7VcW5g+4npmqVACz1bB7RTHYuLj+PRjw9hrCwrUXVQFpChUS0JsyEFvMZ7U/PfmvWgxJhI9g==",
- "requires": {
- "@types/estree": "*",
- "@types/json-schema": "*"
- }
- },
- "@types/eslint-scope": {
- "version": "3.7.4",
- "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.4.tgz",
- "integrity": "sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==",
- "requires": {
- "@types/eslint": "*",
- "@types/estree": "*"
- }
- },
"@types/eslint-visitor-keys": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz",
@@ -10623,9 +10677,9 @@
"dev": true
},
"@types/estree": {
- "version": "0.0.51",
- "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.51.tgz",
- "integrity": "sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ=="
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz",
+ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw=="
},
"@types/express": {
"version": "4.17.13",
@@ -10935,133 +10989,133 @@
}
},
"@webassemblyjs/ast": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz",
- "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz",
+ "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==",
"requires": {
- "@webassemblyjs/helper-numbers": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1"
+ "@webassemblyjs/helper-numbers": "1.11.6",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6"
}
},
"@webassemblyjs/floating-point-hex-parser": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz",
- "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ=="
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz",
+ "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw=="
},
"@webassemblyjs/helper-api-error": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz",
- "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg=="
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz",
+ "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q=="
},
"@webassemblyjs/helper-buffer": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz",
- "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA=="
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz",
+ "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw=="
},
"@webassemblyjs/helper-numbers": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz",
- "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==",
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz",
+ "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==",
"requires": {
- "@webassemblyjs/floating-point-hex-parser": "1.11.1",
- "@webassemblyjs/helper-api-error": "1.11.1",
+ "@webassemblyjs/floating-point-hex-parser": "1.11.6",
+ "@webassemblyjs/helper-api-error": "1.11.6",
"@xtuc/long": "4.2.2"
}
},
"@webassemblyjs/helper-wasm-bytecode": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz",
- "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q=="
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz",
+ "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA=="
},
"@webassemblyjs/helper-wasm-section": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz",
- "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz",
+ "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==",
"requires": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-buffer": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
- "@webassemblyjs/wasm-gen": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-buffer": "1.12.1",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+ "@webassemblyjs/wasm-gen": "1.12.1"
}
},
"@webassemblyjs/ieee754": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz",
- "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==",
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz",
+ "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==",
"requires": {
"@xtuc/ieee754": "^1.2.0"
}
},
"@webassemblyjs/leb128": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz",
- "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==",
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz",
+ "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==",
"requires": {
"@xtuc/long": "4.2.2"
}
},
"@webassemblyjs/utf8": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz",
- "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ=="
+ "version": "1.11.6",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz",
+ "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA=="
},
"@webassemblyjs/wasm-edit": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz",
- "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz",
+ "integrity": "sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g==",
"requires": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-buffer": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
- "@webassemblyjs/helper-wasm-section": "1.11.1",
- "@webassemblyjs/wasm-gen": "1.11.1",
- "@webassemblyjs/wasm-opt": "1.11.1",
- "@webassemblyjs/wasm-parser": "1.11.1",
- "@webassemblyjs/wast-printer": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-buffer": "1.12.1",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+ "@webassemblyjs/helper-wasm-section": "1.12.1",
+ "@webassemblyjs/wasm-gen": "1.12.1",
+ "@webassemblyjs/wasm-opt": "1.12.1",
+ "@webassemblyjs/wasm-parser": "1.12.1",
+ "@webassemblyjs/wast-printer": "1.12.1"
}
},
"@webassemblyjs/wasm-gen": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz",
- "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz",
+ "integrity": "sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w==",
"requires": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
- "@webassemblyjs/ieee754": "1.11.1",
- "@webassemblyjs/leb128": "1.11.1",
- "@webassemblyjs/utf8": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+ "@webassemblyjs/ieee754": "1.11.6",
+ "@webassemblyjs/leb128": "1.11.6",
+ "@webassemblyjs/utf8": "1.11.6"
}
},
"@webassemblyjs/wasm-opt": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz",
- "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz",
+ "integrity": "sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg==",
"requires": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-buffer": "1.11.1",
- "@webassemblyjs/wasm-gen": "1.11.1",
- "@webassemblyjs/wasm-parser": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-buffer": "1.12.1",
+ "@webassemblyjs/wasm-gen": "1.12.1",
+ "@webassemblyjs/wasm-parser": "1.12.1"
}
},
"@webassemblyjs/wasm-parser": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz",
- "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz",
+ "integrity": "sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ==",
"requires": {
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/helper-api-error": "1.11.1",
- "@webassemblyjs/helper-wasm-bytecode": "1.11.1",
- "@webassemblyjs/ieee754": "1.11.1",
- "@webassemblyjs/leb128": "1.11.1",
- "@webassemblyjs/utf8": "1.11.1"
+ "@webassemblyjs/ast": "1.12.1",
+ "@webassemblyjs/helper-api-error": "1.11.6",
+ "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+ "@webassemblyjs/ieee754": "1.11.6",
+ "@webassemblyjs/leb128": "1.11.6",
+ "@webassemblyjs/utf8": "1.11.6"
}
},
"@webassemblyjs/wast-printer": {
- "version": "1.11.1",
- "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz",
- "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==",
+ "version": "1.12.1",
+ "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz",
+ "integrity": "sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA==",
"requires": {
- "@webassemblyjs/ast": "1.11.1",
+ "@webassemblyjs/ast": "1.12.1",
"@xtuc/long": "4.2.2"
}
},
@@ -11376,21 +11430,21 @@
"dev": true
},
"body-parser": {
- "version": "1.20.0",
- "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
- "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==",
+ "version": "1.20.3",
+ "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
+ "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
"dev": true,
"requires": {
"bytes": "3.1.2",
- "content-type": "~1.0.4",
+ "content-type": "~1.0.5",
"debug": "2.6.9",
"depd": "2.0.0",
"destroy": "1.2.0",
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"on-finished": "2.4.1",
- "qs": "6.10.3",
- "raw-body": "2.5.1",
+ "qs": "6.13.0",
+ "raw-body": "2.5.2",
"type-is": "~1.6.18",
"unpipe": "1.0.0"
},
@@ -11446,15 +11500,24 @@
"concat-map": "0.0.1"
}
},
+ "braces": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+ "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
+ "dev": true,
+ "requires": {
+ "fill-range": "^7.1.1"
+ }
+ },
"browserslist": {
- "version": "4.21.9",
- "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.9.tgz",
- "integrity": "sha512-M0MFoZzbUrRU4KNfCrDLnvyE7gub+peetoTid3TBIqtunaDJyXlwhakT+/VkvSXcfIzFfK/nkCs4nmyTmxdNSg==",
+ "version": "4.23.3",
+ "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.3.tgz",
+ "integrity": "sha512-btwCFJVjI4YWDNfau8RhZ+B1Q/VLoUITrm3RlP6y1tYGWIOa+InuYiRGXUBXo8nA1qKmHMyLB/iVQg5TT4eFoA==",
"requires": {
- "caniuse-lite": "^1.0.30001503",
- "electron-to-chromium": "^1.4.431",
- "node-releases": "^2.0.12",
- "update-browserslist-db": "^1.0.11"
+ "caniuse-lite": "^1.0.30001646",
+ "electron-to-chromium": "^1.5.4",
+ "node-releases": "^2.0.18",
+ "update-browserslist-db": "^1.1.0"
}
},
"buffer-from": {
@@ -11469,13 +11532,16 @@
"dev": true
},
"call-bind": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
- "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz",
+ "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==",
"dev": true,
"requires": {
- "function-bind": "^1.1.1",
- "get-intrinsic": "^1.0.2"
+ "es-define-property": "^1.0.0",
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2",
+ "get-intrinsic": "^1.2.4",
+ "set-function-length": "^1.2.1"
}
},
"callsites": {
@@ -11489,9 +11555,9 @@
"integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg=="
},
"caniuse-lite": {
- "version": "1.0.30001517",
- "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001517.tgz",
- "integrity": "sha512-Vdhm5S11DaFVLlyiKu4hiUTkpZu+y1KA/rZZqVQfOD5YdDT/eQKlkt7NaE0WGOFgX32diqt9MiP9CAiFeRklaA=="
+ "version": "1.0.30001653",
+ "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001653.tgz",
+ "integrity": "sha512-XGWQVB8wFQ2+9NZwZ10GxTYC5hk0Fa+q8cSkr0tgvMhYhMHP/QC+WTgrePMDBWiWc/pV+1ik82Al20XOK25Gcw=="
},
"chalk": {
"version": "2.4.2",
@@ -11517,41 +11583,6 @@
"is-glob": "~4.0.1",
"normalize-path": "~3.0.0",
"readdirp": "~3.6.0"
- },
- "dependencies": {
- "braces": {
- "version": "3.0.2",
- "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
- "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
- "dev": true,
- "requires": {
- "fill-range": "^7.0.1"
- }
- },
- "fill-range": {
- "version": "7.0.1",
- "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
- "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
- "dev": true,
- "requires": {
- "to-regex-range": "^5.0.1"
- }
- },
- "is-number": {
- "version": "7.0.0",
- "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
- "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
- "dev": true
- },
- "to-regex-range": {
- "version": "5.0.1",
- "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
- "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
- "dev": true,
- "requires": {
- "is-number": "^7.0.0"
- }
- }
}
},
"chrome-trace-event": {
@@ -11736,9 +11767,9 @@
}
},
"content-type": {
- "version": "1.0.4",
- "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
- "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==",
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
+ "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
"dev": true
},
"convert-source-map": {
@@ -11750,9 +11781,9 @@
}
},
"cookie": {
- "version": "0.5.0",
- "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
- "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
+ "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
"dev": true
},
"cookie-signature": {
@@ -11895,6 +11926,17 @@
"execa": "^5.0.0"
}
},
+ "define-data-property": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
+ "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==",
+ "dev": true,
+ "requires": {
+ "es-define-property": "^1.0.0",
+ "es-errors": "^1.3.0",
+ "gopd": "^1.0.1"
+ }
+ },
"define-lazy-prop": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
@@ -12051,9 +12093,9 @@
"dev": true
},
"electron-to-chromium": {
- "version": "1.4.467",
- "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.467.tgz",
- "integrity": "sha512-2qI70O+rR4poYeF2grcuS/bCps5KJh6y1jtZMDDEteyKJQrzLOEhFyXCLcHW6DTBjKjWkk26JhWoAi+Ux9A0fg=="
+ "version": "1.5.13",
+ "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.13.tgz",
+ "integrity": "sha512-lbBcvtIJ4J6sS4tb5TLp1b4LyfCdMkwStzXPyAgVgTRAsep4bvrAGaBOP7ZJtQMNJpSQ9SqG4brWOroNaQtm7Q=="
},
"emoji-regex": {
"version": "8.0.0",
@@ -12067,11 +12109,20 @@
"integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q=="
},
"encodeurl": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
- "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
+ "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"dev": true
},
+ "enhanced-resolve": {
+ "version": "5.17.1",
+ "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz",
+ "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==",
+ "requires": {
+ "graceful-fs": "^4.2.4",
+ "tapable": "^2.2.0"
+ }
+ },
"enquirer": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
@@ -12128,10 +12179,25 @@
"unbox-primitive": "^1.0.1"
}
},
+ "es-define-property": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
+ "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
+ "dev": true,
+ "requires": {
+ "get-intrinsic": "^1.2.4"
+ }
+ },
+ "es-errors": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+ "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+ "dev": true
+ },
"es-module-lexer": {
- "version": "0.9.3",
- "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.9.3.tgz",
- "integrity": "sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ=="
+ "version": "1.5.4",
+ "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
+ "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw=="
},
"es-to-primitive": {
"version": "1.2.1",
@@ -12145,9 +12211,9 @@
}
},
"escalade": {
- "version": "3.1.1",
- "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
- "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
+ "version": "3.1.2",
+ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz",
+ "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA=="
},
"escape-html": {
"version": "1.0.3",
@@ -12469,37 +12535,37 @@
}
},
"express": {
- "version": "4.18.1",
- "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz",
- "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==",
+ "version": "4.21.0",
+ "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz",
+ "integrity": "sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==",
"dev": true,
"requires": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
- "body-parser": "1.20.0",
+ "body-parser": "1.20.3",
"content-disposition": "0.5.4",
"content-type": "~1.0.4",
- "cookie": "0.5.0",
+ "cookie": "0.6.0",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "2.0.0",
- "encodeurl": "~1.0.2",
+ "encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
- "finalhandler": "1.2.0",
+ "finalhandler": "1.3.1",
"fresh": "0.5.2",
"http-errors": "2.0.0",
- "merge-descriptors": "1.0.1",
+ "merge-descriptors": "1.0.3",
"methods": "~1.1.2",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
- "path-to-regexp": "0.1.7",
+ "path-to-regexp": "0.1.10",
"proxy-addr": "~2.0.7",
- "qs": "6.10.3",
+ "qs": "6.13.0",
"range-parser": "~1.2.1",
"safe-buffer": "5.2.1",
- "send": "0.18.0",
- "serve-static": "1.15.0",
+ "send": "0.19.0",
+ "serve-static": "1.16.2",
"setprototypeof": "1.2.0",
"statuses": "2.0.1",
"type-is": "~1.6.18",
@@ -12594,14 +12660,23 @@
"flat-cache": "^3.0.4"
}
},
+ "fill-range": {
+ "version": "7.1.1",
+ "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+ "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
+ "dev": true,
+ "requires": {
+ "to-regex-range": "^5.0.1"
+ }
+ },
"finalhandler": {
- "version": "1.2.0",
- "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
- "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==",
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz",
+ "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==",
"dev": true,
"requires": {
"debug": "2.6.9",
- "encodeurl": "~1.0.2",
+ "encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
@@ -12712,9 +12787,9 @@
"optional": true
},
"function-bind": {
- "version": "1.1.1",
- "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
- "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+ "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="
},
"functional-red-black-tree": {
"version": "1.0.1",
@@ -12734,14 +12809,16 @@
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
},
"get-intrinsic": {
- "version": "1.1.1",
- "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz",
- "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==",
+ "version": "1.2.4",
+ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
+ "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
"dev": true,
"requires": {
- "function-bind": "^1.1.1",
- "has": "^1.0.3",
- "has-symbols": "^1.0.1"
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2",
+ "has-proto": "^1.0.1",
+ "has-symbols": "^1.0.3",
+ "hasown": "^2.0.0"
}
},
"get-stdin": {
@@ -12821,10 +12898,19 @@
}
}
},
+ "gopd": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+ "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+ "dev": true,
+ "requires": {
+ "get-intrinsic": "^1.1.3"
+ }
+ },
"graceful-fs": {
- "version": "4.2.10",
- "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz",
- "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA=="
+ "version": "4.2.11",
+ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
+ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="
},
"gzip-size": {
"version": "6.0.0",
@@ -12860,10 +12946,25 @@
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
},
- "has-symbols": {
+ "has-property-descriptors": {
"version": "1.0.2",
- "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
- "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
+ "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
+ "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==",
+ "dev": true,
+ "requires": {
+ "es-define-property": "^1.0.0"
+ }
+ },
+ "has-proto": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz",
+ "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==",
+ "dev": true
+ },
+ "has-symbols": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
+ "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
"dev": true
},
"has-tostringtag": {
@@ -12875,6 +12976,15 @@
"has-symbols": "^1.0.2"
}
},
+ "hasown": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+ "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+ "dev": true,
+ "requires": {
+ "function-bind": "^1.1.2"
+ }
+ },
"history": {
"version": "4.10.1",
"resolved": "https://registry.npmjs.org/history/-/history-4.10.1.tgz",
@@ -12998,30 +13108,6 @@
"micromatch": "^4.0.2"
},
"dependencies": {
- "braces": {
- "version": "3.0.2",
- "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
- "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
- "dev": true,
- "requires": {
- "fill-range": "^7.0.1"
- }
- },
- "fill-range": {
- "version": "7.0.1",
- "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
- "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
- "dev": true,
- "requires": {
- "to-regex-range": "^5.0.1"
- }
- },
- "is-number": {
- "version": "7.0.0",
- "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
- "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
- "dev": true
- },
"micromatch": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
@@ -13031,15 +13117,6 @@
"braces": "^3.0.2",
"picomatch": "^2.3.1"
}
- },
- "to-regex-range": {
- "version": "5.0.1",
- "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
- "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
- "dev": true,
- "requires": {
- "is-number": "^7.0.0"
- }
}
}
},
@@ -13231,6 +13308,12 @@
"integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==",
"dev": true
},
+ "is-number": {
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+ "dev": true
+ },
"is-number-object": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.6.tgz",
@@ -13561,9 +13644,9 @@
}
},
"merge-descriptors": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
- "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==",
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
+ "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==",
"dev": true
},
"merge-stream": {
@@ -13668,9 +13751,9 @@
"dev": true
},
"node-releases": {
- "version": "2.0.13",
- "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.13.tgz",
- "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ=="
+ "version": "2.0.18",
+ "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz",
+ "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g=="
},
"normalize-path": {
"version": "3.0.0",
@@ -13693,9 +13776,9 @@
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"object-inspect": {
- "version": "1.11.0",
- "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz",
- "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==",
+ "version": "1.13.2",
+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz",
+ "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==",
"dev": true
},
"object-keys": {
@@ -13927,9 +14010,9 @@
"integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="
},
"path-to-regexp": {
- "version": "0.1.7",
- "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
- "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==",
+ "version": "0.1.10",
+ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz",
+ "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==",
"dev": true
},
"path-type": {
@@ -13938,9 +14021,9 @@
"integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw=="
},
"picocolors": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz",
- "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ=="
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz",
+ "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew=="
},
"picomatch": {
"version": "2.3.1",
@@ -14108,12 +14191,12 @@
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
},
"qs": {
- "version": "6.10.3",
- "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
- "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
+ "version": "6.13.0",
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
+ "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"dev": true,
"requires": {
- "side-channel": "^1.0.4"
+ "side-channel": "^1.0.6"
}
},
"randombytes": {
@@ -14131,9 +14214,9 @@
"dev": true
},
"raw-body": {
- "version": "2.5.1",
- "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
- "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
+ "version": "2.5.2",
+ "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz",
+ "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==",
"dev": true,
"requires": {
"bytes": "3.1.2",
@@ -14639,9 +14722,9 @@
"dev": true
},
"send": {
- "version": "0.18.0",
- "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
- "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==",
+ "version": "0.19.0",
+ "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
+ "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==",
"dev": true,
"requires": {
"debug": "2.6.9",
@@ -14682,6 +14765,12 @@
"integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
"dev": true
},
+ "encodeurl": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
+ "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+ "dev": true
+ },
"mime": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
@@ -14702,6 +14791,14 @@
}
}
},
+ "serialize-javascript": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz",
+ "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==",
+ "requires": {
+ "randombytes": "^2.1.0"
+ }
+ },
"serve-index": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz",
@@ -14759,15 +14856,15 @@
}
},
"serve-static": {
- "version": "1.15.0",
- "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
- "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==",
+ "version": "1.16.2",
+ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
+ "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==",
"dev": true,
"requires": {
- "encodeurl": "~1.0.2",
+ "encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"parseurl": "~1.3.3",
- "send": "0.18.0"
+ "send": "0.19.0"
}
},
"set-blocking": {
@@ -14775,6 +14872,20 @@
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
"integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
},
+ "set-function-length": {
+ "version": "1.2.2",
+ "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
+ "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==",
+ "dev": true,
+ "requires": {
+ "define-data-property": "^1.1.4",
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2",
+ "get-intrinsic": "^1.2.4",
+ "gopd": "^1.0.1",
+ "has-property-descriptors": "^1.0.2"
+ }
+ },
"setprototypeof": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
@@ -14814,14 +14925,15 @@
}
},
"side-channel": {
- "version": "1.0.4",
- "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
- "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz",
+ "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==",
"dev": true,
"requires": {
- "call-bind": "^1.0.0",
- "get-intrinsic": "^1.0.2",
- "object-inspect": "^1.9.0"
+ "call-bind": "^1.0.7",
+ "es-errors": "^1.3.0",
+ "get-intrinsic": "^1.2.4",
+ "object-inspect": "^1.13.1"
}
},
"signal-exit": {
@@ -15125,34 +15237,39 @@
}
}
},
+ "tapable": {
+ "version": "2.2.1",
+ "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
+ "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ=="
+ },
"terser": {
- "version": "5.15.0",
- "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.0.tgz",
- "integrity": "sha512-L1BJiXVmheAQQy+as0oF3Pwtlo4s3Wi1X2zNZ2NxOB4wx9bdS9Vk67XQENLFdLYGCK/Z2di53mTj/hBafR+dTA==",
+ "version": "5.31.6",
+ "resolved": "https://registry.npmjs.org/terser/-/terser-5.31.6.tgz",
+ "integrity": "sha512-PQ4DAriWzKj+qgehQ7LK5bQqCFNMmlhjR2PFFLuqGCpuCAauxemVBWwWOxo3UIwWQx8+Pr61Df++r76wDmkQBg==",
"requires": {
- "@jridgewell/source-map": "^0.3.2",
- "acorn": "^8.5.0",
+ "@jridgewell/source-map": "^0.3.3",
+ "acorn": "^8.8.2",
"commander": "^2.20.0",
"source-map-support": "~0.5.20"
},
"dependencies": {
"acorn": {
- "version": "8.8.0",
- "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz",
- "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w=="
+ "version": "8.12.1",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
+ "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg=="
}
}
},
"terser-webpack-plugin": {
- "version": "5.3.6",
- "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.6.tgz",
- "integrity": "sha512-kfLFk+PoLUQIbLmB1+PZDMRSZS99Mp+/MHqDNmMA6tOItzRt+Npe3E+fsMs5mfcM0wCtrrdU387UnV+vnSffXQ==",
+ "version": "5.3.10",
+ "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz",
+ "integrity": "sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==",
"requires": {
- "@jridgewell/trace-mapping": "^0.3.14",
+ "@jridgewell/trace-mapping": "^0.3.20",
"jest-worker": "^27.4.5",
"schema-utils": "^3.1.1",
- "serialize-javascript": "^6.0.0",
- "terser": "^5.14.1"
+ "serialize-javascript": "^6.0.1",
+ "terser": "^5.26.0"
},
"dependencies": {
"schema-utils": {
@@ -15164,14 +15281,6 @@
"ajv": "^6.12.5",
"ajv-keywords": "^3.5.2"
}
- },
- "serialize-javascript": {
- "version": "6.0.0",
- "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz",
- "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==",
- "requires": {
- "randombytes": "^2.1.0"
- }
}
}
},
@@ -15204,6 +15313,15 @@
"resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
"integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4="
},
+ "to-regex-range": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+ "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+ "dev": true,
+ "requires": {
+ "is-number": "^7.0.0"
+ }
+ },
"toidentifier": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
@@ -15317,12 +15435,12 @@
"dev": true
},
"update-browserslist-db": {
- "version": "1.0.11",
- "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz",
- "integrity": "sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==",
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz",
+ "integrity": "sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==",
"requires": {
- "escalade": "^3.1.1",
- "picocolors": "^1.0.0"
+ "escalade": "^3.1.2",
+ "picocolors": "^1.0.1"
}
},
"uri-js": {
@@ -15376,9 +15494,9 @@
}
},
"watchpack": {
- "version": "2.4.0",
- "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz",
- "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==",
+ "version": "2.4.2",
+ "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz",
+ "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==",
"requires": {
"glob-to-regexp": "^0.4.1",
"graceful-fs": "^4.1.2"
@@ -15394,71 +15512,56 @@
}
},
"webpack": {
- "version": "5.76.0",
- "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.0.tgz",
- "integrity": "sha512-l5sOdYBDunyf72HW8dF23rFtWq/7Zgvt/9ftMof71E/yUb1YLOBmTgA2K4vQthB3kotMrSj609txVE0dnr2fjA==",
- "requires": {
- "@types/eslint-scope": "^3.7.3",
- "@types/estree": "^0.0.51",
- "@webassemblyjs/ast": "1.11.1",
- "@webassemblyjs/wasm-edit": "1.11.1",
- "@webassemblyjs/wasm-parser": "1.11.1",
+ "version": "5.94.0",
+ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.94.0.tgz",
+ "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==",
+ "requires": {
+ "@types/estree": "^1.0.5",
+ "@webassemblyjs/ast": "^1.12.1",
+ "@webassemblyjs/wasm-edit": "^1.12.1",
+ "@webassemblyjs/wasm-parser": "^1.12.1",
"acorn": "^8.7.1",
- "acorn-import-assertions": "^1.7.6",
- "browserslist": "^4.14.5",
+ "acorn-import-attributes": "^1.9.5",
+ "browserslist": "^4.21.10",
"chrome-trace-event": "^1.0.2",
- "enhanced-resolve": "^5.10.0",
- "es-module-lexer": "^0.9.0",
+ "enhanced-resolve": "^5.17.1",
+ "es-module-lexer": "^1.2.1",
"eslint-scope": "5.1.1",
"events": "^3.2.0",
"glob-to-regexp": "^0.4.1",
- "graceful-fs": "^4.2.9",
+ "graceful-fs": "^4.2.11",
"json-parse-even-better-errors": "^2.3.1",
"loader-runner": "^4.2.0",
"mime-types": "^2.1.27",
"neo-async": "^2.6.2",
- "schema-utils": "^3.1.0",
+ "schema-utils": "^3.2.0",
"tapable": "^2.1.1",
- "terser-webpack-plugin": "^5.1.3",
- "watchpack": "^2.4.0",
+ "terser-webpack-plugin": "^5.3.10",
+ "watchpack": "^2.4.1",
"webpack-sources": "^3.2.3"
},
"dependencies": {
"acorn": {
- "version": "8.8.0",
- "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz",
- "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w=="
+ "version": "8.12.1",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
+ "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg=="
},
- "acorn-import-assertions": {
- "version": "1.8.0",
- "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz",
- "integrity": "sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw==",
+ "acorn-import-attributes": {
+ "version": "1.9.5",
+ "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz",
+ "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==",
"requires": {}
},
- "enhanced-resolve": {
- "version": "5.10.0",
- "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz",
- "integrity": "sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ==",
- "requires": {
- "graceful-fs": "^4.2.4",
- "tapable": "^2.2.0"
- }
- },
"schema-utils": {
- "version": "3.1.1",
- "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz",
- "integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==",
+ "version": "3.3.0",
+ "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz",
+ "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==",
"requires": {
"@types/json-schema": "^7.0.8",
"ajv": "^6.12.5",
"ajv-keywords": "^3.5.2"
}
},
- "tapable": {
- "version": "2.2.1",
- "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
- "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ=="
- },
"webpack-sources": {
"version": "3.2.3",
"resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz",
diff --git a/cl/package.json b/cl/package.json
index 4beb099dfd..106063b630 100644
--- a/cl/package.json
+++ b/cl/package.json
@@ -43,7 +43,7 @@
"react-virtual": "^2.2.1",
"terser-webpack-plugin": "^5.3.6",
"typescript": "^4.2.4",
- "webpack": "^5.76.0",
+ "webpack": "^5.94.0",
"webpack-bundle-analyzer": "^4.4.1",
"webpack-cli": "^4.10.0",
"webpack-dev-server": "^4.10.0"
diff --git a/cl/people_db/filters.py b/cl/people_db/filters.py
index cf1cb314f1..8f6b30388e 100644
--- a/cl/people_db/filters.py
+++ b/cl/people_db/filters.py
@@ -276,9 +276,13 @@ class AttorneyFilter(NoEmptyFilterSet):
"cl.search.filters.DocketFilter",
field_name="roles__docket",
queryset=Docket.objects.all(),
+ distinct=True,
)
parties_represented = filters.RelatedFilter(
- PartyFilter, field_name="roles__party", queryset=Party.objects.all()
+ PartyFilter,
+ field_name="roles__party",
+ queryset=Party.objects.all(),
+ distinct=True,
)
class Meta:
diff --git a/cl/people_db/lookup_utils.py b/cl/people_db/lookup_utils.py
index 43be587b38..221e15a2a9 100644
--- a/cl/people_db/lookup_utils.py
+++ b/cl/people_db/lookup_utils.py
@@ -328,7 +328,8 @@ def find_just_name(text: str) -> str:
# Next up is full names followed by a comma
match_titles = re.search(
- "(((Van|VAN|De|DE|Da|DA)\s)?[A-Z][\w\-'']{2,}(\s(IV|I|II|III|V|Jr\.|JR\.|Sr\.|SR\.))?),",
+ # adding 'r' prefix to address python 3.12 strickness around escape sequences in string literals
+ r"(((Van|VAN|De|DE|Da|DA)\s)?[A-Z][\w\-'']{2,}(\s(IV|I|II|III|V|Jr\.|JR\.|Sr\.|SR\.))?),",
cleaned_text,
)
if match_titles:
diff --git a/cl/recap/factories.py b/cl/recap/factories.py
index a1b994d152..718c665451 100644
--- a/cl/recap/factories.py
+++ b/cl/recap/factories.py
@@ -33,7 +33,7 @@ class Meta:
pacer_case_id = Faker("pyint", min_value=100_000, max_value=400_000)
upload_type = FuzzyChoice(UPLOAD_TYPE.NAMES, getter=lambda c: c[0])
- filepath_local = FileField(filename=None)
+ filepath_local = FileField(filename="document.html")
class PacerFetchQueueFactory(DjangoModelFactory):
@@ -132,3 +132,20 @@ class DocketDataFactory(DictFactory):
docket_number = Faker("federal_district_docket_number")
date_filed = Faker("date_object")
ordered_by = "date_filed"
+ federal_dn_office_code = Faker("pyint", min_value=1, max_value=10)
+ federal_dn_case_type = FuzzyText(length=2, chars=string.ascii_lowercase)
+ federal_dn_judge_initials_assigned = FuzzyText(
+ length=5, chars=string.ascii_lowercase
+ )
+ federal_dn_judge_initials_referred = FuzzyText(
+ length=5, chars=string.ascii_lowercase
+ )
+ federal_defendant_number = Faker("pyint", min_value=1, max_value=999)
+
+
+class DocketEntryWithAttachmentsDataFactory(MinuteDocketEntryDataFactory):
+ attachments = List([SubFactory(AppellateAttachmentPageFactory)])
+
+
+class DocketDataWithAttachmentsFactory(DocketDataFactory):
+ docket_entries = List([SubFactory(DocketEntryWithAttachmentsDataFactory)])
diff --git a/cl/recap/management/commands/merge_idb_into_dockets.py b/cl/recap/management/commands/merge_idb_into_dockets.py
index 0fe62e0c85..ba90c71071 100644
--- a/cl/recap/management/commands/merge_idb_into_dockets.py
+++ b/cl/recap/management/commands/merge_idb_into_dockets.py
@@ -142,7 +142,9 @@ def update_any_missing_pacer_case_ids(options):
pass_through=d.pk,
docket_number=d.idb_data.docket_number,
court_id=d.idb_data.district_id,
- cookies=session.cookies,
+ cookies_data=SessionData(
+ session.cookies, session.proxy_address
+ ),
**params,
).set(queue=q),
update_docket_from_hidden_api.s().set(queue=q),
diff --git a/cl/recap/mergers.py b/cl/recap/mergers.py
index 5654275ada..6eccba198e 100644
--- a/cl/recap/mergers.py
+++ b/cl/recap/mergers.py
@@ -66,18 +66,47 @@
def confirm_docket_number_core_lookup_match(
docket: Docket,
docket_number: str,
+ federal_defendant_number: str | None = None,
+ federal_dn_judge_initials_assigned: str | None = None,
+ federal_dn_judge_initials_referred: str | None = None,
) -> Docket | None:
"""Confirm if the docket_number_core lookup match returns the right docket
- by confirming the docket_number also matches.
+ by confirming the docket_number and docket_number components also matches
+ if they're available.
:param docket: The docket matched by the lookup
:param docket_number: The incoming docket_number to lookup.
+ :param federal_defendant_number: The federal defendant number to validate
+ the match.
+ :param federal_dn_judge_initials_assigned: The judge's initials assigned to
+ validate the match.
+ :param federal_dn_judge_initials_referred: The judge's initials referred to
+ validate the match.
:return: The docket object if both dockets matched or otherwise None.
"""
existing_docket_number = clean_docket_number(docket.docket_number)
incoming_docket_number = clean_docket_number(docket_number)
if existing_docket_number != incoming_docket_number:
return None
+
+ # If the incoming data contains docket_number components and the docket
+ # also contains DN components, use them to confirm that the docket matches.
+ dn_components = {
+ "federal_defendant_number": federal_defendant_number,
+ "federal_dn_judge_initials_assigned": federal_dn_judge_initials_assigned,
+ "federal_dn_judge_initials_referred": federal_dn_judge_initials_referred,
+ }
+ # Only compare DN component values if both the incoming data and the docket contain
+ # non-None DN component values.
+ for dn_key, dn_value in dn_components.items():
+ incoming_dn_value = dn_value
+ docket_dn_value = getattr(docket, dn_key, None)
+ if (
+ incoming_dn_value
+ and docket_dn_value
+ and incoming_dn_value != docket_dn_value
+ ):
+ return None
return docket
@@ -85,6 +114,9 @@ async def find_docket_object(
court_id: str,
pacer_case_id: str | None,
docket_number: str,
+ federal_defendant_number: str | None,
+ federal_dn_judge_initials_assigned: str | None,
+ federal_dn_judge_initials_referred: str | None,
using: str = "default",
) -> Docket:
"""Attempt to find the docket based on the parsed docket data. If cannot be
@@ -93,6 +125,12 @@ async def find_docket_object(
:param court_id: The CourtListener court_id to lookup
:param pacer_case_id: The PACER case ID for the docket
:param docket_number: The docket number to lookup.
+ :param federal_defendant_number: The federal defendant number to validate
+ the match.
+ :param federal_dn_judge_initials_assigned: The judge's initials assigned to
+ validate the match.
+ :param federal_dn_judge_initials_referred: The judge's initials referred to
+ validate the match.
:param using: The database to use for the lookup queries.
:return The docket found or created.
"""
@@ -142,16 +180,41 @@ async def find_docket_object(
if kwargs.get("pacer_case_id") is None and kwargs.get(
"docket_number_core"
):
- d = confirm_docket_number_core_lookup_match(d, docket_number)
+ d = confirm_docket_number_core_lookup_match(
+ d,
+ docket_number,
+ federal_defendant_number,
+ federal_dn_judge_initials_assigned,
+ federal_dn_judge_initials_referred,
+ )
if d:
break # Nailed it!
elif count > 1:
- # Choose the oldest one and live with it.
- d = await ds.aearliest("date_created")
- if kwargs.get("pacer_case_id") is None and kwargs.get(
- "docket_number_core"
- ):
- d = confirm_docket_number_core_lookup_match(d, docket_number)
+ # If more than one docket matches, try refining the results using
+ # available docket_number components.
+ dn_components = {
+ "federal_defendant_number": federal_defendant_number,
+ "federal_dn_judge_initials_assigned": federal_dn_judge_initials_assigned,
+ "federal_dn_judge_initials_referred": federal_dn_judge_initials_referred,
+ }
+ dn_lookup = {
+ dn_key: dn_value
+ for dn_key, dn_value in dn_components.items()
+ if dn_value
+ }
+ dn_queryset = ds.filter(**dn_lookup).using(using)
+ count = await dn_queryset.acount()
+ if count == 1:
+ d = await dn_queryset.afirst()
+ else:
+ # Choose the oldest one and live with it.
+ d = await ds.aearliest("date_created")
+ if kwargs.get("pacer_case_id") is None and kwargs.get(
+ "docket_number_core"
+ ):
+ d = confirm_docket_number_core_lookup_match(
+ d, docket_number
+ )
if d:
break
if d is None:
@@ -333,6 +396,26 @@ async def update_docket_metadata(
d.referred_to_str = docket_data.get("referred_to_str") or d.referred_to_str
d.blocked, d.date_blocked = await get_blocked_status(d)
+ # Update docket_number components:
+ d.federal_dn_office_code = (
+ docket_data.get("federal_dn_office_code") or d.federal_dn_office_code
+ )
+ d.federal_dn_case_type = (
+ docket_data.get("federal_dn_case_type") or d.federal_dn_case_type
+ )
+ d.federal_dn_judge_initials_assigned = (
+ docket_data.get("federal_dn_judge_initials_assigned")
+ or d.federal_dn_judge_initials_assigned
+ )
+ d.federal_dn_judge_initials_referred = (
+ docket_data.get("federal_dn_judge_initials_referred")
+ or d.federal_dn_judge_initials_referred
+ )
+ d.federal_defendant_number = (
+ docket_data.get("federal_defendant_number")
+ or d.federal_defendant_number
+ )
+
return d
@@ -733,7 +816,7 @@ async def get_or_make_docket_entry(
async def add_docket_entries(
d: Docket,
docket_entries: list[dict[str, Any]],
- tags: list[str] | None = None,
+ tags: list[Tag] | None = None,
do_not_update_existing: bool = False,
) -> tuple[
tuple[list[DocketEntry], list[RECAPDocument]], list[RECAPDocument], bool
@@ -790,7 +873,7 @@ async def add_docket_entries(
await de.asave()
if tags:
for tag in tags:
- tag.tag_object(de)
+ await sync_to_async(tag.tag_object)(de)
if de_created:
content_updated = True
@@ -835,7 +918,11 @@ async def add_docket_entries(
params["document_type"] = RECAPDocument.ATTACHMENT
params["pacer_doc_id"] = docket_entry["pacer_doc_id"]
try:
- rd = await RECAPDocument.objects.aget(**params)
+ get_params = deepcopy(params)
+ if de_created is False and not appelate_court_id_exists:
+ del get_params["document_type"]
+ get_params["pacer_doc_id"] = docket_entry["pacer_doc_id"]
+ rd = await RECAPDocument.objects.aget(**get_params)
rds_updated.append(rd)
except RECAPDocument.DoesNotExist:
try:
@@ -867,9 +954,24 @@ async def add_docket_entries(
await duplicate_rd_queryset.exclude(pk=rd.pk).adelete()
rd.pacer_doc_id = rd.pacer_doc_id or docket_entry["pacer_doc_id"]
- rd.description = (
- docket_entry.get("short_description") or rd.description
- )
+ description = docket_entry.get("short_description")
+ if rd.document_type == RECAPDocument.PACER_DOCUMENT and description:
+ rd.description = description
+ elif description:
+ rd_qs = de.recap_documents.filter(
+ document_type=RECAPDocument.PACER_DOCUMENT
+ )
+ if await rd_qs.aexists():
+ rd_pd = await rd_qs.afirst()
+ if rd_pd.attachment_number is not None:
+ continue
+ if rd_pd.description != description:
+ rd_pd.description = description
+ try:
+ await rd_pd.asave()
+ except ValidationError:
+ # Happens from race conditions.
+ continue
rd.document_number = docket_entry["document_number"] or ""
try:
await rd.asave()
@@ -878,7 +980,7 @@ async def add_docket_entries(
continue
if tags:
for tag in tags:
- tag.tag_object(rd)
+ await sync_to_async(tag.tag_object)(rd)
attachments = docket_entry.get("attachments")
if attachments is not None:
@@ -1347,7 +1449,7 @@ def add_claims_to_docket(d, new_claims, tag_names=None):
)
db_claim.remarks = new_claim.get("remarks") or db_claim.remarks
db_claim.save()
- add_tags_to_objs(tag_names, [db_claim])
+ async_to_sync(add_tags_to_objs)(tag_names, [db_claim])
for new_history in new_claim["history"]:
add_claim_history_entry(new_history, db_claim)
@@ -1374,7 +1476,7 @@ def get_data_from_appellate_att_report(
return att_data
-async def add_tags_to_objs(tag_names: List[str], objs: Any) -> QuerySet:
+async def add_tags_to_objs(tag_names: List[str], objs: Any) -> list[Tag]:
"""Add tags by name to objects
:param tag_names: A list of tag name strings
@@ -1386,14 +1488,14 @@ async def add_tags_to_objs(tag_names: List[str], objs: Any) -> QuerySet:
if tag_names is None:
return []
- tags = []
+ tags: list[Tag] = []
for tag_name in tag_names:
tag, _ = await Tag.objects.aget_or_create(name=tag_name)
tags.append(tag)
for tag in tags:
for obj in objs:
- tag.tag_object(obj)
+ await sync_to_async(tag.tag_object)(obj)
return tags
@@ -1528,6 +1630,9 @@ async def merge_attachment_page_data(
and the DocketEntry object associated with the RECAPDocuments
:raises: RECAPDocument.MultipleObjectsReturned, RECAPDocument.DoesNotExist
"""
+ # Create/update the attachment items.
+ rds_created = []
+ rds_affected = []
params = {
"pacer_doc_id": pacer_doc_id,
"docket_entry__docket__court": court,
@@ -1575,12 +1680,80 @@ async def merge_attachment_page_data(
# with the wrong case. We must punt.
raise exc
except RECAPDocument.DoesNotExist as exc:
+ found_main_rd = False
+ migrated_description = ""
+ if not is_acms_attachment:
+ for attachment in attachment_dicts:
+ if attachment.get("pacer_doc_id", False):
+ params["pacer_doc_id"] = attachment["pacer_doc_id"]
+ try:
+ main_rd = await RECAPDocument.objects.select_related(
+ "docket_entry", "docket_entry__docket"
+ ).aget(**params)
+ if attachment.get("attachment_number", 0) != 0:
+ main_rd.attachment_number = attachment[
+ "attachment_number"
+ ]
+ main_rd.document_type = RECAPDocument.ATTACHMENT
+ migrated_description = main_rd.description
+ await main_rd.asave()
+ found_main_rd = True
+ break
+ except RECAPDocument.MultipleObjectsReturned as exc:
+ if pacer_case_id:
+ duplicate_rd_queryset = RECAPDocument.objects.filter(
+ **params
+ )
+ rd_with_pdf_queryset = duplicate_rd_queryset.filter(
+ is_available=True
+ ).exclude(filepath_local="")
+ if await rd_with_pdf_queryset.aexists():
+ keep_rd = await rd_with_pdf_queryset.alatest(
+ "date_created"
+ )
+ else:
+ keep_rd = await duplicate_rd_queryset.alatest(
+ "date_created"
+ )
+ await duplicate_rd_queryset.exclude(
+ pk=keep_rd.pk
+ ).adelete()
+ main_rd = await RECAPDocument.objects.select_related(
+ "docket_entry", "docket_entry__docket"
+ ).aget(**params)
+ if attachment.get("attachment_number", 0) != 0:
+ main_rd.attachment_number = attachment[
+ "attachment_number"
+ ]
+ main_rd.document_type = RECAPDocument.ATTACHMENT
+ migrated_description = main_rd.description
+ await main_rd.asave()
+ found_main_rd = True
+ break
+ else:
+ # Unclear how to proceed and we don't want to associate
+ # this data with the wrong case. We must punt.
+ raise exc
+ except RECAPDocument.DoesNotExist:
+ continue
# Can't find the docket to associate with the attachment metadata
# It may be possible to go look for orphaned documents at this stage
# and to then add them here, as we do when adding dockets. This need is
# particularly acute for those that get free look emails and then go to
# the attachment page.
- raise exc
+ if not found_main_rd:
+ raise exc
+ else:
+ rd = RECAPDocument(
+ docket_entry=main_rd.docket_entry,
+ document_type=RECAPDocument.PACER_DOCUMENT,
+ document_number=main_rd.document_number,
+ description=migrated_description,
+ pacer_doc_id=pacer_doc_id,
+ )
+ rds_created.append(rd)
+ rds_affected.append(rd)
+ await rd.asave()
# We got the right item. Update/create all the attachments for
# the docket entry.
@@ -1604,9 +1777,6 @@ async def merge_attachment_page_data(
ContentFile(text.encode()),
)
- # Create/update the attachment items.
- rds_created = []
- rds_affected = []
appellate_court_ids = Court.federal_courts.appellate_pacer_courts()
court_is_appellate = await appellate_court_ids.filter(
pk=court.pk
@@ -1614,11 +1784,9 @@ async def merge_attachment_page_data(
main_rd_to_att = False
for attachment in attachment_dicts:
sanity_checks = [
- attachment["attachment_number"],
+ attachment.get("attachment_number") is not None,
# Missing on sealed items.
attachment.get("pacer_doc_id", False),
- # Missing on some restricted docs (see Juriscraper)
- attachment["page_count"] is not None,
attachment["description"],
]
if not all(sanity_checks):
@@ -1643,25 +1811,76 @@ async def merge_attachment_page_data(
params = {
"docket_entry": de,
"document_number": document_number,
- "attachment_number": attachment["attachment_number"],
- "document_type": RECAPDocument.ATTACHMENT,
}
+ if attachment["attachment_number"] == 0:
+ params["document_type"] = RECAPDocument.PACER_DOCUMENT
+ else:
+ params["attachment_number"] = attachment["attachment_number"]
+ params["document_type"] = RECAPDocument.ATTACHMENT
if "acms_document_guid" in attachment:
params["acms_document_guid"] = attachment["acms_document_guid"]
try:
rd = await RECAPDocument.objects.aget(**params)
except RECAPDocument.DoesNotExist:
- rd = RECAPDocument(**params)
- rds_created.append(rd)
+ try:
+ doc_id_params = deepcopy(params)
+ doc_id_params.pop("attachment_number", None)
+ del doc_id_params["document_type"]
+ doc_id_params["pacer_doc_id"] = attachment["pacer_doc_id"]
+ rd = await RECAPDocument.objects.aget(**doc_id_params)
+ if attachment["attachment_number"] == 0:
+ try:
+ old_main_rd = await RECAPDocument.objects.aget(
+ docket_entry=de,
+ document_type=RECAPDocument.PACER_DOCUMENT,
+ )
+ rd.description = old_main_rd.description
+ except RECAPDocument.DoesNotExist:
+ rd.description = ""
+ except RECAPDocument.MultipleObjectsReturned:
+ rd.description = ""
+ logger.info(
+ f"Failed to migrate description for "
+ f"{attachment["pacer_doc_id"]}, "
+ f"multiple source documents found."
+ )
+ rd.attachment_number = None
+ rd.document_type = RECAPDocument.PACER_DOCUMENT
+ else:
+ rd.attachment_number = attachment["attachment_number"]
+ rd.document_type = RECAPDocument.ATTACHMENT
+ except RECAPDocument.DoesNotExist:
+ rd = RECAPDocument(**params)
+ if attachment["attachment_number"] == 0:
+ try:
+ old_main_rd = await RECAPDocument.objects.aget(
+ docket_entry=de,
+ document_type=RECAPDocument.PACER_DOCUMENT,
+ )
+ rd.description = old_main_rd.description
+ except RECAPDocument.DoesNotExist:
+ rd.description = ""
+ except RECAPDocument.MultipleObjectsReturned:
+ rd.description = ""
+ logger.info(
+ f"Failed to migrate description for "
+ f"{attachment["pacer_doc_id"]}, "
+ f"multiple source documents found."
+ )
+ rds_created.append(rd)
rds_affected.append(rd)
- for field in ["description", "pacer_doc_id"]:
- if attachment[field]:
- setattr(rd, field, attachment[field])
+ if (
+ attachment["description"]
+ and rd.document_type == RECAPDocument.ATTACHMENT
+ ):
+ rd.description = attachment["description"]
+ if attachment["pacer_doc_id"]:
+ rd.pacer_doc_id = attachment["pacer_doc_id"]
# Only set page_count and file_size if they're blank, in case
# we got the real value by measuring.
- if rd.page_count is None:
+ if rd.page_count is None and attachment.get("page_count", None):
rd.page_count = attachment["page_count"]
# If we have file_size_bytes it should have max precision.
file_size_bytes = attachment.get("file_size_bytes")
@@ -1691,6 +1910,7 @@ async def merge_attachment_page_data(
def save_iquery_to_docket(
self,
iquery_data: Dict[str, str],
+ iquery_text: str,
d: Docket,
tag_names: Optional[List[str]],
add_to_solr: bool = False,
@@ -1700,6 +1920,7 @@ def save_iquery_to_docket(
:param self: The celery task calling this function
:param iquery_data: The data from a successful iquery response
+ :param iquery_text: The HTML text data from a successful iquery response
:param d: A docket object to work with
:param tag_names: Tags to add to the items
:param add_to_solr: Whether to save the completed docket to solr
@@ -1725,6 +1946,16 @@ def save_iquery_to_docket(
if add_to_solr:
add_items_to_solr([d.pk], "search.Docket")
logger.info(f"Created/updated docket: {d}")
+
+ # Add the CASE_QUERY_PAGE to the docket in case we need it someday.
+ pacer_file = PacerHtmlFiles.objects.create(
+ content_object=d, upload_type=UPLOAD_TYPE.CASE_QUERY_PAGE
+ )
+ pacer_file.filepath.save(
+ "case_report.html", # We only care about the ext w/S3PrivateUUIDStorageTest
+ ContentFile(iquery_text.encode()),
+ )
+
return d.pk
@@ -1772,6 +2003,7 @@ def process_case_query_report(
court_id: str,
pacer_case_id: int,
report_data: dict[str, Any],
+ report_text: str,
avoid_trigger_signal: bool = False,
) -> None:
"""Process the case query report from probe_iquery_pages task.
@@ -1781,6 +2013,7 @@ def process_case_query_report(
:param court_id: A CL court ID where we'll look things up.
:param pacer_case_id: The internal PACER case ID number
:param report_data: A dictionary containing report data.
+ :param report_text: The HTML text data from a successful iquery response
:param avoid_trigger_signal: Whether to avoid triggering the iquery sweep
signal. Useful for ignoring reports added by the probe daemon or the iquery
sweep itself.
@@ -1790,6 +2023,9 @@ def process_case_query_report(
court_id,
str(pacer_case_id),
report_data["docket_number"],
+ report_data.get("federal_defendant_number"),
+ report_data.get("federal_dn_judge_initials_assigned"),
+ report_data.get("federal_dn_judge_initials_referred"),
using="default",
)
d.pacer_case_id = pacer_case_id
@@ -1802,4 +2038,14 @@ def process_case_query_report(
logger.info(
f"Created/updated docket: {d} from court: {court_id} and pacer_case_id {pacer_case_id}"
)
+
+ # Add the CASE_QUERY_PAGE to the docket in case we need it someday.
+ pacer_file = PacerHtmlFiles.objects.create(
+ content_object=d, upload_type=UPLOAD_TYPE.CASE_QUERY_PAGE
+ )
+ pacer_file.filepath.save(
+ "case_report.html",
+ # We only care about the ext w/S3PrivateUUIDStorageTest
+ ContentFile(report_text.encode()),
+ )
return None
diff --git a/cl/recap/migrations/0002_initial_part_two.py b/cl/recap/migrations/0002_initial_part_two.py
index f159a0cf5d..4f86187afc 100644
--- a/cl/recap/migrations/0002_initial_part_two.py
+++ b/cl/recap/migrations/0002_initial_part_two.py
@@ -92,8 +92,8 @@ class Migration(migrations.Migration):
name='uploader',
field=models.ForeignKey(help_text='The user that sent in the email for processing.', on_delete=django.db.models.deletion.CASCADE, related_name='recap_email_processing_queue', to=settings.AUTH_USER_MODEL),
),
- migrations.AlterIndexTogether(
- name='fjcintegrateddatabase',
- index_together={('district', 'docket_number')},
+ migrations.AddIndex(
+ model_name='fjcintegrateddatabase',
+ index=models.Index(fields=['district', 'docket_number'], name='recap_fjcintegrateddatabase_district_id_455568623a9da568_idx'),
),
]
diff --git a/cl/recap/migrations/0012_rename_fjcintegrateddatabase_district_docket_number_recap_fjcin_distric_731c7b_idx.py b/cl/recap/migrations/0012_rename_fjcintegrateddatabase_district_docket_number_recap_fjcin_distric_731c7b_idx.py
index c03497714b..a0b1f04a0e 100644
--- a/cl/recap/migrations/0012_rename_fjcintegrateddatabase_district_docket_number_recap_fjcin_distric_731c7b_idx.py
+++ b/cl/recap/migrations/0012_rename_fjcintegrateddatabase_district_docket_number_recap_fjcin_distric_731c7b_idx.py
@@ -12,7 +12,7 @@ class Migration(migrations.Migration):
operations = [
migrations.RenameIndex(
model_name="fjcintegrateddatabase",
- new_name="recap_fjcin_distric_731c7b_idx",
+ new_name="recap_fjcintegrateddatabase_district_id_455568623a9da568_idx",
old_fields=("district", "docket_number"),
),
]
diff --git a/cl/recap/migrations/0012_rename_fjcintegrateddatabase_district_docket_number_recap_fjcin_distric_731c7b_idx.sql b/cl/recap/migrations/0012_rename_fjcintegrateddatabase_district_docket_number_recap_fjcin_distric_731c7b_idx.sql
index c26ea5f1e1..7bc0b1abc1 100644
--- a/cl/recap/migrations/0012_rename_fjcintegrateddatabase_district_docket_number_recap_fjcin_distric_731c7b_idx.sql
+++ b/cl/recap/migrations/0012_rename_fjcintegrateddatabase_district_docket_number_recap_fjcin_distric_731c7b_idx.sql
@@ -1,6 +1,6 @@
BEGIN;
--
--- Rename unnamed index for ('district', 'docket_number') on fjcintegrateddatabase to recap_fjcin_distric_731c7b_idx
+-- Rename unnamed index for ('district', 'docket_number') on fjcintegrateddatabase to recap_fjcintegrateddatabase_district_id_455568623a9da568_idx
--
-ALTER INDEX "recap_fjcintegrateddatabase_district_id_455568623a9da568_idx" RENAME TO "recap_fjcin_distric_731c7b_idx";
+-- (no-op)
COMMIT;
diff --git a/cl/recap/migrations/0015_alter_pacerhtmlfiles_upload_type_noop.py b/cl/recap/migrations/0015_alter_pacerhtmlfiles_upload_type_noop.py
new file mode 100644
index 0000000000..f2c53368e6
--- /dev/null
+++ b/cl/recap/migrations/0015_alter_pacerhtmlfiles_upload_type_noop.py
@@ -0,0 +1,66 @@
+# Generated by Django 5.0.6 on 2024-07-10 21:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("recap", "0014_add_acms_upload_type_noop"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="pacerhtmlfiles",
+ name="upload_type",
+ field=models.SmallIntegerField(
+ choices=[
+ (1, "HTML Docket"),
+ (2, "HTML attachment page"),
+ (3, "PDF"),
+ (4, "Docket history report"),
+ (5, "Appellate HTML docket"),
+ (6, "Appellate HTML attachment page"),
+ (7, "Internet Archive XML docket"),
+ (8, "Case report (iquery.pl) page"),
+ (9, "Claims register page"),
+ (10, "Zip archive of RECAP Documents"),
+ (11, "Email in the SES storage format"),
+ (12, "Case query page"),
+ (13, "Appellate Case query page"),
+ (14, "Case query result page"),
+ (15, "Appellate Case query result page"),
+ (16, "ACMS docket JSON object"),
+ (17, "ACMS attachmente page JSON object"),
+ (18, "Free opinions report"),
+ ],
+ help_text="The type of object that is uploaded",
+ ),
+ ),
+ migrations.AlterField(
+ model_name="processingqueue",
+ name="upload_type",
+ field=models.SmallIntegerField(
+ choices=[
+ (1, "HTML Docket"),
+ (2, "HTML attachment page"),
+ (3, "PDF"),
+ (4, "Docket history report"),
+ (5, "Appellate HTML docket"),
+ (6, "Appellate HTML attachment page"),
+ (7, "Internet Archive XML docket"),
+ (8, "Case report (iquery.pl) page"),
+ (9, "Claims register page"),
+ (10, "Zip archive of RECAP Documents"),
+ (11, "Email in the SES storage format"),
+ (12, "Case query page"),
+ (13, "Appellate Case query page"),
+ (14, "Case query result page"),
+ (15, "Appellate Case query result page"),
+ (16, "ACMS docket JSON object"),
+ (17, "ACMS attachmente page JSON object"),
+ (18, "Free opinions report"),
+ ],
+ help_text="The type of object that is uploaded",
+ ),
+ ),
+ ]
diff --git a/cl/recap/migrations/0015_alter_pacerhtmlfiles_upload_type_noop.sql b/cl/recap/migrations/0015_alter_pacerhtmlfiles_upload_type_noop.sql
new file mode 100644
index 0000000000..b0d9d1e378
--- /dev/null
+++ b/cl/recap/migrations/0015_alter_pacerhtmlfiles_upload_type_noop.sql
@@ -0,0 +1,10 @@
+BEGIN;
+--
+-- Alter field upload_type on pacerhtmlfiles
+--
+-- (no-op)
+--
+-- Alter field upload_type on processingqueue
+--
+-- (no-op)
+COMMIT;
diff --git a/cl/recap/models.py b/cl/recap/models.py
index cca2c8ac01..988200d136 100644
--- a/cl/recap/models.py
+++ b/cl/recap/models.py
@@ -30,6 +30,7 @@ class UPLOAD_TYPE:
APPELLATE_CASE_QUERY_RESULT_PAGE = 15
ACMS_DOCKET_JSON = 16
ACMS_ATTACHMENT_PAGE = 17
+ FREE_OPINIONS_REPORT = 18
NAMES = (
(DOCKET, "HTML Docket"),
(ATTACHMENT_PAGE, "HTML attachment page"),
@@ -48,6 +49,7 @@ class UPLOAD_TYPE:
(APPELLATE_CASE_QUERY_RESULT_PAGE, "Appellate Case query result page"),
(ACMS_DOCKET_JSON, "ACMS docket JSON object"),
(ACMS_ATTACHMENT_PAGE, "ACMS attachmente page JSON object"),
+ (FREE_OPINIONS_REPORT, "Free opinions report"),
)
@@ -904,4 +906,9 @@ def __str__(self) -> str:
class Meta:
verbose_name_plural = "FJC Integrated Database Entries"
- indexes = [models.Index(fields=["district", "docket_number"])]
+ indexes = [
+ models.Index(
+ fields=["district", "docket_number"],
+ name="recap_fjcintegrateddatabase_district_id_455568623a9da568_idx",
+ )
+ ]
diff --git a/cl/recap/tasks.py b/cl/recap/tasks.py
index 2b218f3248..ea093f940f 100644
--- a/cl/recap/tasks.py
+++ b/cl/recap/tasks.py
@@ -38,7 +38,6 @@
from juriscraper.pacer.email import DocketType
from redis import ConnectionError as RedisConnectionError
from requests import HTTPError
-from requests.cookies import RequestsCookieJar
from requests.packages.urllib3.exceptions import ReadTimeoutError
from cl.alerts.tasks import enqueue_docket_alert, send_alert_and_webhook
@@ -61,6 +60,7 @@
from cl.lib.pacer import is_pacer_court_accessible, map_cl_to_pacer_id
from cl.lib.pacer_session import (
ProxyPacerSession,
+ SessionData,
delete_pacer_cookie_from_cache,
get_or_cache_pacer_cookies,
get_pacer_cookie_from_cache,
@@ -572,7 +572,12 @@ async def process_recap_docket(pk):
# Merge the contents of the docket into CL.
d = await find_docket_object(
- pq.court_id, pq.pacer_case_id, data["docket_number"]
+ pq.court_id,
+ pq.pacer_case_id,
+ data["docket_number"],
+ data.get("federal_defendant_number"),
+ data.get("federal_dn_judge_initials_assigned"),
+ data.get("federal_dn_judge_initials_referred"),
)
d.add_recap_source()
@@ -749,7 +754,12 @@ async def process_recap_claims_register(pk):
# Merge the contents of the docket into CL.
d = await find_docket_object(
- pq.court_id, pq.pacer_case_id, data["docket_number"]
+ pq.court_id,
+ pq.pacer_case_id,
+ data["docket_number"],
+ data.get("federal_defendant_number"),
+ data.get("federal_dn_judge_initials_assigned"),
+ data.get("federal_dn_judge_initials_referred"),
)
# Merge the contents into CL
@@ -839,7 +849,12 @@ async def process_recap_docket_history_report(pk):
# Merge the contents of the docket into CL.
d = await find_docket_object(
- pq.court_id, pq.pacer_case_id, data["docket_number"]
+ pq.court_id,
+ pq.pacer_case_id,
+ data["docket_number"],
+ data.get("federal_defendant_number"),
+ data.get("federal_dn_judge_initials_assigned"),
+ data.get("federal_dn_judge_initials_referred"),
)
d.add_recap_source()
@@ -943,7 +958,12 @@ async def process_case_query_page(pk):
# Merge the contents of the docket into CL.
d = await find_docket_object(
- pq.court_id, pq.pacer_case_id, data["docket_number"]
+ pq.court_id,
+ pq.pacer_case_id,
+ data["docket_number"],
+ data.get("federal_defendant_number"),
+ data.get("federal_dn_judge_initials_assigned"),
+ data.get("federal_dn_judge_initials_referred"),
)
current_case_name = d.case_name
d.add_recap_source()
@@ -1070,7 +1090,12 @@ async def process_recap_appellate_docket(pk):
# Merge the contents of the docket into CL.
d = await find_docket_object(
- pq.court_id, pq.pacer_case_id, data["docket_number"]
+ pq.court_id,
+ pq.pacer_case_id,
+ data["docket_number"],
+ data.get("federal_defendant_number"),
+ data.get("federal_dn_judge_initials_assigned"),
+ data.get("federal_dn_judge_initials_referred"),
)
d.add_recap_source()
@@ -1169,7 +1194,12 @@ async def process_recap_acms_docket(pk):
# Merge the contents of the docket into CL.
d = await find_docket_object(
- pq.court_id, pq.pacer_case_id, data["docket_number"]
+ pq.court_id,
+ pq.pacer_case_id,
+ data["docket_number"],
+ data.get("federal_defendant_number"),
+ data.get("federal_dn_judge_initials_assigned"),
+ data.get("federal_dn_judge_initials_referred"),
)
d.add_recap_source()
@@ -1640,21 +1670,23 @@ def fetch_pacer_doc_by_rd(
self.request.chain = None
return
- cookies = get_pacer_cookie_from_cache(fq.user_id)
- if not cookies:
+ session_data = get_pacer_cookie_from_cache(fq.user_id)
+ if not session_data:
msg = "Unable to find cached cookies. Aborting request."
mark_fq_status(fq, msg, PROCESSING_STATUS.FAILED)
self.request.chain = None
return
pacer_case_id = rd.docket_entry.docket.pacer_case_id
+ de_seq_num = rd.docket_entry.pacer_sequence_number
try:
r, r_msg = download_pacer_pdf_by_rd(
rd.pk,
pacer_case_id,
rd.pacer_doc_id,
- cookies,
+ session_data,
magic_number,
+ de_seq_num=de_seq_num,
)
except (requests.RequestException, HTTPError):
msg = "Failed to get PDF from network."
@@ -1739,14 +1771,14 @@ def fetch_attachment_page(self: Task, fq_pk: int) -> None:
mark_fq_status(fq, msg, PROCESSING_STATUS.NEEDS_INFO)
return
- cookies = get_pacer_cookie_from_cache(fq.user_id)
- if not cookies:
+ session_data = get_pacer_cookie_from_cache(fq.user_id)
+ if not session_data:
msg = "Unable to find cached cookies. Aborting request."
mark_fq_status(fq, msg, PROCESSING_STATUS.FAILED)
return
try:
- r = get_att_report_by_rd(rd, cookies)
+ r = get_att_report_by_rd(rd, session_data)
except HTTPError as exc:
msg = "Failed to get attachment page from network."
if exc.response.status_code in [
@@ -1875,7 +1907,12 @@ def fetch_docket_by_pacer_case_id(session, court_id, pacer_case_id, fq):
d = Docket.objects.get(pk=fq.docket_id)
else:
d = async_to_sync(find_docket_object)(
- court_id, pacer_case_id, docket_data["docket_number"]
+ court_id,
+ pacer_case_id,
+ docket_data["docket_number"],
+ docket_data.get("federal_defendant_number"),
+ docket_data.get("federal_dn_judge_initials_assigned"),
+ docket_data.get("federal_dn_judge_initials_referred"),
)
rds_created, content_updated = merge_pacer_docket_into_cl_docket(
d, pacer_case_id, docket_data, report, appellate=False
@@ -1918,14 +1955,16 @@ def fetch_docket(self, fq_pk):
async_to_sync(mark_pq_status)(fq, "", PROCESSING_STATUS.IN_PROGRESS)
- cookies = get_pacer_cookie_from_cache(fq.user_id)
- if cookies is None:
+ session_data = get_pacer_cookie_from_cache(fq.user_id)
+ if session_data is None:
msg = f"Cookie cache expired before task could run for user: {fq.user_id}"
mark_fq_status(fq, msg, PROCESSING_STATUS.FAILED)
self.request.chain = None
return None
- s = ProxyPacerSession(cookies=cookies)
+ s = ProxyPacerSession(
+ cookies=session_data.cookies, proxy=session_data.proxy_address
+ )
try:
result = fetch_pacer_case_id_and_title(s, fq, court_id)
except (requests.RequestException, ReadTimeoutError) as exc:
@@ -2164,7 +2203,7 @@ def save_pacer_doc_from_pq(
def download_pacer_pdf_and_save_to_pq(
court_id: str,
- cookies: RequestsCookieJar,
+ session_data: SessionData,
cutoff_date: datetime,
magic_number: str | None,
pacer_case_id: str,
@@ -2172,6 +2211,7 @@ def download_pacer_pdf_and_save_to_pq(
user_pk: int,
appellate: bool,
attachment_number: int = None,
+ de_seq_num: str | None = None,
) -> ProcessingQueue:
"""Try to download a PACER document from the notification via the magic
link and store it in a ProcessingQueue object. So it can be copied to every
@@ -2180,7 +2220,8 @@ def download_pacer_pdf_and_save_to_pq(
PQ object. Increasing the reliability of saving PACER documents.
:param court_id: A CourtListener court ID to query the free document.
- :param cookies: The cookies of a logged in PACER session
+ :param session_data: A SessionData object containing the session's cookies
+ and proxy.
:param cutoff_date: The datetime from which we should query
ProcessingQueue objects. For the main RECAPDocument the datetime the
EmailProcessingQueue was created. For attachments the datetime the
@@ -2193,6 +2234,8 @@ def download_pacer_pdf_and_save_to_pq(
:param appellate: Whether the download belongs to an appellate court.
:param attachment_number: The RECAPDocument attachment_number in case the
request belongs to an attachment document.
+ :param de_seq_num: The sequential number assigned by the PACER system to
+ identify the docket entry within a case.
:return: The ProcessingQueue object that's created or returned if existed.
"""
@@ -2217,9 +2260,10 @@ def download_pacer_pdf_and_save_to_pq(
court_id,
pacer_doc_id,
pacer_case_id,
- cookies,
+ session_data,
magic_number,
appellate,
+ de_seq_num,
)
if response:
file_name = get_document_filename(
@@ -2250,6 +2294,7 @@ def get_and_copy_recap_attachment_docs(
magic_number: str | None,
pacer_case_id: str,
user_pk: int,
+ de_seq_num: str | None = None,
) -> None:
"""Download and copy the corresponding PACER PDF to all the notification
RECAPDocument attachments, including support for multi-docket NEFs.
@@ -2260,17 +2305,19 @@ def get_and_copy_recap_attachment_docs(
:param magic_number: The magic number to fetch PACER documents for free.
:param pacer_case_id: The pacer_case_id to query the free document.
:param user_pk: The user to associate with the ProcessingQueue object.
+ :param de_seq_num: The sequential number assigned by the PACER system to
+ identify the docket entry within a case.
:return: None
"""
- cookies = get_pacer_cookie_from_cache(user_pk)
+ session_data = get_pacer_cookie_from_cache(user_pk)
appellate = False
unique_pqs = []
for rd_att in att_rds:
cutoff_date = rd_att.date_created
pq = download_pacer_pdf_and_save_to_pq(
court_id,
- cookies,
+ session_data,
cutoff_date,
magic_number,
pacer_case_id,
@@ -2278,6 +2325,7 @@ def get_and_copy_recap_attachment_docs(
user_pk,
appellate,
rd_att.attachment_number,
+ de_seq_num=de_seq_num,
)
fq = PacerFetchQueue.objects.create(
user_id=user_pk,
@@ -2374,7 +2422,7 @@ def get_and_merge_rd_attachments(
"""
all_attachment_rds = []
- cookies = get_pacer_cookie_from_cache(user_pk)
+ session_data = get_pacer_cookie_from_cache(user_pk)
# Try to get the attachment page without being logged into PACER
att_report_text = get_attachment_page_by_url(document_url, court_id)
if att_report_text:
@@ -2386,7 +2434,7 @@ def get_and_merge_rd_attachments(
.recap_documents.earliest("date_created")
)
# Get the attachment page being logged into PACER
- att_report = get_att_report_by_rd(main_rd, cookies)
+ att_report = get_att_report_by_rd(main_rd, session_data)
for docket_entry in dockets_updated:
# Merge the attachments for each docket/recap document
@@ -2461,6 +2509,7 @@ def process_recap_email(
pacer_doc_id = docket_entry["pacer_doc_id"]
pacer_case_id = docket_entry["pacer_case_id"]
document_url = docket_entry["document_url"]
+ pacer_seq_no = docket_entry["pacer_seq_no"]
break
# Some notifications don't contain a magic number at all, assign the
@@ -2469,10 +2518,11 @@ def process_recap_email(
pacer_doc_id = dockets[0]["docket_entries"][0]["pacer_doc_id"]
pacer_case_id = dockets[0]["docket_entries"][0]["pacer_case_id"]
document_url = dockets[0]["docket_entries"][0]["document_url"]
+ pacer_seq_no = dockets[0]["docket_entries"][0]["pacer_seq_no"]
start_time = now()
# Ensures we have PACER cookies ready to go.
- cookies = get_or_cache_pacer_cookies(
+ cookies_data = get_or_cache_pacer_cookies(
user_pk, settings.PACER_USERNAME, settings.PACER_PASSWORD
)
appellate = data["appellate"]
@@ -2480,13 +2530,14 @@ def process_recap_email(
# its future processing.
pq = download_pacer_pdf_and_save_to_pq(
epq.court_id,
- cookies,
+ cookies_data,
epq.date_created,
magic_number,
pacer_case_id,
pacer_doc_id,
user_pk,
appellate,
+ de_seq_num=pacer_seq_no,
)
is_potentially_sealed_entry = (
is_docket_entry_sealed(epq.court_id, pacer_case_id, pacer_doc_id)
@@ -2513,6 +2564,9 @@ def process_recap_email(
epq.court_id,
docket_entry["pacer_case_id"],
docket_data["docket_number"],
+ docket_data.get("federal_defendant_number"),
+ docket_data.get("federal_dn_judge_initials_assigned"),
+ docket_data.get("federal_dn_judge_initials_referred"),
)
docket.add_recap_source()
async_to_sync(update_docket_metadata)(docket, docket_data)
@@ -2583,6 +2637,7 @@ def process_recap_email(
magic_number,
pacer_case_id,
user_pk,
+ de_seq_num=pacer_seq_no,
)
# Send docket alerts and webhooks for each docket updated.
diff --git a/cl/recap/tests.py b/cl/recap/tests.py
index 82bbc62e16..16e27db6ff 100644
--- a/cl/recap/tests.py
+++ b/cl/recap/tests.py
@@ -16,7 +16,7 @@
from django.core import mail
from django.core.files.base import ContentFile
from django.core.files.uploadedfile import SimpleUploadedFile
-from django.test import RequestFactory
+from django.test import RequestFactory, override_settings
from django.urls import reverse
from django.utils.timezone import now
from juriscraper.pacer import PacerRssFeed
@@ -56,8 +56,10 @@
AppellateAttachmentFactory,
AppellateAttachmentPageFactory,
DocketDataFactory,
+ DocketDataWithAttachmentsFactory,
DocketEntriesDataFactory,
DocketEntryDataFactory,
+ DocketEntryWithAttachmentsDataFactory,
FjcIntegratedDatabaseFactory,
MinuteDocketEntryDataFactory,
PacerFetchQueueFactory,
@@ -79,6 +81,7 @@
add_parties_and_attorneys,
find_docket_object,
get_order_of_docket,
+ merge_attachment_page_data,
normalize_long_description,
update_case_names,
update_docket_metadata,
@@ -771,7 +774,6 @@ def test_processing_an_acms_attachment_page(self, mock_upload):
)
@mock.patch(
"cl.recap.tasks.get_pacer_cookie_from_cache",
- side_effect=lambda x: True,
)
class RecapDocketFetchApiTest(TestCase):
"""Tests for the RECAP docket Fetch API
@@ -1046,10 +1048,7 @@ def test_key_serialization_with_client_code(self, mock) -> None:
"cl.corpus_importer.tasks.FreeOpinionReport",
new=fakes.FakeFreeOpinionReport,
)
-@mock.patch(
- "cl.recap.tasks.get_pacer_cookie_from_cache",
- return_value={"cookie": "foo"},
-)
+@mock.patch("cl.recap.tasks.get_pacer_cookie_from_cache")
@mock.patch(
"cl.recap.tasks.is_pacer_court_accessible",
side_effect=lambda a: True,
@@ -1151,7 +1150,6 @@ def test_fetch_att_page_no_cookies(self, mock_court_accessible) -> None:
@mock.patch(
"cl.recap.tasks.get_pacer_cookie_from_cache",
- return_value={"pacer_cookie": "foo"},
)
@mock.patch(
"cl.corpus_importer.tasks.AttachmentPage",
@@ -1234,6 +1232,9 @@ def mock_bucket_open(message_id, r, read_file=False):
return recap_mail_example
+@override_settings(
+ EGRESS_PROXY_HOSTS=["http://proxy_1:9090", "http://proxy_2:9090"]
+)
class RecapEmailToEmailProcessingQueueTest(TestCase):
"""Test the rest endpoint, but exclude the processing tasks."""
@@ -1292,10 +1293,7 @@ async def test_missing_receipt_properties_fails(self):
"cl.recap.tasks.RecapEmailSESStorage.open",
side_effect=mock_bucket_open,
)
- @mock.patch(
- "cl.recap.tasks.get_or_cache_pacer_cookies",
- side_effect=lambda x, y, z: None,
- )
+ @mock.patch("cl.recap.tasks.get_or_cache_pacer_cookies")
@mock.patch(
"cl.recap.tasks.is_docket_entry_sealed",
return_value=False,
@@ -2036,7 +2034,12 @@ def test_rss_feed_ingestion(self) -> None:
rss_feed._parse_text(text)
docket = rss_feed.data[0]
d = async_to_sync(find_docket_object)(
- court_id, docket["pacer_case_id"], docket["docket_number"]
+ court_id,
+ docket["pacer_case_id"],
+ docket["docket_number"],
+ docket["federal_defendant_number"],
+ docket["federal_dn_judge_initials_assigned"],
+ docket["federal_dn_judge_initials_referred"],
)
async_to_sync(update_docket_metadata)(d, docket)
d.save()
@@ -2198,6 +2201,45 @@ def test_retain_existing_values_in_absent_rss_fields(
self.assertEqual(docket.assigned_to_str, "John Marshall")
self.assertEqual(docket.referred_to_str, "Sophia Clinton")
+ def test_avoid_deleting_short_description(self) -> None:
+ """Test that merging identical docket entries without a
+ short_description does not delete or overwrite the existing short_description
+ """
+ court_ca10 = CourtFactory(id="ca10", jurisdiction="F")
+ rss_feed = PacerRssFeed(court_ca10.pk)
+ with open(self.make_path("rss_ca10.xml"), "rb") as f:
+ text = f.read().decode()
+ rss_feed._parse_text(text)
+ docket = rss_feed.data[0]
+ d = async_to_sync(find_docket_object)(
+ court_ca10.pk,
+ docket["pacer_case_id"],
+ docket["docket_number"],
+ docket["federal_defendant_number"],
+ docket["federal_dn_judge_initials_assigned"],
+ docket["federal_dn_judge_initials_referred"],
+ )
+ async_to_sync(update_docket_metadata)(d, docket)
+ d.save()
+ async_to_sync(add_docket_entries)(d, docket["docket_entries"])
+ rd = RECAPDocument.objects.all().first()
+ self.assertEqual(rd.description, "Case termination for COA")
+
+ # Merge the identical entry without a short_description.
+ # It should not be removed.
+ docket_entries = [
+ MinuteDocketEntryDataFactory(
+ description="Lorem ipsum",
+ short_description=None,
+ pacer_doc_id="010010808570",
+ document_number="010010808570",
+ ),
+ ]
+ async_to_sync(add_docket_entries)(d, docket_entries)
+ rd = RECAPDocument.objects.all().first()
+ self.assertEqual(d.docket_entries.count(), 1)
+ self.assertEqual(rd.description, "Case termination for COA")
+
class DescriptionCleanupTest(SimpleTestCase):
def test_cleanup(self) -> None:
@@ -2561,12 +2603,46 @@ def test_avoid_overwriting_nature_of_suit_in_update_docket_metadata(
)
d.delete()
+ def test_merge_docket_number_components(
+ self,
+ ) -> None:
+ """Confirm docket_number components are properly merged into the
+ docket instance.
+ """
+
+ d = DocketFactory.create(
+ source=Docket.DEFAULT,
+ pacer_case_id="12345",
+ court_id=self.court.pk,
+ )
+
+ docket_data = DocketDataFactory(
+ court_id=d.court_id,
+ docket_number="3:20-cr-00070-TKW-MAL-1",
+ federal_dn_office_code="3",
+ federal_dn_case_type="cr",
+ federal_dn_judge_initials_assigned="TKW",
+ federal_dn_judge_initials_referred="MAL",
+ federal_defendant_number="1",
+ )
+ async_to_sync(update_docket_metadata)(d, docket_data)
+ d.save()
+ d.refresh_from_db()
+
+ self.assertEqual(d.federal_dn_office_code, "3")
+ self.assertEqual(d.federal_dn_case_type, "cr")
+ self.assertEqual(d.federal_dn_judge_initials_assigned, "TKW")
+ self.assertEqual(d.federal_dn_judge_initials_referred, "MAL")
+ self.assertEqual(d.federal_defendant_number, 1)
+
+ d.delete()
+
@mock.patch("cl.recap.tasks.add_items_to_solr")
class RecapDocketAttachmentTaskTest(TestCase):
@classmethod
def setUpTestData(cls):
- CourtFactory(id="cand", jurisdiction="FD")
+ cls.court = CourtFactory(id="cand", jurisdiction="FD")
def setUp(self) -> None:
self.user = User.objects.get(username="recap")
@@ -2588,9 +2664,7 @@ def tearDown(self) -> None:
self.pq.filepath_local.delete()
self.pq.delete()
Docket.objects.all().delete()
- RECAPDocument.objects.filter(
- document_type=RECAPDocument.ATTACHMENT,
- ).delete()
+ RECAPDocument.objects.all().delete()
def test_attachments_get_created(self, mock):
"""Do attachments get created if we have a RECAPDocument to match
@@ -2606,6 +2680,239 @@ def test_attachments_get_created(self, mock):
self.pq.refresh_from_db()
self.assertEqual(self.pq.status, PROCESSING_STATUS.SUCCESSFUL)
+ @mock.patch(
+ "cl.api.webhooks.requests.post",
+ side_effect=lambda *args, **kwargs: MockResponse(200, mock_raw=True),
+ )
+ def test_main_document_doesnt_match_attachment_zero_on_creation(
+ self,
+ mock_solr,
+ mock_webhook_post,
+ ):
+ """Confirm that attachment 0 is properly set as the Main document if
+ the docket entry's pacer_doc_id does not match the Main document's
+ pacer_doc_id on creation.
+ """
+ docket = DocketFactory(
+ source=Docket.RECAP,
+ court=self.court,
+ pacer_case_id="238743",
+ )
+ docket_data = DocketDataWithAttachmentsFactory(
+ docket_entries=[
+ DocketEntryWithAttachmentsDataFactory(
+ document_number=1,
+ pacer_doc_id="1234567",
+ short_description="Complaint",
+ attachments=[
+ AppellateAttachmentFactory(
+ attachment_number=0,
+ pacer_doc_id="1234566",
+ description="Main Document",
+ ),
+ AppellateAttachmentFactory(
+ attachment_number=1,
+ pacer_doc_id="1234567",
+ description="Attachment 1",
+ ),
+ ],
+ ),
+ ],
+ )
+ async_to_sync(add_docket_entries)(
+ docket, docket_data["docket_entries"]
+ )
+ main_rd = RECAPDocument.objects.get(pacer_doc_id="1234566")
+ attachment_1 = RECAPDocument.objects.get(pacer_doc_id="1234567")
+ self.assertEqual(
+ main_rd.document_type,
+ RECAPDocument.PACER_DOCUMENT,
+ msg="PACER_DOCUMENT type didn't match.",
+ )
+ self.assertEqual(main_rd.attachment_number, None)
+ self.assertEqual(main_rd.description, "Complaint")
+
+ self.assertEqual(
+ attachment_1.document_type,
+ RECAPDocument.ATTACHMENT,
+ msg="ATTACHMENT type didn't match.",
+ )
+ self.assertEqual(attachment_1.attachment_number, 1)
+ self.assertEqual(attachment_1.description, "Attachment 1")
+
+ @mock.patch(
+ "cl.api.webhooks.requests.post",
+ side_effect=lambda *args, **kwargs: MockResponse(200, mock_raw=True),
+ )
+ def test_main_document_doesnt_match_attachment_zero_existing(
+ self,
+ mock_solr,
+ mock_webhook_post,
+ ):
+ """Confirm that attachment 0 is properly set as the Main document if
+ the docket entry's pacer_doc_id does not match the Main document's
+ pacer_doc_id on an existing document.
+ """
+ docket = DocketFactory(
+ source=Docket.RECAP,
+ court=self.court,
+ pacer_case_id="238743",
+ )
+ docket_data_no_att = DocketDataWithAttachmentsFactory(
+ docket_entries=[
+ DocketEntryWithAttachmentsDataFactory(
+ document_number=1, pacer_doc_id="1234567", attachments=[]
+ ),
+ ],
+ )
+ async_to_sync(add_docket_entries)(
+ docket, docket_data_no_att["docket_entries"]
+ )
+
+ # When attachment data is unknown, the main PACER_DOCUMENT should be
+ # set to pacer_doc_id 1234567.
+ main_rd = RECAPDocument.objects.get(pacer_doc_id="1234567")
+ self.assertEqual(
+ main_rd.document_type,
+ RECAPDocument.PACER_DOCUMENT,
+ msg="PACER_DOCUMENT type didn't match.",
+ )
+ self.assertEqual(main_rd.attachment_number, None)
+
+ docket_data_att = DocketDataWithAttachmentsFactory(
+ docket_entries=[
+ DocketEntryWithAttachmentsDataFactory(
+ document_number=1,
+ pacer_doc_id="1234567",
+ short_description="Complaint",
+ attachments=[
+ AppellateAttachmentFactory(
+ attachment_number=0,
+ pacer_doc_id="1234566",
+ description="Main Document",
+ ),
+ AppellateAttachmentFactory(
+ attachment_number=1,
+ pacer_doc_id="1234567",
+ description="Attachment 1",
+ ),
+ ],
+ ),
+ ],
+ )
+ async_to_sync(add_docket_entries)(
+ docket, docket_data_att["docket_entries"]
+ )
+
+ # After merging attachments, the main PACER_DOCUMENT should now be set
+ # to attachment 0 with pacer_doc_id 1234566.
+ main_rd = RECAPDocument.objects.get(pacer_doc_id="1234566")
+ self.assertEqual(
+ main_rd.document_type,
+ RECAPDocument.PACER_DOCUMENT,
+ msg="PACER_DOCUMENT type didn't match.",
+ )
+ self.assertEqual(main_rd.attachment_number, None)
+ self.assertEqual(main_rd.description, "Complaint")
+
+ # pacer_doc_id 1234567 should now be an attachment.
+ attachment_1 = RECAPDocument.objects.get(pacer_doc_id="1234567")
+ self.assertEqual(
+ attachment_1.document_type,
+ RECAPDocument.ATTACHMENT,
+ msg="ATTACHMENT type didn't match.",
+ )
+ self.assertEqual(attachment_1.attachment_number, 1)
+ self.assertEqual(attachment_1.description, "Attachment 1")
+
+ @mock.patch(
+ "cl.api.webhooks.requests.post",
+ side_effect=lambda *args, **kwargs: MockResponse(200, mock_raw=True),
+ )
+ def test_main_rd_lookup_fallback_for_attachment_merging(
+ self,
+ mock_solr,
+ mock_webhook_post,
+ ):
+ """Confirm that attachment data can be properly merged when the current
+ main_rd pacer_doc_id mismatches the main document's pacer_doc_id from
+ the attachment page.
+ """
+ docket = DocketFactory(
+ source=Docket.RECAP,
+ court=self.court,
+ pacer_case_id="238743",
+ )
+ docket_data_no_att = DocketDataWithAttachmentsFactory(
+ docket_entries=[
+ DocketEntryWithAttachmentsDataFactory(
+ document_number=1,
+ pacer_doc_id="12606200429",
+ short_description="Complaint",
+ attachments=[],
+ ),
+ ],
+ )
+ async_to_sync(add_docket_entries)(
+ docket, docket_data_no_att["docket_entries"]
+ )
+
+ # When attachment data is unknown, the main PACER_DOCUMENT is the one
+ # with pacer_doc_id: 12606200429
+ main_rd = RECAPDocument.objects.get(pacer_doc_id="12606200429")
+ self.assertEqual(
+ main_rd.document_type,
+ RECAPDocument.PACER_DOCUMENT,
+ msg="PACER_DOCUMENT type didn't match.",
+ )
+ self.assertEqual(main_rd.attachment_number, None)
+
+ # Merge attachment data where the main_document pacer_doc_id has a
+ # different pacer_doc_id: 12606201629
+ attachments_data = AppellateAttachmentPageFactory(
+ document_number=1,
+ pacer_doc_id="12606201629",
+ attachments=[
+ AppellateAttachmentFactory(
+ attachment_number=1,
+ pacer_doc_id="12606200429",
+ description="Attachment 1",
+ ),
+ ],
+ )
+ async_to_sync(merge_attachment_page_data)(
+ docket.court,
+ docket.pacer_case_id,
+ attachments_data["pacer_doc_id"],
+ None,
+ "",
+ attachments_data["attachments"],
+ )
+
+ # Now we should have 2 RDs in the entry: the main document + 1 attachment
+ de_rds = RECAPDocument.objects.all()
+ self.assertEqual(de_rds.count(), 2)
+
+ # Confirm main_rd is now the one with pacer_doc_id:12606201629
+ main_rd = RECAPDocument.objects.get(pacer_doc_id="12606201629")
+ self.assertEqual(
+ main_rd.document_type,
+ RECAPDocument.PACER_DOCUMENT,
+ msg="PACER_DOCUMENT type didn't match.",
+ )
+ self.assertEqual(main_rd.attachment_number, None)
+ self.assertEqual(main_rd.description, "Complaint")
+
+ # Confirm attachment 1 is the one with pacer_doc_id:12606200429
+ attachment_1 = RECAPDocument.objects.get(pacer_doc_id="12606200429")
+ self.assertEqual(
+ attachment_1.document_type,
+ RECAPDocument.ATTACHMENT,
+ msg="ATTACHMENT type didn't match.",
+ )
+ self.assertEqual(attachment_1.attachment_number, 1)
+ self.assertEqual(attachment_1.description, "Attachment 1")
+
class ClaimsRegistryTaskTest(TestCase):
"""Can we handle claims registry uploads?"""
@@ -2946,7 +3253,7 @@ def test_create_from_idb_chunk(self) -> None:
)
@mock.patch(
"cl.recap.tasks.get_or_cache_pacer_cookies",
- side_effect=lambda x, y, z: None,
+ side_effect=lambda x, y, z: (None, None),
)
@mock.patch(
"cl.recap.tasks.is_pacer_court_accessible",
@@ -3125,7 +3432,7 @@ def setUp(self) -> None:
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -3200,7 +3507,7 @@ async def test_new_recap_email_case_auto_subscription(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -3293,7 +3600,7 @@ async def test_new_recap_email_case_auto_subscription_prev_user(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -3356,7 +3663,7 @@ async def test_new_recap_email_case_no_auto_subscription(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -3447,7 +3754,7 @@ async def test_new_recap_email_case_no_auto_subscription_prev_user(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(
200,
mock_bucket_open("nda_document.pdf", "rb", True),
@@ -3494,7 +3801,7 @@ async def test_no_recap_email_user_found(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -3650,7 +3957,7 @@ async def test_receive_same_recap_email_notification_different_users(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -3720,7 +4027,7 @@ async def test_new_recap_email_subscribe_by_email_link(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -3814,7 +4121,7 @@ async def test_new_recap_email_unsubscribe_by_email_link(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -3965,7 +4272,7 @@ async def test_new_recap_email_alerts_integration(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -4051,7 +4358,7 @@ async def test_docket_alert_toggle_confirmation_fails(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(200, b""),
"OK",
),
@@ -4208,7 +4515,7 @@ async def test_new_recap_email_with_attachments(
)
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(
200,
mock_bucket_open(
@@ -4254,7 +4561,7 @@ async def test_extract_pdf_for_recap_email(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(200, b""),
"OK",
),
@@ -4309,7 +4616,7 @@ async def test_new_nda_recap_email(
)
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(200, b""),
"OK",
),
@@ -4381,7 +4688,7 @@ async def test_new_nda_recap_email_case_auto_subscription(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(200, b""),
"OK",
),
@@ -4452,7 +4759,7 @@ async def test_new_nda_recap_email_case_no_auto_subscription(
)
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(200, b"Hello World"),
"OK",
),
@@ -4623,7 +4930,7 @@ async def test_multiple_docket_nef(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -4709,7 +5016,7 @@ async def test_recap_email_no_magic_number(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
None,
"Document not available from magic link.",
),
@@ -4757,7 +5064,7 @@ async def test_mark_as_sealed_nda_document_not_available_from_magic_link(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -5075,7 +5382,7 @@ async def test_recap_email_sealed_entry_with_attachments(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
@mock.patch(
"cl.api.webhooks.requests.post",
@@ -5240,7 +5547,7 @@ def test_copy_pdf_attachments_from_pqs(self):
)
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(200, b"Hello World from magic"),
"OK",
),
@@ -5404,7 +5711,7 @@ def test_clean_up_recap_document_file(self, mock_open):
)
@mock.patch(
"cl.recap.tasks.get_or_cache_pacer_cookies",
- side_effect=lambda x, y, z: "Cookie",
+ side_effect=lambda x, y, z: ("Cookie", settings.EGRESS_PROXY_HOSTS[0]),
)
@mock.patch(
"cl.recap.tasks.get_pacer_cookie_from_cache",
@@ -5473,7 +5780,7 @@ def setUp(self) -> None:
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(
200,
mock_bucket_open("nda_document.pdf", "rb", True),
@@ -5511,7 +5818,7 @@ async def test_nda_get_document_number_from_pdf(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(
200,
mock_bucket_open(
@@ -5558,7 +5865,7 @@ async def test_nda_get_document_number_from_confirmation_page(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(
200,
mock_bucket_open(
@@ -5604,7 +5911,7 @@ async def test_nda_get_document_number_fallback(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(200, b""),
"OK",
),
@@ -5643,7 +5950,7 @@ async def test_nda_not_document_number_available(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
MockResponse(200, b""),
"OK",
),
@@ -5693,7 +6000,7 @@ async def test_receive_same_recap_email_nda_notification_different_users(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (
+ side_effect=lambda z, x, c, v, b, d, e: (
None,
"Document not available from magic link.",
),
@@ -5778,7 +6085,7 @@ def test_is_pacer_court_accessible_fails(
)
@mock.patch(
"cl.recap.tasks.get_or_cache_pacer_cookies",
- side_effect=lambda x, y, z: None,
+ side_effect=lambda x, y, z: (None, None),
)
@mock.patch(
"cl.recap.tasks.is_pacer_court_accessible",
@@ -6089,7 +6396,7 @@ def test_webhook_response_status_codes(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
async def test_update_webhook_after_http_error(
self,
@@ -6161,7 +6468,7 @@ async def test_update_webhook_after_http_error(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
async def test_update_webhook_after_network_error(
self,
@@ -6234,7 +6541,7 @@ async def test_update_webhook_after_network_error(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
async def test_success_webhook_delivery(
self,
@@ -6300,7 +6607,7 @@ async def test_success_webhook_delivery(
@mock.patch(
"cl.recap.tasks.download_pdf_by_magic_number",
- side_effect=lambda z, x, c, v, b, d: (None, ""),
+ side_effect=lambda z, x, c, v, b, d, e: (None, ""),
)
async def test_retry_webhooks_integration(
self,
@@ -6965,7 +7272,6 @@ def test_send_notifications_if_webhook_still_disabled(
)
@mock.patch(
"cl.recap.tasks.get_pacer_cookie_from_cache",
- side_effect=lambda x: True,
)
class RecapFetchWebhooksTest(TestCase):
"""Test RECAP Fetch Webhooks"""
@@ -7113,7 +7419,7 @@ def test_recap_attachment_page_webhook(
@mock.patch(
"cl.recap.tasks.download_pacer_pdf_by_rd",
- side_effect=lambda z, x, c, v, b: (
+ side_effect=lambda z, x, c, v, b, de_seq_num: (
MockResponse(
200,
mock_bucket_open(
@@ -7386,7 +7692,12 @@ def test_case_id_and_docket_number_core_lookup(self):
"""
d = async_to_sync(find_docket_object)(
- self.court.pk, "12345", self.docket_data["docket_number"]
+ self.court.pk,
+ "12345",
+ self.docket_data["docket_number"],
+ None,
+ "",
+ "",
)
async_to_sync(update_docket_metadata)(d, self.docket_data)
d.save()
@@ -7405,7 +7716,12 @@ def test_case_id_and_docket_number_no_match(self):
dockets = Docket.objects.all()
self.assertEqual(dockets.count(), 4)
d = async_to_sync(find_docket_object)(
- self.court.pk, "12346", self.docket_data["docket_number"]
+ self.court.pk,
+ "12346",
+ self.docket_data["docket_number"],
+ 1,
+ "RM",
+ "LM",
)
async_to_sync(update_docket_metadata)(d, self.docket_data)
d.save()
@@ -7421,7 +7737,12 @@ def test_case_id_lookup(self):
"""Confirm if lookup by only pacer_case_id works properly."""
d = async_to_sync(find_docket_object)(
- self.court.pk, "54321", self.docket_data["docket_number"]
+ self.court.pk,
+ "54321",
+ self.docket_data["docket_number"],
+ 1,
+ "RM",
+ "LM",
)
async_to_sync(update_docket_metadata)(d, self.docket_data)
d.save()
@@ -7439,6 +7760,9 @@ def test_docket_number_core_lookup(self):
self.court.pk,
None,
self.docket_core_data["docket_number"],
+ 1,
+ "RM",
+ "LM",
)
async_to_sync(update_docket_metadata)(d, self.docket_core_data)
d.save()
@@ -7456,6 +7780,9 @@ def test_docket_number_lookup(self):
self.court.pk,
None,
self.docket_no_core_data["docket_number"],
+ None,
+ "",
+ "",
)
async_to_sync(update_docket_metadata)(d, self.docket_no_core_data)
d.save()
@@ -7475,6 +7802,9 @@ def test_avoid_overwrite_docket_by_number_core(self):
self.court.pk,
self.docket_data["docket_entries"][0]["pacer_case_id"],
self.docket_data["docket_number"],
+ 1,
+ "RM",
+ "LM",
)
async_to_sync(update_docket_metadata)(d, self.docket_data)
@@ -7503,6 +7833,9 @@ def test_avoid_overwrite_docket_by_number_core_multiple_results(self):
self.court.pk,
self.docket_data["docket_entries"][0]["pacer_case_id"],
self.docket_data["docket_number"],
+ 1,
+ "RM",
+ "LM",
)
async_to_sync(update_docket_metadata)(d, self.docket_data)
@@ -7537,12 +7870,227 @@ def test_lookup_by_normalized_docket_number_case(self):
self.court_appellate.pk,
None,
docket_data_lower_number["docket_number"],
+ 1,
+ "RM",
+ "LM",
)
async_to_sync(update_docket_metadata)(new_d, docket_data_lower_number)
new_d.save()
# The existing docket is matched instead of creating a new one.
self.assertEqual(new_d.pk, d.pk)
+ def test_lookup_by_docket_number_components(self):
+ """Can we match a docket using the components of the docket number?
+ If two or more dockets are found without a matching pacer_case_id, and
+ they are matched by docket_number or docket_number_core, use the
+ components of the docket number as a last resort to find the correct
+ match. If a single docket is matched, and it contains DN components use
+ them to confirm the docket matched is the right one.
+ """
+
+ # Single docket doesn't match.
+ d_1 = DocketFactory(
+ case_name="Young v. State",
+ docket_number="1:03-cr-00076",
+ court=self.court_appellate,
+ source=Docket.RECAP,
+ pacer_case_id=None,
+ federal_defendant_number=2,
+ federal_dn_judge_initials_assigned="MA",
+ federal_dn_judge_initials_referred="DH",
+ federal_dn_case_type="cv",
+ federal_dn_office_code="1",
+ )
+ docket_matched = async_to_sync(find_docket_object)(
+ self.court_appellate.pk,
+ None,
+ "1:03-cr-00076",
+ federal_defendant_number=2,
+ federal_dn_judge_initials_assigned="ML",
+ federal_dn_judge_initials_referred="DHL",
+ )
+ self.assertNotEqual(docket_matched.pk, d_1.pk)
+
+ # Single docket match.
+ d_1_2 = DocketFactory(
+ case_name="Young v. State",
+ docket_number="1:03-cr-00073",
+ court=self.court_appellate,
+ source=Docket.RECAP,
+ pacer_case_id=None,
+ federal_defendant_number=2,
+ federal_dn_judge_initials_assigned="MA",
+ federal_dn_judge_initials_referred="DH",
+ federal_dn_case_type="cr",
+ federal_dn_office_code="1",
+ )
+ docket_matched = async_to_sync(find_docket_object)(
+ self.court_appellate.pk,
+ None,
+ "1:03-cr-00073",
+ federal_defendant_number=2,
+ federal_dn_judge_initials_assigned="MA",
+ federal_dn_judge_initials_referred="DH",
+ )
+ self.assertEqual(docket_matched.pk, d_1_2.pk)
+
+ # Partial DN components single docket match.
+ d_1_2_3 = DocketFactory(
+ case_name="Young v. State",
+ docket_number="1:03-cr-00072",
+ court=self.court_appellate,
+ source=Docket.RECAP,
+ pacer_case_id=None,
+ federal_defendant_number=None,
+ federal_dn_judge_initials_assigned="MA",
+ federal_dn_judge_initials_referred="",
+ federal_dn_case_type="cr",
+ federal_dn_office_code="1",
+ )
+ docket_matched = async_to_sync(find_docket_object)(
+ self.court_appellate.pk,
+ None,
+ "1:03-cr-00072",
+ federal_defendant_number=2,
+ federal_dn_judge_initials_assigned="MA",
+ federal_dn_judge_initials_referred="DH",
+ )
+ self.assertEqual(docket_matched.pk, d_1_2_3.pk)
+ docket_data = self.docket_data.copy()
+ docket_data["federal_dn_judge_initials_assigned"] = "MA"
+ async_to_sync(update_docket_metadata)(docket_matched, docket_data)
+ docket_matched.save()
+ docket_matched.refresh_from_db()
+ self.assertEqual(
+ docket_matched.federal_dn_judge_initials_assigned, "MA"
+ )
+
+ # Two or more Dockets matched. Partial DN components matched.
+ d_2 = DocketFactory(
+ case_name="Young v. State",
+ docket_number="1:03-cr-00050",
+ court=self.court_appellate,
+ source=Docket.RECAP,
+ pacer_case_id=None,
+ federal_defendant_number=None,
+ federal_dn_judge_initials_assigned="MR",
+ federal_dn_judge_initials_referred="",
+ federal_dn_case_type="cr",
+ federal_dn_office_code="1",
+ )
+ docket_matched = async_to_sync(find_docket_object)(
+ self.court_appellate.pk,
+ None,
+ "1:03-cr-00050",
+ federal_defendant_number=None,
+ federal_dn_judge_initials_assigned="MR",
+ federal_dn_judge_initials_referred="",
+ )
+ self.assertEqual(docket_matched.pk, d_2.pk)
+
+ # Two or more Dockets matched. All DN components matched.
+ d_3 = DocketFactory(
+ case_name="Young v. State",
+ docket_number="1:03-cr-00076",
+ court=self.court_appellate,
+ source=Docket.RECAP,
+ pacer_case_id=None,
+ federal_defendant_number=1,
+ federal_dn_judge_initials_assigned="MR",
+ federal_dn_judge_initials_referred="DLH",
+ federal_dn_case_type="cr",
+ federal_dn_office_code="1",
+ )
+ docket_matched = async_to_sync(find_docket_object)(
+ self.court_appellate.pk,
+ None,
+ "1:03-cr-00076",
+ federal_defendant_number=1,
+ federal_dn_judge_initials_assigned="MR",
+ federal_dn_judge_initials_referred="DLH",
+ )
+ self.assertEqual(docket_matched.pk, d_3.pk)
+
+ def test_avoid_lookup_by_docket_number_components(self):
+ """If either the docket or the docket_data contains None values for
+ DN components. Avoid using them to match the docket.
+ """
+
+ # Dockets in DB contains docket_number components but the incoming data
+ # doesn't.
+ oldest_d = DocketFactory(
+ case_name="Young v. State",
+ docket_number="1:03-cr-00075",
+ court=self.court_appellate,
+ source=Docket.RECAP,
+ pacer_case_id=None,
+ federal_defendant_number=None,
+ federal_dn_judge_initials_assigned="MR",
+ federal_dn_judge_initials_referred="LM",
+ federal_dn_case_type="cv",
+ federal_dn_office_code="1",
+ )
+ DocketFactory(
+ case_name="Young v. State",
+ docket_number="1:03-cr-00075",
+ court=self.court_appellate,
+ source=Docket.RECAP,
+ pacer_case_id=None,
+ federal_defendant_number=None,
+ federal_dn_judge_initials_assigned="MR",
+ federal_dn_judge_initials_referred="LM",
+ federal_dn_case_type="cv",
+ federal_dn_office_code="1",
+ )
+ docket_matched = async_to_sync(find_docket_object)(
+ self.court_appellate.pk,
+ None,
+ "1:03-cr-00075",
+ federal_defendant_number=None,
+ federal_dn_judge_initials_assigned="",
+ federal_dn_judge_initials_referred="",
+ )
+ # DN components are not used to match the docket. The oldest one is
+ # selected instead.
+ self.assertEqual(docket_matched.pk, oldest_d.pk)
+
+ # Dockets in DB lacks of docket_number components.
+ oldest_d_1 = DocketFactory(
+ case_name="Young v. State",
+ docket_number="1:03-cr-00076",
+ court=self.court_appellate,
+ source=Docket.RECAP,
+ pacer_case_id=None,
+ federal_defendant_number=None,
+ federal_dn_judge_initials_assigned="",
+ federal_dn_judge_initials_referred="",
+ federal_dn_case_type="",
+ federal_dn_office_code="",
+ )
+ DocketFactory(
+ case_name="Young v. State",
+ docket_number="1:03-cr-00076",
+ court=self.court_appellate,
+ source=Docket.RECAP,
+ pacer_case_id=None,
+ federal_defendant_number=None,
+ federal_dn_judge_initials_assigned="",
+ federal_dn_judge_initials_referred="",
+ federal_dn_case_type="",
+ federal_dn_office_code="",
+ )
+ docket_matched = async_to_sync(find_docket_object)(
+ self.court_appellate.pk,
+ None,
+ "1:03-cr-00076",
+ federal_defendant_number=1,
+ federal_dn_judge_initials_assigned="MR",
+ federal_dn_judge_initials_referred="DLH",
+ )
+ # DN components are not used to match the docket. The oldest one is
+ # selected instead.
+ self.assertEqual(docket_matched.pk, oldest_d_1.pk)
+
class CleanUpDuplicateAppellateEntries(TestCase):
"""Test clean_up_duplicate_appellate_entries method that finds and clean
diff --git a/cl/recap_rss/tasks.py b/cl/recap_rss/tasks.py
index 747c5c9e6e..82b5cd2f24 100644
--- a/cl/recap_rss/tasks.py
+++ b/cl/recap_rss/tasks.py
@@ -340,7 +340,12 @@ def merge_rss_feed_contents(self, feed_data, court_pk, metadata_only=False):
# in another thread/process and we had a race condition.
continue
d = async_to_sync(find_docket_object)(
- court_pk, docket["pacer_case_id"], docket["docket_number"]
+ court_pk,
+ docket["pacer_case_id"],
+ docket["docket_number"],
+ docket.get("federal_defendant_number"),
+ docket.get("federal_dn_judge_initials_assigned"),
+ docket.get("federal_dn_judge_initials_referred"),
)
d.add_recap_source()
diff --git a/cl/scrapers/DupChecker.py b/cl/scrapers/DupChecker.py
index 7d5581955e..a5a5df729d 100644
--- a/cl/scrapers/DupChecker.py
+++ b/cl/scrapers/DupChecker.py
@@ -1,5 +1,9 @@
from juriscraper.AbstractSite import logger
+from cl.scrapers.exceptions import (
+ ConsecutiveDuplicatesError,
+ SingleDuplicateError,
+)
from cl.scrapers.models import UrlHash
from cl.search.models import Court
@@ -19,7 +23,6 @@ def __init__(
self.url_hash = None
self.dup_count = 0
self.last_found_date = None
- self.emulate_break = False
super().__init__(*args, **kwargs)
def _increment(self, current_date):
@@ -83,29 +86,29 @@ def press_on(
lookup_by="sha1",
):
"""Checks if a we have an `object_type` with identical content in the CL
- corpus by looking up `lookup_value` in the `lookup_by` field. Depending
- on the result of that, we either return True or False. True represents
- the fact that the next item should be processed. False means that either
- the item was a duplicate or that we've hit so many duplicates that we've
- stopped checking (we hit a duplicate threshold). Either way, the caller
- should move to the next item and try it.
+ corpus by looking up `lookup_value` in the `lookup_by` field.
- The effect of this is that this emulates for loop constructs for
- continue (False), break (False), return (True).
+ If the item is not a duplicate, we will return None, and the caller
+ will proceed normally
+
+ If the item is a duplicate, we will raise SingleDuplicateError
+
+ If the item is a duplicate following a series of duplicates greater than
+ our tolerance threshold, we will raise ConsecutiveDuplicatesError
+
+ If the item is a duplicate and the next item is from an already scraped
+ date, we will raise ConsecutiveDuplicatesError
Following logic applies:
+ - if we do not have the item
+ - early return
- if we have the item already
- and if the next date is before this date
- or if this is our duplicate threshold is exceeded
- break
- otherwise
- continue
- - if not
- - carry on
"""
- if self.emulate_break:
- return False
-
# check for a duplicate in the db.
if lookup_by == "sha1":
exists = object_type.objects.filter(sha1=lookup_value).exists()
@@ -116,41 +119,36 @@ def press_on(
else:
raise NotImplementedError("Unknown lookup_by parameter.")
- if exists:
- logger.info(
- f"Duplicate found on date: {current_date}, with lookup value: {lookup_value}"
- )
- self._increment(current_date)
-
- # If the next date in the Site object is less than (before) the
- # current date, we needn't continue because we should already have
- # that item.
- if next_date:
- already_scraped_next_date = next_date < current_date
- else:
- already_scraped_next_date = True
- if not self.full_crawl:
- if already_scraped_next_date:
- if self.court.pk == "mich":
- # Michigan sometimes has multiple occurrences of the
- # same case with different dates on a page.
- return False
- else:
- logger.info(
- "Next case occurs prior to when we found a "
- "duplicate. Court is up to date."
- )
- self.emulate_break = True
- return False
- elif self.dup_count >= self.dup_threshold:
- logger.info(
- f"Found {self.dup_count} duplicates in a row. Court is up to date."
- )
- self.emulate_break = True
- return False
- else:
- # This is a full crawl. Do not emulate a break, BUT be sure to
- # say that we shouldn't press on, since the item already exists.
- return False
+ if not exists:
+ return
+
+ logger.info(
+ f"Duplicate found on date: {current_date}, with lookup value: {lookup_value}"
+ )
+ self._increment(current_date)
+
+ # If the next date in the Site object is less than (before) the
+ # current date, we needn't continue because we should already have
+ # that item.
+ if next_date:
+ already_scraped_next_date = next_date < current_date
else:
- return True
+ already_scraped_next_date = True
+
+ # When in a full crawl, we do not raise a loop breaking
+ # `ConsecutiveDuplicatesError`
+ if not self.full_crawl:
+ if already_scraped_next_date:
+ if self.court.pk == "mich":
+ # Michigan sometimes has multiple occurrences of the
+ # same case with different dates on a page.
+ raise SingleDuplicateError(logger=logger)
+
+ message = "Next case occurs prior to when we found a duplicate. Court is up to date."
+ raise ConsecutiveDuplicatesError(message, logger=logger)
+ elif self.dup_count >= self.dup_threshold:
+ message = f"Found {self.dup_count} duplicates in a row. Court is up to date."
+ raise ConsecutiveDuplicatesError(message, logger=logger)
+
+ # Full crawl or not, this is a duplicate and we shouldn't store it
+ raise SingleDuplicateError(logger=logger)
diff --git a/cl/scrapers/exceptions.py b/cl/scrapers/exceptions.py
new file mode 100644
index 0000000000..dcefbb80d1
--- /dev/null
+++ b/cl/scrapers/exceptions.py
@@ -0,0 +1,82 @@
+import logging
+from typing import Optional
+
+from cl.lib.command_utils import logger
+
+
+class AutoLoggingException(Exception):
+ """Exception with defaults for logging, to be subclassed
+
+ We log expected exceptions to better understand what went wrong
+ Logger calls with level `logging.ERROR` are sent to Sentry, and
+ it's useful to send a `fingerprint` to force a specific grouping by court
+
+ Other `logger` calls are just printed on the console when using a
+ VerboseCommand with proper verbosity levels
+ """
+
+ logging_level = logging.DEBUG
+ message = ""
+ logger = logger
+
+ def __init__(
+ self,
+ message: str = "",
+ logger: Optional[logging.Logger] = None,
+ logging_level: Optional[int] = None,
+ fingerprint: Optional[list[str]] = None,
+ ):
+ if not message:
+ message = self.message
+ if not logger:
+ logger = self.logger
+ if not logging_level:
+ logging_level = self.logging_level
+
+ log_kwargs = {}
+ if fingerprint:
+ log_kwargs["extra"] = {"fingerprint": fingerprint}
+
+ logger.log(logging_level, message, **log_kwargs)
+ super().__init__(message)
+
+
+class ConsecutiveDuplicatesError(AutoLoggingException):
+ """Occurs when consecutive `SingleDuplicateError` are found,
+ which may be used as a signal to break the scraping loop
+ """
+
+ message = "DupChecker emulate break triggered."
+
+
+class SingleDuplicateError(AutoLoggingException):
+ """Occurs when an opinion or audio file already exists
+ in our database
+ """
+
+ message = "Skipping opinion due to duplicated content hash"
+
+
+class BadContentError(AutoLoggingException):
+ """Parent class for errors raised when downloading binary content"""
+
+
+class UnexpectedContentTypeError(BadContentError):
+ """Occurs when the content received from the server has
+ a different content type than the ones listed on
+ site.expected_content_types
+ """
+
+ logging_level = logging.ERROR
+
+
+class NoDownloadUrlError(BadContentError):
+ """Occurs when a DeferredList fetcher fails."""
+
+ logging_level = logging.ERROR
+
+
+class EmptyFileError(BadContentError):
+ """Occurs when the content of the response has lenght 0"""
+
+ logging_level = logging.ERROR
diff --git a/cl/scrapers/management/commands/cl_back_scrape_citations.py b/cl/scrapers/management/commands/cl_back_scrape_citations.py
new file mode 100644
index 0000000000..b2da0a4581
--- /dev/null
+++ b/cl/scrapers/management/commands/cl_back_scrape_citations.py
@@ -0,0 +1,150 @@
+"""
+When opinions are first published on the courts' sites, they won't have
+all their citations assigned. Some courts will publish the citations
+in the same pages we scrape, but months later
+
+This command re-uses the (back)scraper we use to get opinions, to get
+the lagged citations and associate them with the Opinions we first
+downloaded. If we find an Opinion we don't have in the database,
+we ingest it as in a regular scrape
+"""
+
+from django.db import IntegrityError
+from django.utils.encoding import force_bytes
+
+from cl.lib.command_utils import logger
+from cl.lib.crypto import sha1
+from cl.scrapers.DupChecker import DupChecker
+from cl.scrapers.exceptions import BadContentError
+from cl.scrapers.management.commands import cl_back_scrape_opinions
+from cl.scrapers.management.commands.cl_scrape_opinions import make_citation
+from cl.scrapers.utils import get_binary_content
+from cl.search.models import Citation, Court, Opinion
+
+
+class Command(cl_back_scrape_opinions.Command):
+ scrape_target_descr = "citations"
+
+ def scrape_court(
+ self,
+ site,
+ full_crawl: bool = False,
+ ocr_available: bool = True,
+ backscrape: bool = False,
+ ):
+ """
+ If the scraped case has citation data
+ Check for Opinion existance via content hash
+ If we have the Opinion
+ if we don't have the citation -> ingest
+ if we already have the citation -> pass
+ If we don't have the Opinion
+ ingest the opinion with it's citation, that is to say,
+ use the regular scraping process!
+
+ :param site: scraper object that has already downloaded
+ it's case data
+ """
+ court_str = site.court_id.split(".")[-1].split("_")[0]
+ court = Court.objects.get(id=court_str)
+ dup_checker = DupChecker(court, full_crawl=True)
+
+ for case in site:
+ citation = case.get("citations")
+ parallel_citation = case.get("parallel_citations")
+ if not citation and not parallel_citation:
+ logger.debug(
+ "No citation, skipping row for case %s",
+ case.get("case_names"),
+ )
+ continue
+
+ try:
+ content = get_binary_content(case["download_urls"], site)
+ except BadContentError:
+ continue
+
+ sha1_hash = sha1(force_bytes(content))
+
+ try:
+ cluster = Opinion.objects.get(sha1=sha1_hash).cluster
+ except Opinion.DoesNotExist:
+ # populate special key to avoid downloading the file again
+ case["content"] = content
+
+ logger.info(
+ "Case '%s', opinion '%s' has no matching hash in the DB. "
+ "Has a citation '%s'. Will try to ingest all objects",
+ case["case_names"],
+ case["download_urls"],
+ citation or parallel_citation,
+ )
+
+ self.ingest_a_case(case, None, True, site, dup_checker, court)
+ continue
+
+ for cite in [citation, parallel_citation]:
+ if not cite:
+ continue
+
+ citation_candidate = make_citation(cite, cluster, court_str)
+ if not citation_candidate:
+ continue
+
+ if self.citation_is_duplicated(citation_candidate, cite):
+ continue
+
+ try:
+ citation_candidate.save()
+ logger.info(
+ "Saved citation %s for cluster %s", cite, cluster
+ )
+ except IntegrityError:
+ logger.warning(
+ "Error when saving citation %s for cluster %s",
+ cite,
+ cluster,
+ )
+
+ def citation_is_duplicated(
+ self, citation_candidate: Citation, cite: str
+ ) -> bool:
+ """Checks if the citation is duplicated for the cluster
+
+ Following corpus_importer.utils.add_citations_to_cluster we
+ identify 2 types of duplication:
+ - exact: a citation with the same fields already exists for the cluster
+ - duplication in the same reporter: the cluster already has a citation
+ in that reporter
+
+ :param citation_candidate: the citation object
+ :param cite: citation string
+
+ :return: True if citation is duplicated, False if not
+ """
+ citation_params = {**citation_candidate.__dict__}
+ citation_params.pop("_state", "")
+ citation_params.pop("id", "")
+ cluster_id = citation_candidate.cluster.id
+
+ # Exact duplication
+ if Citation.objects.filter(**citation_params).exists():
+ logger.info(
+ "Citation '%s' already exists for cluster %s",
+ cite,
+ cluster_id,
+ )
+ return True
+
+ # Duplication in the same reporter
+ if Citation.objects.filter(
+ cluster_id=cluster_id, reporter=citation_candidate.reporter
+ ).exists():
+ logger.info(
+ "Another citation in the same reporter '%s' exists for cluster %s",
+ citation_candidate.reporter,
+ cluster_id,
+ )
+ return True
+
+ return False
diff --git a/cl/scrapers/management/commands/cl_back_scrape_oral_arguments.py b/cl/scrapers/management/commands/cl_back_scrape_oral_arguments.py
index b1105f01e0..299a091597 100644
--- a/cl/scrapers/management/commands/cl_back_scrape_oral_arguments.py
+++ b/cl/scrapers/management/commands/cl_back_scrape_oral_arguments.py
@@ -5,7 +5,7 @@
class Command(cl_scrape_oral_arguments.Command):
- def parse_and_scrape_site(self, mod, full_crawl):
+ def parse_and_scrape_site(self, mod, options: dict):
court_str = mod.__name__.split(".")[-1].split("_")[0]
logger.info(f'Using court_str: "{court_str}"')
diff --git a/cl/scrapers/management/commands/cl_scrape_opinions.py b/cl/scrapers/management/commands/cl_scrape_opinions.py
index 94decc0b9b..67dac880ab 100644
--- a/cl/scrapers/management/commands/cl_scrape_opinions.py
+++ b/cl/scrapers/management/commands/cl_scrape_opinions.py
@@ -1,6 +1,7 @@
import signal
import sys
import time
+import traceback
from datetime import date
from typing import Any, Dict, List, Optional, Tuple, Union
@@ -22,6 +23,11 @@
from cl.lib.string_utils import trunc
from cl.people_db.lookup_utils import lookup_judges_by_messy_str
from cl.scrapers.DupChecker import DupChecker
+from cl.scrapers.exceptions import (
+ BadContentError,
+ ConsecutiveDuplicatesError,
+ SingleDuplicateError,
+)
from cl.scrapers.tasks import extract_doc_content
from cl.scrapers.utils import (
get_binary_content,
@@ -83,6 +89,15 @@ def make_objects(
) -> Tuple[Docket, Opinion, OpinionCluster, List[Citation]]:
"""Takes the meta data from the scraper and associates it with objects.
+ The keys returned by juriscraper scrapers are defined by `self._all_attrs`
+ on OpinionSite and OralArgumentSite, where the legacy convention is to use
+ plural names.
+
+ However, this function is also used by importers and user pages, that
+ may not respect this convention, thus the duplication of singular and
+ plural names, like in
+ `item.get("disposition") or item.get("dispositions", "")`
+
Returns the created objects.
"""
blocked = item["blocked_statuses"]
@@ -98,27 +113,36 @@ def make_objects(
docket = update_or_create_docket(
item["case_names"],
case_name_short,
- court.pk,
+ court,
item.get("docket_numbers", ""),
item.get("source") or Docket.SCRAPER,
+ from_harvard=False,
blocked=blocked,
date_blocked=date_blocked,
+ appeal_from_str=item.get("lower_courts", ""),
)
+ # Note that if opinion.author_str has no value, and cluster.judges find
+ # a single judge, opinion.author will be populated with that Person object
+ # Check `save_everything`
+
+ # For a discussion on syllabus vs summary, check
+ # https://github.com/freelawproject/juriscraper/issues/66
cluster = OpinionCluster(
- judges=item.get("judges", ""),
date_filed=item["case_dates"],
date_filed_is_approximate=item["date_filed_is_approximate"],
case_name=item["case_names"],
case_name_short=case_name_short,
source=item.get("cluster_source") or SOURCES.COURT_WEBSITE,
precedential_status=item["precedential_statuses"],
- nature_of_suit=item.get("nature_of_suit", ""),
- summary=item.get("summary", ""),
blocked=blocked,
date_blocked=date_blocked,
+ judges=item.get("judges", ""),
+ nature_of_suit=item.get("nature_of_suit", ""),
+ disposition=item.get("disposition") or item.get("dispositions", ""),
+ other_dates=item.get("other_dates", ""),
+ summary=item.get("summary", ""),
syllabus=item.get("summaries", ""),
- disposition=item.get("disposition") or "",
)
cites = [item.get(key, "") for key in ["citations", "parallel_citations"]]
@@ -133,10 +157,12 @@ def make_objects(
url = ""
opinion = Opinion(
- type=Opinion.COMBINED,
+ type=item.get("types", Opinion.COMBINED),
sha1=sha1_hash,
download_url=url,
- author_str=item.get("author_str") or "",
+ joined_by_str=item.get("joined_by", ""),
+ per_curiam=item.get("per_curiam", False),
+ author_str=item.get("author_str") or item.get("authors", ""),
)
cf = ContentFile(content)
@@ -165,14 +191,21 @@ def save_everything(
citation.cluster_id = cluster.pk
citation.save()
+ if opinion.author_str:
+ candidate = async_to_sync(lookup_judges_by_messy_str)(
+ opinion.author_str, docket.court.pk, cluster.date_filed
+ )
+ if len(candidate) == 1:
+ opinion.author = candidate[0]
+
if cluster.judges:
candidate_judges = async_to_sync(lookup_judges_by_messy_str)(
cluster.judges, docket.court.pk, cluster.date_filed
)
- if len(candidate_judges) == 1:
- opinion.author = candidate_judges[0]
- if len(candidate_judges) > 1:
+ if len(candidate_judges) == 1 and not opinion.author_str:
+ opinion.author = candidate_judges[0]
+ elif len(candidate_judges) > 1:
for candidate in candidate_judges:
cluster.panel.add(candidate)
@@ -186,6 +219,7 @@ def save_everything(
class Command(VerboseCommand):
help = "Runs the Juriscraper toolkit against one or many jurisdictions."
+ scrape_target_descr = "opinions" # for logging purposes
def __init__(self, stdout=None, stderr=None, no_color=False):
super().__init__(stdout=None, stderr=None, no_color=False)
@@ -234,7 +268,13 @@ def add_arguments(self, parser):
help="Disable duplicate aborting.",
)
- def scrape_court(self, site, full_crawl=False, ocr_available=True):
+ def scrape_court(
+ self,
+ site,
+ full_crawl: bool = False,
+ ocr_available: bool = True,
+ backscrape: bool = False,
+ ):
# Get the court object early for logging
# opinions.united_states.federal.ca9_u --> ca9
court_str = site.court_id.split(".")[-1].split("_")[0]
@@ -246,112 +286,116 @@ def scrape_court(self, site, full_crawl=False, ocr_available=True):
return
if site.cookies:
- logger.info(f"Using cookies: {site.cookies}")
- logger.debug(f"#{len(site)} opinions found.")
- added = 0
- for i, item in enumerate(site):
- # Minn and Mass currently require browser specific user agents
- if court_str in ["minn", "minnctapp", "mass", "massappct"]:
- headers = site.headers
- else:
- headers = {"User-Agent": "CourtListener"}
-
- msg, r = get_binary_content(
- item["download_urls"],
- site,
- headers,
- method=site.method,
- )
- if msg:
- fingerprint = [f"{court_str}-unexpected-content-type"]
- logger.error(msg, extra={"fingerprint": fingerprint})
- continue
+ logger.info("Using cookies: %s", site.cookies)
- content = site.cleanup_content(r.content)
+ logger.debug("#%s %s found.", len(site), self.scrape_target_descr)
- current_date = item["case_dates"]
+ added = 0
+ for i, item in enumerate(site):
try:
next_date = site[i + 1]["case_dates"]
except IndexError:
next_date = None
- # request.content is sometimes a str, sometimes unicode, so
- # force it all to be bytes, pleasing hashlib.
- sha1_hash = sha1(force_bytes(content))
- if (
- court_str == "nev"
- and item["precedential_statuses"] == "Unpublished"
- ) or court_str in ["neb"]:
- # Nevada's non-precedential cases have different SHA1 sums
- # every time.
-
- # Nebraska updates the pdf causing the SHA1 to not match
- # the opinions in CL causing duplicates. See CL issue #1452
-
- lookup_params = {
- "lookup_value": item["download_urls"],
- "lookup_by": "download_url",
- }
- else:
- lookup_params = {
- "lookup_value": sha1_hash,
- "lookup_by": "sha1",
- }
-
- proceed = dup_checker.press_on(
- Opinion, current_date, next_date, **lookup_params
- )
- if dup_checker.emulate_break:
- logger.debug("Emulate break triggered.")
+ try:
+ self.ingest_a_case(
+ item, next_date, ocr_available, site, dup_checker, court
+ )
+ added += 1
+ except ConsecutiveDuplicatesError:
break
- if not proceed:
- logger.debug("Skipping opinion.")
- continue
-
- # Not a duplicate, carry on
- logger.info(
- f"Adding new document found at: {item['download_urls'].encode()}"
- )
- dup_checker.reset()
-
- child_court = get_child_court(
- item.get("child_courts", ""), court.id
- )
-
- docket, opinion, cluster, citations = make_objects(
- item, child_court or court, sha1_hash, content
- )
-
- save_everything(
- items={
- "docket": docket,
- "opinion": opinion,
- "cluster": cluster,
- "citations": citations,
- },
- index=False,
- )
- extract_doc_content.delay(
- opinion.pk,
- ocr_available=ocr_available,
- citation_jitter=True,
- juriscraper_module=site.court_id,
- )
-
- logger.info(
- f"Successfully added opinion {opinion.pk}: "
- f"{item['case_names'].encode()}"
- )
- added += 1
+ except (SingleDuplicateError, BadContentError):
+ pass
# Update the hash if everything finishes properly.
logger.debug(
- f"{site.court_id}: Successfully crawled {added}/{len(site)} opinions."
+ "%s: Successfully crawled %s/%s %s.",
+ site.court_id,
+ added,
+ len(site),
+ self.scrape_target_descr,
)
if not full_crawl:
# Only update the hash if no errors occurred.
dup_checker.update_site_hash(site.hash)
+ def ingest_a_case(
+ self,
+ item,
+ next_case_date: date | None,
+ ocr_available: bool,
+ site,
+ dup_checker: DupChecker,
+ court: Court,
+ ):
+ if item.get("content"):
+ content = item.pop("content")
+ else:
+ content = get_binary_content(item["download_urls"], site)
+
+ # request.content is sometimes a str, sometimes unicode, so
+ # force it all to be bytes, pleasing hashlib.
+ sha1_hash = sha1(force_bytes(content))
+
+ if (
+ court.pk == "nev"
+ and item["precedential_statuses"] == "Unpublished"
+ ) or court.pk in ["neb"]:
+ # Nevada's non-precedential cases have different SHA1 sums
+ # every time.
+
+ # Nebraska updates the pdf causing the SHA1 to not match
+ # the opinions in CL causing duplicates. See CL issue #1452
+
+ lookup_params = {
+ "lookup_value": item["download_urls"],
+ "lookup_by": "download_url",
+ }
+ else:
+ lookup_params = {
+ "lookup_value": sha1_hash,
+ "lookup_by": "sha1",
+ }
+
+ # Duplicates will raise errors
+ dup_checker.press_on(
+ Opinion, item["case_dates"], next_case_date, **lookup_params
+ )
+
+ # Not a duplicate, carry on
+ logger.info(
+ "Adding new document found at: %s", item["download_urls"].encode()
+ )
+ dup_checker.reset()
+
+ child_court = get_child_court(item.get("child_courts", ""), court.id)
+
+ docket, opinion, cluster, citations = make_objects(
+ item, child_court or court, sha1_hash, content
+ )
+
+ save_everything(
+ items={
+ "docket": docket,
+ "opinion": opinion,
+ "cluster": cluster,
+ "citations": citations,
+ },
+ index=False,
+ )
+ extract_doc_content.delay(
+ opinion.pk,
+ ocr_available=ocr_available,
+ citation_jitter=True,
+ juriscraper_module=site.court_id,
+ )
+
+ logger.info(
+ "Successfully added opinion %s: %s",
+ opinion.pk,
+ item["case_names"].encode(),
+ )
+
def parse_and_scrape_site(self, mod, options: dict):
site = mod.Site().parse()
self.scrape_court(site, options["full_crawl"])
@@ -395,6 +439,7 @@ def handle(self, *args, **options):
capture_exception(
e, fingerprint=[module_string, "{{ default }}"]
)
+ logger.debug(traceback.format_exc())
last_court_in_list = i == (num_courts - 1)
daemon_mode = options["daemon"]
if last_court_in_list:
diff --git a/cl/scrapers/management/commands/cl_scrape_oral_arguments.py b/cl/scrapers/management/commands/cl_scrape_oral_arguments.py
index ed21e1ae26..ad284381f4 100644
--- a/cl/scrapers/management/commands/cl_scrape_oral_arguments.py
+++ b/cl/scrapers/management/commands/cl_scrape_oral_arguments.py
@@ -74,9 +74,10 @@ def make_objects(
docket = update_or_create_docket(
item["case_names"],
case_name_short,
- court.pk,
+ court,
item.get("docket_numbers", ""),
item.get("source") or Docket.SCRAPER,
+ from_harvard=False,
blocked=blocked,
date_blocked=date_blocked,
date_argued=item["case_dates"],
@@ -105,84 +106,49 @@ def make_objects(
class Command(cl_scrape_opinions.Command):
- def scrape_court(
+ scrape_target_descr = "oral arguments"
+
+ def ingest_a_case(
self,
+ item,
+ next_case_date: date | None,
+ ocr_available: bool,
site,
- full_crawl: bool = False,
+ dup_checker: DupChecker,
+ court: Court,
backscrape: bool = False,
- ) -> None:
- # Get the court object early for logging
- # opinions.united_states.federal.ca9_u --> ca9
- court_str = site.court_id.split(".")[-1].split("_")[0]
- court = Court.objects.get(pk=court_str)
-
- dup_checker = DupChecker(court, full_crawl=full_crawl)
- abort = dup_checker.abort_by_url_hash(site.url, site.hash)
- if abort:
- return
-
- if site.cookies:
- logger.info(f"Using cookies: {site.cookies}")
- for i, item in enumerate(site):
- msg, r = get_binary_content(
- item["download_urls"],
- site,
- headers={"User-Agent": "CourtListener"},
- method=site.method,
- )
- if msg:
- fingerprint = [f"{court_str}-unexpected-content-type"]
- logger.error(msg, extra={"fingerprint": fingerprint})
- continue
-
- content = site.cleanup_content(r.content)
-
- current_date = item["case_dates"]
- try:
- next_date = site[i + 1]["case_dates"]
- except IndexError:
- next_date = None
-
- # request.content is sometimes a str, sometimes unicode, so
- # force it all to be bytes, pleasing hashlib.
- sha1_hash = sha1(force_bytes(content))
- onwards = dup_checker.press_on(
- Audio,
- current_date,
- next_date,
- lookup_value=sha1_hash,
- lookup_by="sha1",
- )
- if dup_checker.emulate_break:
- break
-
- if onwards:
- # Not a duplicate, carry on
- logger.info(
- f"Adding new document found at: {item['download_urls'].encode()}"
- )
- dup_checker.reset()
-
- docket, audio_file = make_objects(
- item, court, sha1_hash, content
- )
-
- save_everything(
- items={"docket": docket, "audio_file": audio_file},
- index=False,
- backscrape=backscrape,
- )
- process_audio_file.delay(audio_file.pk)
-
- logger.info(
- "Successfully added audio file {pk}: {name}".format(
- pk=audio_file.pk,
- name=item["case_names"].encode(),
- )
- )
-
- # Update the hash if everything finishes properly.
- logger.info(f"{site.court_id}: Successfully crawled oral arguments.")
- if not full_crawl:
- # Only update the hash if no errors occurred.
- dup_checker.update_site_hash(site.hash)
+ ):
+ content = get_binary_content(item["download_urls"], site)
+ # request.content is sometimes a str, sometimes unicode, so
+ # force it all to be bytes, pleasing hashlib.
+ sha1_hash = sha1(force_bytes(content))
+
+ dup_checker.press_on(
+ Audio,
+ item["case_dates"],
+ next_case_date,
+ lookup_value=sha1_hash,
+ lookup_by="sha1",
+ )
+
+ logger.info(
+ "Adding new %s found at: %s",
+ self.scrape_target_descr,
+ item["download_urls"].encode(),
+ )
+ dup_checker.reset()
+
+ docket, audio_file = make_objects(item, court, sha1_hash, content)
+
+ save_everything(
+ items={"docket": docket, "audio_file": audio_file},
+ index=False,
+ backscrape=backscrape,
+ )
+ process_audio_file.delay(audio_file.pk)
+
+ logger.info(
+ "Successfully added audio file %s: %s",
+ audio_file.pk,
+ item["case_names"].encode(),
+ )
diff --git a/cl/scrapers/tasks.py b/cl/scrapers/tasks.py
index 85f0af1796..c60971c572 100644
--- a/cl/scrapers/tasks.py
+++ b/cl/scrapers/tasks.py
@@ -410,15 +410,16 @@ def update_docket_info_iquery(self, d_pk: int, court_id: str) -> None:
:param court_id: The court of the docket. Needed for throttling by court.
:return: None
"""
- cookies = get_or_cache_pacer_cookies(
+ session_data = get_or_cache_pacer_cookies(
"pacer_scraper",
settings.PACER_USERNAME,
password=settings.PACER_PASSWORD,
)
s = ProxyPacerSession(
- cookies=cookies,
+ cookies=session_data.cookies,
username=settings.PACER_USERNAME,
password=settings.PACER_PASSWORD,
+ proxy=session_data.proxy_address,
)
d = Docket.objects.get(pk=d_pk, court_id=court_id)
report = CaseQuery(map_cl_to_pacer_id(d.court_id), s)
@@ -438,6 +439,7 @@ def update_docket_info_iquery(self, d_pk: int, court_id: str) -> None:
save_iquery_to_docket(
self,
report.data,
+ report.response.text,
d,
tag_names=None,
add_to_solr=True,
diff --git a/cl/scrapers/tests.py b/cl/scrapers/tests.py
index 558423dc4c..375987426a 100644
--- a/cl/scrapers/tests.py
+++ b/cl/scrapers/tests.py
@@ -8,6 +8,7 @@
from django.conf import settings
from django.core.files.base import ContentFile
from django.utils.timezone import now
+from juriscraper.AbstractSite import logger
from cl.alerts.factories import AlertFactory
from cl.alerts.models import Alert
@@ -20,16 +21,33 @@
from cl.lib.microservice_utils import microservice
from cl.lib.test_helpers import generate_docket_target_sources
from cl.scrapers.DupChecker import DupChecker
+from cl.scrapers.exceptions import (
+ ConsecutiveDuplicatesError,
+ SingleDuplicateError,
+ UnexpectedContentTypeError,
+)
from cl.scrapers.management.commands import (
+ cl_back_scrape_citations,
cl_scrape_opinions,
cl_scrape_oral_arguments,
)
from cl.scrapers.models import UrlHash
from cl.scrapers.tasks import extract_doc_content, process_audio_file
from cl.scrapers.test_assets import test_opinion_scraper, test_oral_arg_scraper
-from cl.scrapers.utils import get_binary_content, get_extension
-from cl.search.factories import CourtFactory, DocketFactory
-from cl.search.models import Court, Docket, Opinion
+from cl.scrapers.utils import (
+ case_names_are_too_different,
+ get_binary_content,
+ get_existing_docket,
+ get_extension,
+ update_or_create_docket,
+)
+from cl.search.factories import (
+ CourtFactory,
+ DocketFactory,
+ OpinionClusterFactory,
+ OpinionFactory,
+)
+from cl.search.models import Citation, Court, Docket, Opinion
from cl.settings import MEDIA_ROOT
from cl.tests.cases import ESIndexTestCase, SimpleTestCase, TestCase
from cl.tests.fixtures import ONE_SECOND_MP3_BYTES, SMALL_WAV_BYTES
@@ -71,7 +89,7 @@ def test_ingest_opinions_from_scraper(self) -> None:
"""Can we successfully ingest opinions at a high level?"""
d_1 = DocketFactory(
- case_name="Tarrant Regional Water District v. Herrmann old",
+ case_name="Tarrant Regional Water District v. Herrmann",
docket_number="11-889",
court=self.court,
source=Docket.RECAP,
@@ -79,7 +97,7 @@ def test_ingest_opinions_from_scraper(self) -> None:
)
d_2 = DocketFactory(
- case_name="State of Indiana v. Charles Barker old",
+ case_name="State of Indiana v. Charles Barker",
docket_number="49S00-0308-DP-392",
court=self.court,
source=Docket.IDB,
@@ -87,7 +105,7 @@ def test_ingest_opinions_from_scraper(self) -> None:
)
d_3 = DocketFactory(
- case_name="Intl Fidlty Ins Co v. Ideal Elec Sec Co old",
+ case_name="Intl Fidlty Ins Co v. Ideal Elec Sec Co",
docket_number="96-7169",
court=self.court,
source=Docket.RECAP_AND_IDB,
@@ -205,7 +223,7 @@ def test_ingest_oral_arguments(self) -> None:
"""Can we successfully ingest oral arguments at a high level?"""
d_1 = DocketFactory(
- case_name="Jeremy v. Julian old",
+ case_name="Jeremy v. Julian",
docket_number="23-232388",
court=self.court,
source=Docket.RECAP,
@@ -230,7 +248,6 @@ def test_ingest_oral_arguments(self) -> None:
f"Should have 2 dockets, not {dockets.count()}",
)
d_1.refresh_from_db()
- self.assertEqual(d_1.case_name, "Jeremy v. Julian")
self.assertEqual(d_1.source, Docket.RECAP_AND_SCRAPER)
# Confirm that OA Search Alerts are properly triggered after an OA is
@@ -442,121 +459,111 @@ def test_press_on_with_an_empty_database(self) -> None:
site = test_opinion_scraper.Site()
site.hash = "this is a dummy hash code string"
for dup_checker in self.dup_checkers:
- onwards = dup_checker.press_on(
- Opinion,
- now(),
- now() - timedelta(days=1),
- lookup_value="content",
- lookup_by="sha1",
- )
- if dup_checker.full_crawl:
- self.assertTrue(
- onwards,
- "DupChecker says to abort during a full crawl. This should "
- "never happen.",
- )
- elif dup_checker.full_crawl is False:
- count = Opinion.objects.all().count()
- self.assertTrue(
- onwards,
- "DupChecker says to abort on dups when the database has %s "
- "Documents." % count,
+ try:
+ dup_checker.press_on(
+ Opinion,
+ now(),
+ now() - timedelta(days=1),
+ lookup_value="content",
+ lookup_by="sha1",
)
+ except (SingleDuplicateError, ConsecutiveDuplicatesError):
+ if dup_checker.full_crawl:
+ failure = "DupChecker says to abort during a full crawl. This should never happen."
+ else:
+ count = Opinion.objects.all().count()
+ failure = f"DupChecker says to abort on dups when the database has {count} Documents."
+ self.fail(failure)
-class DupcheckerWithFixturesTest(TestCase):
- fixtures = [
- "test_court.json",
- "judge_judy.json",
- "test_objects_search.json",
- ]
-
+class DupcheckerPressOnTest(TestCase):
def setUp(self) -> None:
super().setUp()
self.court = Court.objects.get(pk="test")
- # Set the dup_threshold to zero for these tests
- self.dup_checkers = [
- DupChecker(self.court, full_crawl=True, dup_threshold=0),
- DupChecker(self.court, full_crawl=False, dup_threshold=0),
- ]
+ self.dc_full_crawl = DupChecker(self.court, True, 2)
+ self.dc_not_full_crawl = DupChecker(self.court, False, 2)
+ self.dup_checkers = [self.dc_full_crawl, self.dc_not_full_crawl]
+ self.dup_hash = "1" * 40
+ self.press_on_args = [Opinion, now(), now(), self.dup_hash]
- # Set up the hash value using one in the fixture.
- self.content_hash = "asdfasdfasdfasdfasdfasddf"
+ docket = DocketFactory()
+ cluster = OpinionClusterFactory(docket=docket)
+ opinion = OpinionFactory(sha1=self.dup_hash, cluster=cluster)
+
+ def test_press_on_no_dup(self) -> None:
+ """Does the DupChecker raises no error when seeing a new hash?"""
+ self.dc_full_crawl.press_on(*self.press_on_args[:-1], "not a dup")
+ self.dc_not_full_crawl.press_on(*self.press_on_args[:-1], "not a dup")
def test_press_on_with_a_dup_found(self) -> None:
- for dup_checker in self.dup_checkers:
- onwards = dup_checker.press_on(
- Opinion,
- now(),
- now(),
- lookup_value=self.content_hash,
- lookup_by="sha1",
+ """Do we raise the appropiate exceptions when a dup is found?"""
+ # First duplicate
+ try:
+ self.dc_full_crawl.press_on(*self.press_on_args)
+ self.fail("Should raise SingleDuplicateError")
+ except ConsecutiveDuplicatesError:
+ self.fail("Full crawl raised a loop breaking exception")
+ except SingleDuplicateError:
+ pass # we expect this to happen
+
+ # Second duplicate, dup threshold = 2
+ try:
+ self.dc_full_crawl.press_on(*self.press_on_args)
+ self.fail("Should raise SingleDuplicateError")
+ except ConsecutiveDuplicatesError:
+ self.fail("Full crawl raised a loop breaking exception")
+ except SingleDuplicateError:
+ pass
+
+ # First duplicate
+ try:
+ self.dc_not_full_crawl.press_on(*self.press_on_args)
+ self.fail("Should raise SingleDuplicateError")
+ except SingleDuplicateError:
+ pass
+ except ConsecutiveDuplicatesError:
+ self.fail(
+ "Dup threshold is 1, should not raise ConsecutiveDuplicatesError"
)
- if dup_checker.full_crawl:
- self.assertFalse(
- onwards,
- "DupChecker returned True during a full crawl, but there "
- "should be duplicates in the database.",
- )
- self.assertFalse(
- dup_checker.emulate_break,
- "DupChecker said to emulate a break during a full crawl. "
- "Nothing should stop a full crawl!",
- )
- elif dup_checker.full_crawl is False:
- self.assertFalse(
- onwards,
- "DupChecker returned %s but there should be a duplicate in "
- "the database. dup_count is %s, and dup_threshold is %s"
- % (
- onwards,
- dup_checker.dup_count,
- dup_checker.dup_threshold,
- ),
- )
- self.assertTrue(
- dup_checker.emulate_break,
- "We should have hit a break but didn't.",
- )
+ # Second duplicate, dup threshold = 2
+ try:
+ self.dc_not_full_crawl.press_on(*self.press_on_args)
+ self.fail("Should raise ConsecutiveDuplicatesError")
+ except SingleDuplicateError:
+ self.fail("Should raise ConsecutiveDuplicatesError")
+ except ConsecutiveDuplicatesError:
+ pass # expected behavior
def test_press_on_with_dup_found_and_older_date(self) -> None:
- for dup_checker in self.dup_checkers:
- # Note that the next case occurs prior to the current one
- onwards = dup_checker.press_on(
- Opinion,
- now(),
- now() - timedelta(days=1),
- lookup_value=self.content_hash,
- lookup_by="sha1",
+ """Do we raise the appropiate exception when a duplicate is found
+ and we account for case dates?
+ """
+ self.dc_not_full_crawl.reset()
+ self.dc_full_crawl.reset()
+
+ # duplicated case occurs prior to the current one
+ args = [*self.press_on_args]
+ args[2] = now() - timedelta(days=1)
+
+ try:
+ self.dc_full_crawl.press_on(*args)
+ self.fail("Expected SingleDuplicateError")
+ except SingleDuplicateError:
+ pass
+ except ConsecutiveDuplicatesError:
+ self.fail(
+ "This a full crawl, ConsecutiveDuplicatesError should not be raised"
)
- if dup_checker.full_crawl:
- self.assertFalse(
- onwards,
- "DupChecker returned True during a full crawl, but there "
- "should be duplicates in the database.",
- )
- self.assertFalse(
- dup_checker.emulate_break,
- "DupChecker said to emulate a break during a full crawl. "
- "Nothing should stop a full crawl!",
- )
- else:
- self.assertFalse(
- onwards,
- "DupChecker returned %s but there should be a duplicate in "
- "the database. dup_count is %s, and dup_threshold is %s"
- % (
- onwards,
- dup_checker.dup_count,
- dup_checker.dup_threshold,
- ),
- )
- self.assertTrue(
- dup_checker.emulate_break,
- "We should have hit a break but didn't.",
- )
+
+ try:
+ self.dc_not_full_crawl.press_on(*args)
+ self.fail("Expected loop breaking ConsecutiveDuplicatesError")
+ except SingleDuplicateError:
+ self.fail("Expected loop breaking ConsecutiveDuplicatesError")
+ except ConsecutiveDuplicatesError:
+ pass
class AudioFileTaskTest(TestCase):
@@ -627,15 +634,20 @@ def setUp(self):
self.mock_response.content = b"not empty"
self.mock_response.headers = {"Content-Type": "application/pdf"}
self.site = test_opinion_scraper.Site()
+ self.site.method = "GET"
+ self.logger = logger
@mock.patch("requests.Session.get")
def test_unexpected_content_type(self, mock_get):
"""Test when content type doesn't match scraper expectation."""
mock_get.return_value = self.mock_response
self.site.expected_content_types = ["text/html"]
-
- msg, _ = get_binary_content("/dummy/url/", self.site, headers={})
- self.assertIn("UnexpectedContentTypeError:", msg)
+ self.assertRaises(
+ UnexpectedContentTypeError,
+ get_binary_content,
+ "/dummy/url/",
+ self.site,
+ )
@mock.patch("requests.Session.get")
def test_correct_content_type(self, mock_get):
@@ -643,15 +655,15 @@ def test_correct_content_type(self, mock_get):
mock_get.return_value = self.mock_response
self.site.expected_content_types = ["application/pdf"]
- msg, _ = get_binary_content("/dummy/url/", self.site, headers={})
- self.assertEqual("", msg)
+ with mock.patch.object(self.logger, "error") as error_mock:
+ _ = get_binary_content("/dummy/url/", self.site)
- self.mock_response.headers = {
- "Content-Type": "application/pdf;charset=utf-8"
- }
- mock_get.return_value = self.mock_response
- msg, _ = get_binary_content("/dummy/url/", self.site, headers={})
- self.assertEqual("", msg)
+ self.mock_response.headers = {
+ "Content-Type": "application/pdf;charset=utf-8"
+ }
+ mock_get.return_value = self.mock_response
+ _ = get_binary_content("/dummy/url/", self.site)
+ error_mock.assert_not_called()
@mock.patch("requests.Session.get")
def test_no_content_type(self, mock_get):
@@ -659,5 +671,199 @@ def test_no_content_type(self, mock_get):
mock_get.return_value = self.mock_response
self.site.expected_content_types = None
- msg, _ = get_binary_content("/dummy/url/", self.site, headers={})
- self.assertEqual("", msg)
+ with mock.patch.object(self.logger, "error") as error_mock:
+ _ = get_binary_content("/dummy/url/", self.site)
+ error_mock.assert_not_called()
+
+
+class ScrapeCitationsTest(TestCase):
+ """This class only tests the update of existing clusters
+ Since the ingestion of new clusters and their citations call
+ super().scrape_court(), it should be tested in the superclass
+ """
+
+ def setUp(self):
+ keys = [
+ "download_urls",
+ "case_names",
+ "citations",
+ "parallel_citations",
+ ]
+ self.mock_site = mock.MagicMock()
+ self.mock_site.__iter__.return_value = [
+ # update
+ dict(zip(keys, ["", "something", "482 Md. 342", ""])),
+ # exact duplicate
+ dict(zip(keys, ["", "something", "", "482 Md. 342"])),
+ # reporter duplicate
+ dict(zip(keys, ["", "something", "485 Md. 111", ""])),
+ # no citation, ignore
+ dict(zip(keys, ["", "something", "", ""])),
+ ]
+ self.mock_site.court_id = "juriscraper.md"
+ self.hash = "1234" * 10
+ self.hashes = [self.hash, self.hash, self.hash, "111"]
+
+ court = CourtFactory(id="md")
+ docket = DocketFactory(
+ case_name="Attorney Grievance v. Taniform",
+ docket_number="40ag/21",
+ court_id="md",
+ source=Docket.SCRAPER,
+ pacer_case_id=None,
+ )
+ self.cluster = OpinionClusterFactory(docket=docket)
+ opinion = OpinionFactory(sha1=self.hash, cluster=self.cluster)
+
+ def test_citation_scraper(self):
+ """Test if citation scraper creates a citation or ignores duplicates"""
+ cmd = "cl.scrapers.management.commands.cl_back_scrape_citations"
+ with mock.patch(f"{cmd}.sha1", side_effect=self.hashes), mock.patch(
+ f"{cmd}.get_binary_content", return_value="placeholder"
+ ):
+ cl_back_scrape_citations.Command().scrape_court(self.mock_site)
+
+ citations = Citation.objects.filter(cluster=self.cluster).count()
+ self.assertEqual(citations, 1, "Exactly 1 citation was expected")
+
+
+class ScraperDocketMatchingTest(TestCase):
+ """Docket matching behaves differently depending on court jurisdiction
+ - Federal courts use `docket_number_core`
+ - State courts do not
+ - There are also special cases such as ohioctapp
+
+ Also, test if we can detect when a docket match has a
+ case_name to different than the incoming case_name
+ """
+
+ def setUp(self):
+ self.ariz = CourtFactory(id="ariz")
+ DocketFactory(
+ docket_number="1 CA-CR 23-0297",
+ court=self.ariz,
+ source=Docket.SCRAPER,
+ pacer_case_id=None,
+ )
+ # To test query for multi docket dockets without
+ # a semicolon
+ DocketFactory(
+ docket_number="23-1374 23-1880",
+ court=self.ariz,
+ source=Docket.SCRAPER,
+ pacer_case_id=None,
+ )
+
+ # Need to disambiguate using `appeal_from_str`
+ self.ohioctapp = CourtFactory(id="ohioctapp")
+ self.ohioctapp_dn = "22CA15"
+ DocketFactory(
+ docket_number=self.ohioctapp_dn,
+ appeal_from_str="Pickaway County",
+ case_name="Dietrich v. Dietrich",
+ court=self.ohioctapp,
+ source=Docket.SCRAPER,
+ pacer_case_id=None,
+ )
+ DocketFactory(
+ docket_number=self.ohioctapp_dn,
+ appeal_from_str="Athens County",
+ case_name="State v. Myers",
+ court=self.ohioctapp,
+ source=Docket.SCRAPER,
+ pacer_case_id=None,
+ )
+
+ self.ca2 = CourtFactory(id="ca2", jurisdiction=Court.FEDERAL_APPELLATE)
+ self.ca2_docket = DocketFactory(
+ court=self.ca2,
+ docket_number="10-1039-pr",
+ case_name="Garbutt v. Conway",
+ docket_number_core="10001039",
+ )
+
+ def test_get_existing_docket(self):
+ """Can we get an existing docket if it exists,
+ or None if it doesn't?
+
+ Can we handle special cases like ohioctapp and
+ multi-docket docket numbers without semicolons?
+ """
+ # Return Docket
+ docket = get_existing_docket(self.ariz.id, "1 CA-CR 23-0297")
+ self.assertEqual(docket.docket_number, "1 CA-CR 23-0297")
+
+ docket = get_existing_docket(
+ self.ohioctapp.id, self.ohioctapp_dn, "Athens County"
+ )
+ self.assertEqual(
+ docket.appeal_from_str,
+ "Athens County",
+ "Incorrect docket match for ohioctapp",
+ )
+
+ # Test for OR query with or without semicolons
+ docket = get_existing_docket(self.ariz.id, "23-1374; 23-1880")
+ self.assertEqual(
+ get_existing_docket(self.ariz.id, "23-1374 23-1880").id,
+ docket.id,
+ "should match the same docket",
+ )
+
+ # Return None
+ docket = get_existing_docket(self.ariz.id, "1 CA-CV 23-0297-FC")
+ self.assertIsNone(docket, "Expected None")
+
+ docket = get_existing_docket(
+ self.ohioctapp.id, self.ohioctapp_dn, "Gallia County"
+ )
+ self.assertIsNone(docket, "Expected None, ohioctapp special case")
+
+ def test_different_case_names_detection(self):
+ """Can we detect case names that are too different?"""
+ similar_names = [
+ ("Miller v. Doe", "Miller v. Nelson"),
+ (
+ "IN RE: KIRKLAND LAKE GOLD LTD. SECURITIES LITIGATION",
+ "In Re: Kirkland Lake Gold",
+ ),
+ # Docket 14734478
+ (
+ "State ex rel. AWMS Water Solutions, L.L.C. v. Zehringer",
+ "State ex rel. AWMS Water Solutions, L.L.C. v. Mertz",
+ ),
+ # Docket 61614696
+ (
+ "Fortis Advisors LLC v. Johnson & Johnson, Ethicon, Inc., Alex Gorsky, Ashley McEvoy, Peter Shen and Susan Morano",
+ "Fortis Advisors LLC v. Johnson & Johnson",
+ ),
+ ]
+ different_names = [
+ # Docket 68390253, ohioctapp error
+ ("M.A.N.S.O. Holding, L.L.C. v. Marquette", "State v. Sweeney"),
+ # Docket 68295573, az error
+ ("Van Camp v. Van Camp", "State v. Snyder"),
+ ]
+ for first, second in similar_names:
+ self.assertFalse(
+ case_names_are_too_different(first, second),
+ "Case names should not be marked as too different",
+ )
+
+ for first, second in different_names:
+ self.assertTrue(
+ case_names_are_too_different(first, second),
+ "Case names should be marked as too different",
+ )
+
+ def test_federal_jurisdictions(self):
+ """These courts should follow the flow that uses
+ cl.recap.mergers.find_docket_object and relies on
+ Docket.docket_number_core
+ """
+ docket = update_or_create_docket(
+ "Garbutt v Conway", "", self.ca2, "10-1039", Docket.SCRAPER, False
+ )
+ self.assertEqual(
+ docket, self.ca2_docket, "Should match using docket number core"
+ )
diff --git a/cl/scrapers/utils.py b/cl/scrapers/utils.py
index 221e86b57d..31134ce3d2 100644
--- a/cl/scrapers/utils.py
+++ b/cl/scrapers/utils.py
@@ -1,6 +1,5 @@
import os
import sys
-import traceback
from datetime import date
from typing import Optional, Tuple
from urllib.parse import urljoin
@@ -10,17 +9,23 @@
from asgiref.sync import async_to_sync
from courts_db import find_court_by_id, find_court_ids_by_name
from django.conf import settings
-from django.db.models import QuerySet
+from django.db.models import Q, QuerySet
from juriscraper import AbstractSite
from juriscraper.AbstractSite import logger
from juriscraper.lib.test_utils import MockRequest
from lxml import html
from requests import Response, Session
+from cl.corpus_importer.utils import winnow_case_name
from cl.lib.celery_utils import CeleryThrottle
from cl.lib.decorators import retry
from cl.lib.microservice_utils import microservice
from cl.recap.mergers import find_docket_object
+from cl.scrapers.exceptions import (
+ EmptyFileError,
+ NoDownloadUrlError,
+ UnexpectedContentTypeError,
+)
from cl.scrapers.tasks import extract_recap_pdf
from cl.search.models import Court, Docket, RECAPDocument
@@ -155,32 +160,26 @@ def get_extension(content: bytes) -> str:
def get_binary_content(
download_url: str,
site: AbstractSite,
- headers: dict,
- method: str = "GET",
-) -> Tuple[str, Optional[Response]]:
+) -> bytes | str:
"""Downloads the file, covering a few special cases such as invalid SSL
certificates and empty file errors.
:param download_url: The URL for the item you wish to download.
:param site: Site object used to download data
- :param headers: Headers that might be necessary to download the item.
- :param method: The HTTP method used to get the item, or "LOCAL" to get an
- item during testing
- :return: Two values. The first is a msg indicating any errors encountered.
- If blank, that indicates success. The second value is the response object
- containing the downloaded file.
+
+ :return: The downloaded and cleaned content
+ :raises: NoDownloadUrlError, UnexpectedContentTypeError, EmptyFileError
"""
if not download_url:
- # Occurs when a DeferredList fetcher fails.
- msg = f"NoDownloadUrlError: {download_url}\n{traceback.format_exc()}"
- return msg, None
+ raise NoDownloadUrlError(download_url)
+
# noinspection PyBroadException
- if method == "LOCAL":
+ if site.method == "LOCAL":
+ # "LOCAL" is the method when testing
url = os.path.join(settings.MEDIA_ROOT, download_url)
mr = MockRequest(url=url)
r = mr.get()
- r = follow_redirections(r, requests.Session())
- r.raise_for_status()
+ s = requests.Session()
else:
# some sites require a custom ssl_context, contained in the Site's
# session. However, we can't send a request with both a
@@ -188,6 +187,11 @@ def get_binary_content(
has_cipher = hasattr(site, "cipher")
s = site.request["session"] if has_cipher else requests.session()
+ if site.needs_special_headers:
+ headers = site.request["headers"]
+ else:
+ headers = {"User-Agent": "CourtListener"}
+
# Note that we do a GET even if site.method is POST. This is
# deliberate.
r = s.get(
@@ -200,8 +204,7 @@ def get_binary_content(
# test for empty files (thank you CA1)
if len(r.content) == 0:
- msg = f"EmptyFileError: {download_url}\n{traceback.format_exc()}"
- return msg, None
+ raise EmptyFileError(f"EmptyFileError: '{download_url}'")
# test for expected content type (thanks mont for nil)
if site.expected_content_types:
@@ -214,19 +217,20 @@ def get_binary_content(
content_type in mime.lower()
for mime in site.expected_content_types
)
+
if not m:
- msg = (
- f"UnexpectedContentTypeError: {download_url}\n"
- f'\'"{content_type}" not in {site.expected_content_types}'
- )
- return msg, None
+ court_str = site.court_id.split(".")[-1].split("_")[0]
+ fingerprint = [f"{court_str}-unexpected-content-type"]
+ msg = f"'{download_url}' '{content_type}' not in {site.expected_content_types}"
+ raise UnexpectedContentTypeError(msg, fingerprint=fingerprint)
# test for and follow meta redirects
r = follow_redirections(r, s)
r.raise_for_status()
- # Success!
- return "", r
+ content = site.cleanup_content(r.content)
+
+ return content
def signal_handler(signal, frame):
@@ -285,70 +289,180 @@ def extract_recap_documents(
sys.stdout.flush()
+def get_existing_docket(
+ court_id: str, docket_number: str, appeal_from_str: str = ""
+) -> Docket | None:
+ """Look for an existing docket for a given court_id and docket number
+
+ recap.mergers.find_docket_object prioritizes lookups by docket_number_core
+ which is designed for federal / PACER sources. This function is rough
+ equivalent with lookup priorities inverted, intended to be used with
+ scraped sources
+
+ Even when make_docket_number_core returns an empty string for most state
+ courts that we scrape, it causes mismatches in courts like `az`, where
+ 2 different dockets like '1 CA-CR 23-0297' and '1 CA-CV 23-0297-FC'
+ have the same core number
+
+ Examples of docket numbers do not map to a docket_number_core
+ (fldistctapp '5D2023-0888'), (ohioctapp, '22CA15')
+
+ :param court_id: the court id
+ :param docket_number: the docket number
+ :param appeal_from_str: useful for disambiguating `ohioctapp` dockets,
+ this is the "lower_courts" returned juriscraper field
+
+ :return: Docket if find a match, None if we don't
+ """
+ # Avoid lookups by blank docket number
+ if not docket_number.strip():
+ return
+
+ # delete semicolons only for the lookup, for back compatibility
+ # with juriscraper string formatting
+ # https://github.com/freelawproject/juriscraper/pull/1166
+ lookup = Q(court_id=court_id) & (
+ Q(docket_number=docket_number.replace(";", ""))
+ | Q(docket_number=docket_number)
+ )
+
+ # Special case where docket numbers are the same and repeated
+ # across districts, but can be disambiguated using the lower court
+ if court_id == "ohioctapp" and appeal_from_str:
+ lookup = lookup & Q(appeal_from_str=appeal_from_str)
+
+ queryset = Docket.objects.filter(lookup)
+ count = queryset.count()
+ if count == 1:
+ return queryset[0]
+ if count > 1:
+ logger.error(
+ "%s: more than 1 docket match for docket number '%s'",
+ court_id,
+ docket_number,
+ )
+ return queryset[0]
+
+
+def case_names_are_too_different(
+ first: str, second: str, threshold: float = 0.5
+) -> bool:
+ """Compares 2 case names' words as a similitude measure
+ Useful to raise a warning when updating a docket and names are found
+ to be too different
+
+ :param first: first case name
+ :param second: second case name
+ :param threshold: minimum percentage of words in common
+
+ :return: True if case names are too different according to the threshold;
+ False if names are similar
+ """
+ new_parts = winnow_case_name(first.lower())
+ old_parts = winnow_case_name(second.lower())
+ # or 1 to prevent 0 lenght minimum
+ denominator = min(len(old_parts), len(new_parts)) or 1
+ return len(new_parts.intersection(old_parts)) / denominator < threshold
+
+
def update_or_create_docket(
case_name: str,
case_name_short: str,
- court_id: str,
+ court: Court,
docket_number: str,
source: int,
+ from_harvard: bool,
blocked: bool = False,
case_name_full: str = "",
date_blocked: date | None = None,
date_argued: date | None = None,
ia_needs_upload: bool | None = None,
+ appeal_from_str: str = "",
) -> Docket:
"""Look for an existing Docket and update it or create a new one if it's
not found.
:param case_name: The docket case_name.
:param case_name_short: The docket case_name_short
- :param court_id: The court id the docket belongs to.
+ :param court: The court objects the docket belongs to
:param docket_number: The docket number.
:param source: The docket source.
+ :param from_harvard: True when this function is called from the
+ Harvard importer; the Harvard data is considered
+ more trustable and should overwrite an existing docket's data
+ Should be False when called from scrapers.
:param blocked: If the docket should be blocked, default False.
:param case_name_full: The docket case_name_full.
:param date_blocked: The docket date_blocked if it's blocked.
:param date_argued: The docket date_argued if it's an oral argument.
:param ia_needs_upload: If the docket needs upload to IA, default None.
+ :param appeal_from_str: Name (not standardized id) of the lower level court.
:return: The docket.
"""
-
docket_fields = {
"case_name": case_name,
"case_name_short": case_name_short,
"case_name_full": case_name_full,
"blocked": blocked,
"ia_needs_upload": ia_needs_upload,
+ "appeal_from_str": appeal_from_str,
"date_blocked": date_blocked,
+ "date_argued": date_argued,
}
- docket = async_to_sync(find_docket_object)(court_id, None, docket_number)
- if docket.pk:
- # Update the existing docket with the new values
- docket.add_opinions_source(source)
-
- # Prevent overwriting Docket.date_argued if it exists
- if date_argued:
- if docket.date_argued and date_argued != docket.date_argued:
- logger.error(
- "Docket %s already has a date_argued %s, different than new date %s",
- docket.pk,
- docket.date_argued,
- date_argued,
- )
- else:
- docket.date_argued = date_argued
+ court_id = court.pk
+ uses_docket_number_core = court.jurisdiction in Court.FEDERAL_JURISDICTIONS
- for field, value in docket_fields.items():
- setattr(docket, field, value)
+ if from_harvard or uses_docket_number_core:
+ docket = async_to_sync(find_docket_object)(
+ court_id, None, docket_number, None, None, None
+ )
else:
- # Create a new docket with docket_fields and additional fields
- docket = Docket(
+ docket = get_existing_docket(court_id, docket_number, appeal_from_str)
+
+ if not docket or not docket.pk:
+ return Docket(
**docket_fields,
- date_argued=date_argued,
source=source,
docket_number=docket_number,
court_id=court_id,
)
+ # Update the existing docket with the new values
+ docket.add_opinions_source(source)
+
+ for field, value in docket_fields.items():
+ # do not use blanket `if not value:`, since
+ # blocked and ia_needs_upload are booleans and would be skipped
+ if value is None or value == "":
+ continue
+
+ if (
+ not from_harvard
+ and field == "case_name"
+ and getattr(docket, field)
+ and getattr(docket, field) != value
+ ):
+ # Safeguard to catch possible docket mismatches, check that they
+ # have at least 50% of words in common
+ if case_names_are_too_different(value, docket.case_name, 0.5):
+ logger.error(
+ "New case_name '%s' looks too different from old '%s'. Court %s. Docket %s",
+ value,
+ docket.case_name,
+ court_id,
+ docket.pk,
+ )
+ continue
+
+ # Most times, we find updated values for case_name that may
+ # be a longer form than what we currently have, which we can
+ # take advantage of to populate case_name_full
+ if not getattr(docket, "case_name_full") and len(value) > len(
+ getattr(docket, field)
+ ):
+ setattr(docket, "case_name_full", value)
+ else:
+ setattr(docket, field, value)
+
return docket
diff --git a/cl/search/admin.py b/cl/search/admin.py
index 00e7776dee..b1e87d995b 100644
--- a/cl/search/admin.py
+++ b/cl/search/admin.py
@@ -285,6 +285,7 @@ class DocketAdmin(CursorPaginatorAdmin):
"referred_to",
"originating_court_information",
"idb_data",
+ "parent_docket",
)
def save_model(
diff --git a/cl/search/api_utils.py b/cl/search/api_utils.py
index dbd1d3e127..8f2b11b74b 100644
--- a/cl/search/api_utils.py
+++ b/cl/search/api_utils.py
@@ -23,7 +23,7 @@
)
from cl.lib.scorched_utils import ExtraSolrInterface
from cl.lib.utils import map_to_docket_entry_sorting
-from cl.search.constants import SEARCH_HL_TAG
+from cl.search.constants import SEARCH_HL_TAG, cardinality_query_unique_ids
from cl.search.documents import (
AudioDocument,
DocketDocument,
@@ -289,13 +289,13 @@ class CursorESList:
well as the pagination logic for cursor-based pagination.
"""
- cardinality_query = {
- SEARCH_TYPES.RECAP: ("docket_id", DocketDocument),
- SEARCH_TYPES.DOCKETS: ("docket_id", DocketDocument),
- SEARCH_TYPES.RECAP_DOCUMENT: ("id", DocketDocument),
- SEARCH_TYPES.OPINION: ("cluster_id", OpinionClusterDocument),
- SEARCH_TYPES.PEOPLE: ("id", PersonDocument),
- SEARCH_TYPES.ORAL_ARGUMENT: ("id", AudioDocument),
+ cardinality_base_document = {
+ SEARCH_TYPES.RECAP: DocketDocument,
+ SEARCH_TYPES.DOCKETS: DocketDocument,
+ SEARCH_TYPES.RECAP_DOCUMENT: DocketDocument,
+ SEARCH_TYPES.OPINION: OpinionClusterDocument,
+ SEARCH_TYPES.PEOPLE: PersonDocument,
+ SEARCH_TYPES.ORAL_ARGUMENT: AudioDocument,
}
def __init__(
@@ -350,23 +350,27 @@ def get_paginated_results(
# Cardinality query parameters
query = Q(self.main_query.to_dict(count=True)["query"])
- unique_field, search_document = self.cardinality_query[
+ unique_field = cardinality_query_unique_ids[self.clean_data["type"]]
+ search_document = self.cardinality_base_document[
self.clean_data["type"]
]
- base_search = search_document.search()
+ main_count_query = search_document.search().query(query)
cardinality_query = build_cardinality_count(
- base_search, query, unique_field
+ main_count_query, unique_field
)
# Build a cardinality query to count child documents.
child_cardinality_query = None
child_cardinality_count_response = None
if self.child_docs_query:
- child_unique_field, _ = self.cardinality_query[
+ child_unique_field = cardinality_query_unique_ids[
SEARCH_TYPES.RECAP_DOCUMENT
]
+ child_count_query = search_document.search().query(
+ self.child_docs_query
+ )
child_cardinality_query = build_cardinality_count(
- base_search, self.child_docs_query, child_unique_field
+ child_count_query, child_unique_field
)
try:
multi_search = MultiSearch()
@@ -476,8 +480,7 @@ def get_api_query_sorting(self):
default_unique_order = {
"type": self.clean_data["type"],
}
-
- unique_field, _ = self.cardinality_query[self.clean_data["type"]]
+ unique_field = cardinality_query_unique_ids[self.clean_data["type"]]
# Use a document unique field as a unique sorting key for the current
# search type.
default_unique_order.update(
diff --git a/cl/search/constants.py b/cl/search/constants.py
index 78cf3dd4de..a210beb801 100644
--- a/cl/search/constants.py
+++ b/cl/search/constants.py
@@ -102,7 +102,6 @@
"citation",
"judge",
"caseNameFull",
- "caseName",
"status",
"suitNature",
"attorney",
@@ -226,12 +225,19 @@
}
recap_boosts_es = {
# Docket fields
- "caseName": 4.0,
+ "caseName.exact": 4.0,
"docketNumber": 3.0,
# RECAPDocument fields:
"description": 2.0,
}
recap_boosts_pf = {"text": 3.0, "caseName": 3.0, "description": 3.0}
+opinion_boosts_es = {
+ "text": 1.0,
+ "type": 1.0,
+ # Cluster fields
+ "caseName.exact": 4.0,
+ "docketNumber": 2.0,
+}
BOOSTS: Dict[str, Dict[str, Dict[str, float]]] = {
"qf": {
SEARCH_TYPES.OPINION: {
@@ -246,7 +252,7 @@
SEARCH_TYPES.RECAP_DOCUMENT: recap_boosts_qf,
SEARCH_TYPES.ORAL_ARGUMENT: {
"text": 1.0,
- "caseName": 4.0,
+ "caseName.exact": 4.0,
"docketNumber": 2.0,
},
SEARCH_TYPES.PEOPLE: {
@@ -264,6 +270,7 @@
SEARCH_TYPES.RECAP: recap_boosts_es,
SEARCH_TYPES.DOCKETS: recap_boosts_es,
SEARCH_TYPES.RECAP_DOCUMENT: recap_boosts_es,
+ SEARCH_TYPES.OPINION: opinion_boosts_es,
},
# Phrase-based boosts.
"pf": {
@@ -293,3 +300,13 @@
Opinion.ON_MOTION_TO_STRIKE: "on-motion-to-strike",
Opinion.TRIAL_COURT: "trial-court-document",
}
+
+cardinality_query_unique_ids = {
+ SEARCH_TYPES.RECAP: "docket_id",
+ SEARCH_TYPES.DOCKETS: "docket_id",
+ SEARCH_TYPES.RECAP_DOCUMENT: "id",
+ SEARCH_TYPES.OPINION: "cluster_id",
+ SEARCH_TYPES.PEOPLE: "id",
+ SEARCH_TYPES.ORAL_ARGUMENT: "id",
+ SEARCH_TYPES.PARENTHETICAL: "id",
+}
diff --git a/cl/search/documents.py b/cl/search/documents.py
index d7b18f9472..e059b58f22 100644
--- a/cl/search/documents.py
+++ b/cl/search/documents.py
@@ -13,7 +13,7 @@
from cl.lib.command_utils import logger
from cl.lib.elasticsearch_utils import build_es_base_query
from cl.lib.fields import JoinField, PercolatorField
-from cl.lib.search_index_utils import null_map
+from cl.lib.search_index_utils import get_parties_from_case_name, null_map
from cl.lib.utils import deepgetattr
from cl.people_db.models import (
Attorney,
@@ -1231,6 +1231,14 @@ def prepare_parties(self, instance):
out["party_id"].add(pk)
out["party"].add(name)
+ if not out["party"]:
+ # Get party from docket case_name if no normalized parties are
+ # available.
+ party_from_case_name = get_parties_from_case_name(
+ instance.case_name
+ )
+ out["party"] = party_from_case_name if party_from_case_name else []
+
# Extract only required attorney values.
atty_values = (
Attorney.objects.filter(roles__docket=instance)
diff --git a/cl/search/factories.py b/cl/search/factories.py
index cdaaf711ad..c0fd5baea1 100644
--- a/cl/search/factories.py
+++ b/cl/search/factories.py
@@ -279,7 +279,7 @@ class Meta:
pacer_case_id = Faker("pyint", min_value=100_000, max_value=400_000)
docket_number = Faker("federal_district_docket_number")
slug = Faker("slug")
- filepath_local = FileField(filename=None)
+ filepath_local = FileField(filename="docket.xml")
date_argued = Faker("date_object")
diff --git a/cl/search/fixtures/functest_opinions.json b/cl/search/fixtures/functest_opinions.json
index e4fa89a260..f1e6f2da44 100644
--- a/cl/search/fixtures/functest_opinions.json
+++ b/cl/search/fixtures/functest_opinions.json
@@ -64,7 +64,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 10
@@ -134,7 +135,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 11
@@ -184,7 +186,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 12
@@ -254,7 +257,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 12
diff --git a/cl/search/fixtures/opinions-issue-412.json b/cl/search/fixtures/opinions-issue-412.json
index ca6ac33971..fa7d716ccb 100644
--- a/cl/search/fixtures/opinions-issue-412.json
+++ b/cl/search/fixtures/opinions-issue-412.json
@@ -64,7 +64,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 10
@@ -134,7 +135,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 11
diff --git a/cl/search/fixtures/test_objects_query_counts.json b/cl/search/fixtures/test_objects_query_counts.json
index aa909b2fb2..ca69a08ccc 100644
--- a/cl/search/fixtures/test_objects_query_counts.json
+++ b/cl/search/fixtures/test_objects_query_counts.json
@@ -300,7 +300,8 @@
"date_created":"2015-08-15T14:10:56.801Z",
"html_lawbox":"",
"per_curiam":false,
- "type":"020lead"
+ "type":"020lead",
+ "ordering_key": null
},
"model":"search.opinion",
"pk":1
@@ -324,7 +325,8 @@
"date_created":"2015-08-15T14:10:56.801Z",
"html_lawbox":"",
"per_curiam":false,
- "type":"010combined"
+ "type":"010combined",
+ "ordering_key": null
},
"model":"search.opinion",
"pk":2
@@ -348,7 +350,8 @@
"date_created":"2015-08-15T14:10:56.801Z",
"html_lawbox":"",
"per_curiam":false,
- "type":"010combined"
+ "type":"010combined",
+ "ordering_key": null
},
"model":"search.opinion",
"pk":3
@@ -371,7 +374,8 @@
"date_created":"2015-08-15T14:10:56.801Z",
"html_lawbox":"",
"per_curiam":false,
- "type":"010combined"
+ "type":"010combined",
+ "ordering_key": null
},
"model":"search.opinion",
"pk":4
@@ -395,7 +399,8 @@
"date_created":"2015-08-15T14:10:56.801Z",
"html_lawbox":"",
"per_curiam":false,
- "type":"010combined"
+ "type":"010combined",
+ "ordering_key": null
},
"model":"search.opinion",
"pk":5
@@ -418,7 +423,8 @@
"date_created":"2015-08-15T14:10:56.801Z",
"html_lawbox":"",
"per_curiam":false,
- "type":"010combined"
+ "type":"010combined",
+ "ordering_key": null
},
"model":"search.opinion",
"pk":6
diff --git a/cl/search/fixtures/test_objects_search.json b/cl/search/fixtures/test_objects_search.json
index 2255c7edcf..66c9915581 100644
--- a/cl/search/fixtures/test_objects_search.json
+++ b/cl/search/fixtures/test_objects_search.json
@@ -239,7 +239,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "020lead"
+ "type": "020lead",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 1
@@ -261,7 +262,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 2
@@ -283,7 +285,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 3
@@ -305,7 +308,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 4
@@ -327,7 +331,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 5
@@ -349,7 +354,8 @@
"date_created": "2015-08-15T14:10:56.801Z",
"html_lawbox": "",
"per_curiam": false,
- "type": "010combined"
+ "type": "010combined",
+ "ordering_key": null
},
"model": "search.opinion",
"pk": 6
diff --git a/cl/search/management/commands/sweep_indexer.py b/cl/search/management/commands/sweep_indexer.py
index 02da4e3687..4cc7b0bc4f 100644
--- a/cl/search/management/commands/sweep_indexer.py
+++ b/cl/search/management/commands/sweep_indexer.py
@@ -27,14 +27,7 @@
)
from cl.search.types import ESDocumentClassType
-supported_models = [
- "audio.Audio",
- "people_db.Person",
- "search.OpinionCluster",
- "search.Opinion",
- "search.Docket",
- "search.RECAPDocument",
-]
+supported_models = settings.ELASTICSEARCH_SWEEP_INDEXER_MODELS # type: ignore
r = get_redis_interface("CACHE")
diff --git a/cl/search/migrations/0001_initial.py b/cl/search/migrations/0001_initial.py
index a40ffad040..1e96896b35 100644
--- a/cl/search/migrations/0001_initial.py
+++ b/cl/search/migrations/0001_initial.py
@@ -419,17 +419,17 @@ class Migration(migrations.Migration):
name='recapdocument',
unique_together={('docket_entry', 'document_number', 'attachment_number')},
),
- migrations.AlterIndexTogether(
- name='recapdocument',
- index_together={('document_type', 'document_number', 'attachment_number')},
+ migrations.AddIndex(
+ model_name='recapdocument',
+ index=models.Index(fields=['document_type', 'document_number', 'attachment_number'], name='search_recapdocument_document_type_303cccac79571217_idx'),
),
migrations.AlterUniqueTogether(
name='opinionscited',
unique_together={('citing_opinion', 'cited_opinion')},
),
- migrations.AlterIndexTogether(
- name='docketentry',
- index_together={('recap_sequence_number', 'entry_number')},
+ migrations.AddIndex(
+ model_name='docketentry',
+ index=models.Index(fields=['recap_sequence_number', 'entry_number'], name='search_docketentry_recap_sequence_number_1c82e51988e2d89f_idx')
),
migrations.AddIndex(
model_name='docket',
@@ -439,16 +439,17 @@ class Migration(migrations.Migration):
name='docket',
unique_together={('docket_number', 'pacer_case_id', 'court')},
),
- migrations.AlterIndexTogether(
- name='docket',
- index_together={('ia_upload_failure_count', 'ia_needs_upload', 'ia_date_first_change')},
- ),
migrations.AlterUniqueTogether(
name='citation',
unique_together={('cluster', 'volume', 'reporter', 'page')},
),
- migrations.AlterIndexTogether(
- name='citation',
- index_together={('volume', 'reporter', 'page'), ('volume', 'reporter')},
+ migrations.AddIndex(
+ model_name='citation',
+ index=models.Index(fields=['volume', 'reporter', 'page'], name='search_citation_volume_ae340b5b02e8912_idx')
),
+ migrations.AddIndex(
+ model_name='citation',
+ index=models.Index(fields=['volume', 'reporter'], name='search_citation_volume_251bc1d270a8abee_idx')
+ ),
+
]
diff --git a/cl/search/migrations/0006_delete_unused_indexes.py b/cl/search/migrations/0006_delete_unused_indexes.py
index 0ee038ea56..40a5de2a8e 100644
--- a/cl/search/migrations/0006_delete_unused_indexes.py
+++ b/cl/search/migrations/0006_delete_unused_indexes.py
@@ -144,9 +144,5 @@ class Migration(migrations.Migration):
help_text="The moment when this item first changed and was marked as needing an upload. Used for determining when to upload an item.",
null=True,
),
- ),
- migrations.AlterIndexTogether(
- name="docket",
- index_together=set(),
- ),
+ )
]
diff --git a/cl/search/migrations/0017_remove_bankruptcyinformation_update_or_delete_snapshot_update_and_more.py b/cl/search/migrations/0017_remove_bankruptcyinformation_update_or_delete_snapshot_update_and_more.py
index d083358250..04f5ed3513 100644
--- a/cl/search/migrations/0017_remove_bankruptcyinformation_update_or_delete_snapshot_update_and_more.py
+++ b/cl/search/migrations/0017_remove_bankruptcyinformation_update_or_delete_snapshot_update_and_more.py
@@ -57,22 +57,22 @@ class Migration(migrations.Migration):
),
migrations.RenameIndex(
model_name="citation",
- new_name="search_cita_volume_464334_idx",
+ new_name="search_citation_volume_251bc1d270a8abee_idx",
old_fields=("volume", "reporter"),
),
migrations.RenameIndex(
model_name="citation",
- new_name="search_cita_volume_92c344_idx",
+ new_name="search_citation_volume_ae340b5b02e8912_idx",
old_fields=("volume", "reporter", "page"),
),
migrations.RenameIndex(
model_name="docketentry",
- new_name="search_dock_recap_s_306ab9_idx",
+ new_name="search_docketentry_recap_sequence_number_1c82e51988e2d89f_idx",
old_fields=("recap_sequence_number", "entry_number"),
),
migrations.RenameIndex(
model_name="recapdocument",
- new_name="search_reca_documen_cc5acd_idx",
+ new_name="search_recapdocument_document_type_303cccac79571217_idx",
old_fields=(
"document_type",
"document_number",
diff --git a/cl/search/migrations/0017_remove_bankruptcyinformation_update_or_delete_snapshot_update_and_more.sql b/cl/search/migrations/0017_remove_bankruptcyinformation_update_or_delete_snapshot_update_and_more.sql
index 1099c5fdbc..f9c13fb928 100644
--- a/cl/search/migrations/0017_remove_bankruptcyinformation_update_or_delete_snapshot_update_and_more.sql
+++ b/cl/search/migrations/0017_remove_bankruptcyinformation_update_or_delete_snapshot_update_and_more.sql
@@ -44,21 +44,21 @@ DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_8a108 ON "sear
--
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_c9dd9 ON "search_tag";
--
--- Rename unnamed index for ('volume', 'reporter') on citation to search_cita_volume_464334_idx
+-- Rename unnamed index for ('volume', 'reporter') on citation to search_citation_volume_251bc1d270a8abee_idx
--
-ALTER INDEX "search_citation_volume_251bc1d270a8abee_idx" RENAME TO "search_cita_volume_464334_idx";
+-- (no-op)
--
--- Rename unnamed index for ('volume', 'reporter', 'page') on citation to search_cita_volume_92c344_idx
+-- Rename unnamed index for ('volume', 'reporter', 'page') on citation to search_citation_volume_ae340b5b02e8912_idx
--
-ALTER INDEX "search_citation_volume_ae340b5b02e8912_idx" RENAME TO "search_cita_volume_92c344_idx";
+-- (no-op)
--
--- Rename unnamed index for ('recap_sequence_number', 'entry_number') on docketentry to search_dock_recap_s_306ab9_idx
+-- Rename unnamed index for ('recap_sequence_number', 'entry_number') on docketentry to search_docketentry_recap_sequence_number_1c82e51988e2d89f_idx
--
-ALTER INDEX "search_docketentry_recap_sequence_number_1c82e51988e2d89f_idx" RENAME TO "search_dock_recap_s_306ab9_idx";
+-- (no-op)
--
--- Rename unnamed index for ('document_type', 'document_number', 'attachment_number') on recapdocument to search_reca_documen_cc5acd_idx
+-- Rename unnamed index for ('document_type', 'document_number', 'attachment_number') on recapdocument to search_recapdocument_document_type_303cccac79571217_idx
--
-ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO "recapdocument to search_reca_documen_cc5acd_idx";
+-- (no-op)
--
-- Create trigger update_or_delete_snapshot_update on model bankruptcyinformation
--
@@ -88,7 +88,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_17e86()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -104,13 +104,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_17e86 ON "search_bankruptcyinformation";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_17e86
AFTER UPDATE ON "search_bankruptcyinformation"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."docket_id" IS DISTINCT FROM (NEW."docket_id") OR OLD."date_converted" IS DISTINCT FROM (NEW."date_converted") OR OLD."date_last_to_file_claims" IS DISTINCT FROM (NEW."date_last_to_file_claims") OR OLD."date_last_to_file_govt" IS DISTINCT FROM (NEW."date_last_to_file_govt") OR OLD."date_debtor_dismissed" IS DISTINCT FROM (NEW."date_debtor_dismissed") OR OLD."chapter" IS DISTINCT FROM (NEW."chapter") OR OLD."trustee_str" IS DISTINCT FROM (NEW."trustee_str"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_17e86();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_17e86 ON "search_bankruptcyinformation" IS '85d1a7878d466326c90c68b401f107b1158c2796';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model claim
--
@@ -140,7 +140,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_bb32f()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -156,13 +156,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_bb32f ON "search_claim";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_bb32f
AFTER UPDATE ON "search_claim"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."docket_id" IS DISTINCT FROM (NEW."docket_id") OR OLD."date_claim_modified" IS DISTINCT FROM (NEW."date_claim_modified") OR OLD."date_original_entered" IS DISTINCT FROM (NEW."date_original_entered") OR OLD."date_original_filed" IS DISTINCT FROM (NEW."date_original_filed") OR OLD."date_last_amendment_entered" IS DISTINCT FROM (NEW."date_last_amendment_entered") OR OLD."date_last_amendment_filed" IS DISTINCT FROM (NEW."date_last_amendment_filed") OR OLD."claim_number" IS DISTINCT FROM (NEW."claim_number") OR OLD."creditor_details" IS DISTINCT FROM (NEW."creditor_details") OR OLD."creditor_id" IS DISTINCT FROM (NEW."creditor_id") OR OLD."status" IS DISTINCT FROM (NEW."status") OR OLD."entered_by" IS DISTINCT FROM (NEW."entered_by") OR OLD."filed_by" IS DISTINCT FROM (NEW."filed_by") OR OLD."amount_claimed" IS DISTINCT FROM (NEW."amount_claimed") OR OLD."unsecured_claimed" IS DISTINCT FROM (NEW."unsecured_claimed") OR OLD."secured_claimed" IS DISTINCT FROM (NEW."secured_claimed") OR OLD."priority_claimed" IS DISTINCT FROM (NEW."priority_claimed") OR OLD."description" IS DISTINCT FROM (NEW."description") OR OLD."remarks" IS DISTINCT FROM (NEW."remarks"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_bb32f();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_bb32f ON "search_claim" IS '5a3fde0d49f7f04afe30f9151a8b3535710ec1a0';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model claimhistory
--
@@ -192,7 +192,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_137a5()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -208,13 +208,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_137a5 ON "search_claimhistory";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_137a5
AFTER UPDATE ON "search_claimhistory"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."sha1" IS DISTINCT FROM (NEW."sha1") OR OLD."page_count" IS DISTINCT FROM (NEW."page_count") OR OLD."file_size" IS DISTINCT FROM (NEW."file_size") OR OLD."filepath_local" IS DISTINCT FROM (NEW."filepath_local") OR OLD."filepath_ia" IS DISTINCT FROM (NEW."filepath_ia") OR OLD."ia_upload_failure_count" IS DISTINCT FROM (NEW."ia_upload_failure_count") OR OLD."thumbnail" IS DISTINCT FROM (NEW."thumbnail") OR OLD."thumbnail_status" IS DISTINCT FROM (NEW."thumbnail_status") OR OLD."plain_text" IS DISTINCT FROM (NEW."plain_text") OR OLD."ocr_status" IS DISTINCT FROM (NEW."ocr_status") OR OLD."date_upload" IS DISTINCT FROM (NEW."date_upload") OR OLD."document_number" IS DISTINCT FROM (NEW."document_number") OR OLD."attachment_number" IS DISTINCT FROM (NEW."attachment_number") OR OLD."pacer_doc_id" IS DISTINCT FROM (NEW."pacer_doc_id") OR OLD."is_available" IS DISTINCT FROM (NEW."is_available") OR OLD."is_free_on_pacer" IS DISTINCT FROM (NEW."is_free_on_pacer") OR OLD."is_sealed" IS DISTINCT FROM (NEW."is_sealed") OR OLD."claim_id" IS DISTINCT FROM (NEW."claim_id") OR OLD."date_filed" IS DISTINCT FROM (NEW."date_filed") OR OLD."claim_document_type" IS DISTINCT FROM (NEW."claim_document_type") OR OLD."description" IS DISTINCT FROM (NEW."description") OR OLD."claim_doc_id" IS DISTINCT FROM (NEW."claim_doc_id") OR OLD."pacer_dm_id" IS DISTINCT FROM (NEW."pacer_dm_id") OR OLD."pacer_case_id" IS DISTINCT FROM (NEW."pacer_case_id"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_137a5();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_137a5 ON "search_claimhistory" IS 'c4f2a33aa09534f0db6c38a62b0c4e2d656d1db0';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model court
--
@@ -244,7 +244,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_c94ab()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -260,13 +260,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_c94ab ON "search_court";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_c94ab
AFTER UPDATE ON "search_court"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."pacer_court_id" IS DISTINCT FROM (NEW."pacer_court_id") OR OLD."pacer_has_rss_feed" IS DISTINCT FROM (NEW."pacer_has_rss_feed") OR OLD."pacer_rss_entry_types" IS DISTINCT FROM (NEW."pacer_rss_entry_types") OR OLD."date_last_pacer_contact" IS DISTINCT FROM (NEW."date_last_pacer_contact") OR OLD."fjc_court_id" IS DISTINCT FROM (NEW."fjc_court_id") OR OLD."in_use" IS DISTINCT FROM (NEW."in_use") OR OLD."has_opinion_scraper" IS DISTINCT FROM (NEW."has_opinion_scraper") OR OLD."has_oral_argument_scraper" IS DISTINCT FROM (NEW."has_oral_argument_scraper") OR OLD."position" IS DISTINCT FROM (NEW."position") OR OLD."citation_string" IS DISTINCT FROM (NEW."citation_string") OR OLD."short_name" IS DISTINCT FROM (NEW."short_name") OR OLD."full_name" IS DISTINCT FROM (NEW."full_name") OR OLD."url" IS DISTINCT FROM (NEW."url") OR OLD."start_date" IS DISTINCT FROM (NEW."start_date") OR OLD."end_date" IS DISTINCT FROM (NEW."end_date") OR OLD."jurisdiction" IS DISTINCT FROM (NEW."jurisdiction") OR OLD."notes" IS DISTINCT FROM (NEW."notes"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_c94ab();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_c94ab ON "search_court" IS '3d7ee4371f809a112d0ca08ebac797bfe18e404d';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model docket
--
@@ -296,7 +296,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_7e039()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -312,13 +312,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_7e039 ON "search_docket";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_7e039
AFTER UPDATE ON "search_docket"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."source" IS DISTINCT FROM (NEW."source") OR OLD."court_id" IS DISTINCT FROM (NEW."court_id") OR OLD."appeal_from_id" IS DISTINCT FROM (NEW."appeal_from_id") OR OLD."appeal_from_str" IS DISTINCT FROM (NEW."appeal_from_str") OR OLD."originating_court_information_id" IS DISTINCT FROM (NEW."originating_court_information_id") OR OLD."idb_data_id" IS DISTINCT FROM (NEW."idb_data_id") OR OLD."assigned_to_id" IS DISTINCT FROM (NEW."assigned_to_id") OR OLD."assigned_to_str" IS DISTINCT FROM (NEW."assigned_to_str") OR OLD."referred_to_id" IS DISTINCT FROM (NEW."referred_to_id") OR OLD."referred_to_str" IS DISTINCT FROM (NEW."referred_to_str") OR OLD."panel_str" IS DISTINCT FROM (NEW."panel_str") OR OLD."date_last_index" IS DISTINCT FROM (NEW."date_last_index") OR OLD."date_cert_granted" IS DISTINCT FROM (NEW."date_cert_granted") OR OLD."date_cert_denied" IS DISTINCT FROM (NEW."date_cert_denied") OR OLD."date_argued" IS DISTINCT FROM (NEW."date_argued") OR OLD."date_reargued" IS DISTINCT FROM (NEW."date_reargued") OR OLD."date_reargument_denied" IS DISTINCT FROM (NEW."date_reargument_denied") OR OLD."date_filed" IS DISTINCT FROM (NEW."date_filed") OR OLD."date_terminated" IS DISTINCT FROM (NEW."date_terminated") OR OLD."date_last_filing" IS DISTINCT FROM (NEW."date_last_filing") OR OLD."case_name_short" IS DISTINCT FROM (NEW."case_name_short") OR OLD."case_name" IS DISTINCT FROM (NEW."case_name") OR OLD."case_name_full" IS DISTINCT FROM (NEW."case_name_full") OR OLD."slug" IS DISTINCT FROM (NEW."slug") OR OLD."docket_number" IS DISTINCT FROM (NEW."docket_number") OR OLD."docket_number_core" IS DISTINCT FROM (NEW."docket_number_core") OR OLD."pacer_case_id" IS DISTINCT FROM (NEW."pacer_case_id") OR OLD."cause" IS DISTINCT FROM (NEW."cause") OR OLD."nature_of_suit" IS DISTINCT FROM (NEW."nature_of_suit") OR OLD."jury_demand" IS DISTINCT FROM (NEW."jury_demand") OR OLD."jurisdiction_type" IS DISTINCT FROM (NEW."jurisdiction_type") OR OLD."appellate_fee_status" IS DISTINCT FROM (NEW."appellate_fee_status") OR OLD."appellate_case_type_information" IS DISTINCT FROM (NEW."appellate_case_type_information") OR OLD."mdl_status" IS DISTINCT FROM (NEW."mdl_status") OR OLD."filepath_local" IS DISTINCT FROM (NEW."filepath_local") OR OLD."filepath_ia" IS DISTINCT FROM (NEW."filepath_ia") OR OLD."filepath_ia_json" IS DISTINCT FROM (NEW."filepath_ia_json") OR OLD."ia_upload_failure_count" IS DISTINCT FROM (NEW."ia_upload_failure_count") OR OLD."ia_needs_upload" IS DISTINCT FROM (NEW."ia_needs_upload") OR OLD."ia_date_first_change" IS DISTINCT FROM (NEW."ia_date_first_change") OR OLD."date_blocked" IS DISTINCT FROM (NEW."date_blocked") OR OLD."blocked" IS DISTINCT FROM (NEW."blocked"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_7e039();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_7e039 ON "search_docket" IS 'cab7d35a7309b21c85f837b8a6c4759febe46fd8';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model docketentry
--
@@ -348,7 +348,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_46e1e()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -364,13 +364,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_46e1e ON "search_docketentry";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_46e1e
AFTER UPDATE ON "search_docketentry"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."docket_id" IS DISTINCT FROM (NEW."docket_id") OR OLD."date_filed" IS DISTINCT FROM (NEW."date_filed") OR OLD."time_filed" IS DISTINCT FROM (NEW."time_filed") OR OLD."entry_number" IS DISTINCT FROM (NEW."entry_number") OR OLD."recap_sequence_number" IS DISTINCT FROM (NEW."recap_sequence_number") OR OLD."pacer_sequence_number" IS DISTINCT FROM (NEW."pacer_sequence_number") OR OLD."description" IS DISTINCT FROM (NEW."description"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_46e1e();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_46e1e ON "search_docketentry" IS '2330fe784864bcc2d76ebe1d4a07e7819fa8de38';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model opinion
--
@@ -400,7 +400,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_67ecd()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -416,13 +416,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_67ecd ON "search_opinion";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_67ecd
AFTER UPDATE ON "search_opinion"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."cluster_id" IS DISTINCT FROM (NEW."cluster_id") OR OLD."author_id" IS DISTINCT FROM (NEW."author_id") OR OLD."author_str" IS DISTINCT FROM (NEW."author_str") OR OLD."per_curiam" IS DISTINCT FROM (NEW."per_curiam") OR OLD."joined_by_str" IS DISTINCT FROM (NEW."joined_by_str") OR OLD."type" IS DISTINCT FROM (NEW."type") OR OLD."sha1" IS DISTINCT FROM (NEW."sha1") OR OLD."page_count" IS DISTINCT FROM (NEW."page_count") OR OLD."download_url" IS DISTINCT FROM (NEW."download_url") OR OLD."local_path" IS DISTINCT FROM (NEW."local_path") OR OLD."plain_text" IS DISTINCT FROM (NEW."plain_text") OR OLD."html" IS DISTINCT FROM (NEW."html") OR OLD."html_lawbox" IS DISTINCT FROM (NEW."html_lawbox") OR OLD."html_columbia" IS DISTINCT FROM (NEW."html_columbia") OR OLD."html_anon_2020" IS DISTINCT FROM (NEW."html_anon_2020") OR OLD."xml_harvard" IS DISTINCT FROM (NEW."xml_harvard") OR OLD."html_with_citations" IS DISTINCT FROM (NEW."html_with_citations") OR OLD."extracted_by_ocr" IS DISTINCT FROM (NEW."extracted_by_ocr"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_67ecd();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_67ecd ON "search_opinion" IS '4a3d82790ac0cbd840d6a7f6c136d4cc65419e5e';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model opinioncluster
--
@@ -452,7 +452,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_6a181()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -468,13 +468,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_6a181 ON "search_opinioncluster";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_6a181
AFTER UPDATE ON "search_opinioncluster"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."docket_id" IS DISTINCT FROM (NEW."docket_id") OR OLD."judges" IS DISTINCT FROM (NEW."judges") OR OLD."date_filed" IS DISTINCT FROM (NEW."date_filed") OR OLD."date_filed_is_approximate" IS DISTINCT FROM (NEW."date_filed_is_approximate") OR OLD."slug" IS DISTINCT FROM (NEW."slug") OR OLD."case_name_short" IS DISTINCT FROM (NEW."case_name_short") OR OLD."case_name" IS DISTINCT FROM (NEW."case_name") OR OLD."case_name_full" IS DISTINCT FROM (NEW."case_name_full") OR OLD."scdb_id" IS DISTINCT FROM (NEW."scdb_id") OR OLD."scdb_decision_direction" IS DISTINCT FROM (NEW."scdb_decision_direction") OR OLD."scdb_votes_majority" IS DISTINCT FROM (NEW."scdb_votes_majority") OR OLD."scdb_votes_minority" IS DISTINCT FROM (NEW."scdb_votes_minority") OR OLD."source" IS DISTINCT FROM (NEW."source") OR OLD."procedural_history" IS DISTINCT FROM (NEW."procedural_history") OR OLD."attorneys" IS DISTINCT FROM (NEW."attorneys") OR OLD."nature_of_suit" IS DISTINCT FROM (NEW."nature_of_suit") OR OLD."posture" IS DISTINCT FROM (NEW."posture") OR OLD."syllabus" IS DISTINCT FROM (NEW."syllabus") OR OLD."headnotes" IS DISTINCT FROM (NEW."headnotes") OR OLD."summary" IS DISTINCT FROM (NEW."summary") OR OLD."disposition" IS DISTINCT FROM (NEW."disposition") OR OLD."history" IS DISTINCT FROM (NEW."history") OR OLD."other_dates" IS DISTINCT FROM (NEW."other_dates") OR OLD."cross_reference" IS DISTINCT FROM (NEW."cross_reference") OR OLD."correction" IS DISTINCT FROM (NEW."correction") OR OLD."citation_count" IS DISTINCT FROM (NEW."citation_count") OR OLD."precedential_status" IS DISTINCT FROM (NEW."precedential_status") OR OLD."date_blocked" IS DISTINCT FROM (NEW."date_blocked") OR OLD."blocked" IS DISTINCT FROM (NEW."blocked") OR OLD."filepath_json_harvard" IS DISTINCT FROM (NEW."filepath_json_harvard"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_6a181();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_6a181 ON "search_opinioncluster" IS '907cc0f72768dba7763ab81e6e1c65f362301716';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model originatingcourtinformation
--
@@ -504,7 +504,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_49538()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -520,13 +520,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_49538 ON "search_originatingcourtinformation";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_49538
AFTER UPDATE ON "search_originatingcourtinformation"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."docket_number" IS DISTINCT FROM (NEW."docket_number") OR OLD."assigned_to_id" IS DISTINCT FROM (NEW."assigned_to_id") OR OLD."assigned_to_str" IS DISTINCT FROM (NEW."assigned_to_str") OR OLD."ordering_judge_id" IS DISTINCT FROM (NEW."ordering_judge_id") OR OLD."ordering_judge_str" IS DISTINCT FROM (NEW."ordering_judge_str") OR OLD."court_reporter" IS DISTINCT FROM (NEW."court_reporter") OR OLD."date_disposed" IS DISTINCT FROM (NEW."date_disposed") OR OLD."date_filed" IS DISTINCT FROM (NEW."date_filed") OR OLD."date_judgment" IS DISTINCT FROM (NEW."date_judgment") OR OLD."date_judgment_eod" IS DISTINCT FROM (NEW."date_judgment_eod") OR OLD."date_filed_noa" IS DISTINCT FROM (NEW."date_filed_noa") OR OLD."date_received_coa" IS DISTINCT FROM (NEW."date_received_coa"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_49538();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_49538 ON "search_originatingcourtinformation" IS '5d249a18e8be51afa8c54132770efcdde2b47a61';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model recapdocument
--
@@ -556,7 +556,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_8a108()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -572,13 +572,13 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_8a108 ON "search_recapdocument";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_8a108
AFTER UPDATE ON "search_recapdocument"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."sha1" IS DISTINCT FROM (NEW."sha1") OR OLD."page_count" IS DISTINCT FROM (NEW."page_count") OR OLD."file_size" IS DISTINCT FROM (NEW."file_size") OR OLD."filepath_local" IS DISTINCT FROM (NEW."filepath_local") OR OLD."filepath_ia" IS DISTINCT FROM (NEW."filepath_ia") OR OLD."ia_upload_failure_count" IS DISTINCT FROM (NEW."ia_upload_failure_count") OR OLD."thumbnail" IS DISTINCT FROM (NEW."thumbnail") OR OLD."thumbnail_status" IS DISTINCT FROM (NEW."thumbnail_status") OR OLD."plain_text" IS DISTINCT FROM (NEW."plain_text") OR OLD."ocr_status" IS DISTINCT FROM (NEW."ocr_status") OR OLD."date_upload" IS DISTINCT FROM (NEW."date_upload") OR OLD."document_number" IS DISTINCT FROM (NEW."document_number") OR OLD."attachment_number" IS DISTINCT FROM (NEW."attachment_number") OR OLD."pacer_doc_id" IS DISTINCT FROM (NEW."pacer_doc_id") OR OLD."is_available" IS DISTINCT FROM (NEW."is_available") OR OLD."is_free_on_pacer" IS DISTINCT FROM (NEW."is_free_on_pacer") OR OLD."is_sealed" IS DISTINCT FROM (NEW."is_sealed") OR OLD."docket_entry_id" IS DISTINCT FROM (NEW."docket_entry_id") OR OLD."document_type" IS DISTINCT FROM (NEW."document_type") OR OLD."description" IS DISTINCT FROM (NEW."description"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_8a108();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_8a108 ON "search_recapdocument" IS 'a3e0c759d8c03f380dd3eddfcff551091fcee1d1';
-
+
--
-- Create trigger update_or_delete_snapshot_update on model tag
--
@@ -608,7 +608,7 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
CREATE OR REPLACE FUNCTION pgtrigger_update_or_delete_snapshot_update_c9dd9()
RETURNS TRIGGER AS $$
-
+
BEGIN
IF ("public"._pgtrigger_should_ignore(TG_NAME) IS TRUE) THEN
IF (TG_OP = 'DELETE') THEN
@@ -624,11 +624,11 @@ ALTER INDEX "search_recapdocument_document_type_303cccac79571217_idx" RENAME TO
DROP TRIGGER IF EXISTS pgtrigger_update_or_delete_snapshot_update_c9dd9 ON "search_tag";
CREATE TRIGGER pgtrigger_update_or_delete_snapshot_update_c9dd9
AFTER UPDATE ON "search_tag"
-
-
+
+
FOR EACH ROW WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."name" IS DISTINCT FROM (NEW."name"))
EXECUTE PROCEDURE pgtrigger_update_or_delete_snapshot_update_c9dd9();
COMMENT ON TRIGGER pgtrigger_update_or_delete_snapshot_update_c9dd9 ON "search_tag" IS '4071657dcfe71811e9e7a5c24dd77c22f81d7377';
-
+
COMMIT;
diff --git a/cl/search/migrations/0032_update_docket_numbering_fields.py b/cl/search/migrations/0032_update_docket_numbering_fields.py
new file mode 100644
index 0000000000..875639c687
--- /dev/null
+++ b/cl/search/migrations/0032_update_docket_numbering_fields.py
@@ -0,0 +1,168 @@
+# Generated by Django 5.0.7 on 2024-08-01 16:31
+
+import django.db.models.deletion
+import pgtrigger.compiler
+import pgtrigger.migrations
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("search", "0031_alter_opinion_type_alter_opinioncluster_source_noop"),
+ ]
+
+ operations = [
+ pgtrigger.migrations.RemoveTrigger(
+ model_name="docket",
+ name="update_or_delete_snapshot_delete",
+ ),
+ pgtrigger.migrations.RemoveTrigger(
+ model_name="docket",
+ name="update_or_delete_snapshot_update",
+ ),
+ migrations.AddField(
+ model_name="docket",
+ name="federal_defendant_number",
+ field=models.SmallIntegerField(
+ blank=True,
+ help_text="A unique number assigned to each defendant in a case, typically found in pacer criminal cases as a -1, -2 after the judge initials. Example: 1:14-cr-10363-RGS-1.",
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docket",
+ name="federal_dn_case_type",
+ field=models.CharField(
+ blank=True,
+ help_text="Case type, e.g., civil (cv), magistrate (mj), criminal (cr), petty offense (po), and miscellaneous (mc). These codes can be upper case or lower case, and may vary in number of characters.",
+ max_length=6,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docket",
+ name="federal_dn_judge_initials_assigned",
+ field=models.CharField(
+ blank=True,
+ help_text="A typically three-letter upper cased abbreviation of the judge's initials. In the example 2:07-cv-34911-MJL, MJL is the judge's initials. Judge initials change if a new judge takes over a case.",
+ max_length=5,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docket",
+ name="federal_dn_judge_initials_referred",
+ field=models.CharField(
+ blank=True,
+ help_text="A typically three-letter upper cased abbreviation of the judge's initials. In the example 2:07-cv-34911-MJL-GOG, GOG is the magistrate judge initials.",
+ max_length=5,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docket",
+ name="federal_dn_office_code",
+ field=models.CharField(
+ blank=True,
+ help_text="A one digit statistical code (either alphabetic or numeric) of the office within the federal district. In this example, 2:07-cv-34911-MJL, the 2 preceding the : is the office code.",
+ max_length=3,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docket",
+ name="parent_docket",
+ field=models.ForeignKey(
+ blank=True,
+ help_text="In criminal cases (and some magistrate) PACER creates a parent docket and one or more child dockets. Child dockets contain docket information for each individual defendant while parent dockets are a superset of all docket entries.",
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="child_dockets",
+ to="search.docket",
+ ),
+ ),
+ migrations.AddField(
+ model_name="docketevent",
+ name="federal_defendant_number",
+ field=models.SmallIntegerField(
+ blank=True,
+ help_text="A unique number assigned to each defendant in a case, typically found in pacer criminal cases as a -1, -2 after the judge initials. Example: 1:14-cr-10363-RGS-1.",
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docketevent",
+ name="federal_dn_case_type",
+ field=models.CharField(
+ blank=True,
+ help_text="Case type, e.g., civil (cv), magistrate (mj), criminal (cr), petty offense (po), and miscellaneous (mc). These codes can be upper case or lower case, and may vary in number of characters.",
+ max_length=6,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docketevent",
+ name="federal_dn_judge_initials_assigned",
+ field=models.CharField(
+ blank=True,
+ help_text="A typically three-letter upper cased abbreviation of the judge's initials. In the example 2:07-cv-34911-MJL, MJL is the judge's initials. Judge initials change if a new judge takes over a case.",
+ max_length=5,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docketevent",
+ name="federal_dn_judge_initials_referred",
+ field=models.CharField(
+ blank=True,
+ help_text="A typically three-letter upper cased abbreviation of the judge's initials. In the example 2:07-cv-34911-MJL-GOG, GOG is the magistrate judge initials.",
+ max_length=5,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docketevent",
+ name="federal_dn_office_code",
+ field=models.CharField(
+ blank=True,
+ help_text="A one digit statistical code (either alphabetic or numeric) of the office within the federal district. In this example, 2:07-cv-34911-MJL, the 2 preceding the : is the office code.",
+ max_length=3,
+ ),
+ ),
+ migrations.AddField(
+ model_name="docketevent",
+ name="parent_docket",
+ field=models.ForeignKey(
+ blank=True,
+ db_constraint=False,
+ help_text="In criminal cases (and some magistrate) PACER creates a parent docket and one or more child dockets. Child dockets contain docket information for each individual defendant while parent dockets are a superset of all docket entries.",
+ null=True,
+ on_delete=django.db.models.deletion.DO_NOTHING,
+ related_name="+",
+ related_query_name="+",
+ to="search.docket",
+ ),
+ ),
+ pgtrigger.migrations.AddTrigger(
+ model_name="docket",
+ trigger=pgtrigger.compiler.Trigger(
+ name="update_or_delete_snapshot_update",
+ sql=pgtrigger.compiler.UpsertTriggerSql(
+ condition='WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."source" IS DISTINCT FROM (NEW."source") OR OLD."court_id" IS DISTINCT FROM (NEW."court_id") OR OLD."appeal_from_id" IS DISTINCT FROM (NEW."appeal_from_id") OR OLD."parent_docket_id" IS DISTINCT FROM (NEW."parent_docket_id") OR OLD."appeal_from_str" IS DISTINCT FROM (NEW."appeal_from_str") OR OLD."originating_court_information_id" IS DISTINCT FROM (NEW."originating_court_information_id") OR OLD."idb_data_id" IS DISTINCT FROM (NEW."idb_data_id") OR OLD."assigned_to_id" IS DISTINCT FROM (NEW."assigned_to_id") OR OLD."assigned_to_str" IS DISTINCT FROM (NEW."assigned_to_str") OR OLD."referred_to_id" IS DISTINCT FROM (NEW."referred_to_id") OR OLD."referred_to_str" IS DISTINCT FROM (NEW."referred_to_str") OR OLD."panel_str" IS DISTINCT FROM (NEW."panel_str") OR OLD."date_last_index" IS DISTINCT FROM (NEW."date_last_index") OR OLD."date_cert_granted" IS DISTINCT FROM (NEW."date_cert_granted") OR OLD."date_cert_denied" IS DISTINCT FROM (NEW."date_cert_denied") OR OLD."date_argued" IS DISTINCT FROM (NEW."date_argued") OR OLD."date_reargued" IS DISTINCT FROM (NEW."date_reargued") OR OLD."date_reargument_denied" IS DISTINCT FROM (NEW."date_reargument_denied") OR OLD."date_filed" IS DISTINCT FROM (NEW."date_filed") OR OLD."date_terminated" IS DISTINCT FROM (NEW."date_terminated") OR OLD."date_last_filing" IS DISTINCT FROM (NEW."date_last_filing") OR OLD."case_name_short" IS DISTINCT FROM (NEW."case_name_short") OR OLD."case_name" IS DISTINCT FROM (NEW."case_name") OR OLD."case_name_full" IS DISTINCT FROM (NEW."case_name_full") OR OLD."slug" IS DISTINCT FROM (NEW."slug") OR OLD."docket_number" IS DISTINCT FROM (NEW."docket_number") OR OLD."docket_number_core" IS DISTINCT FROM (NEW."docket_number_core") OR OLD."federal_dn_office_code" IS DISTINCT FROM (NEW."federal_dn_office_code") OR OLD."federal_dn_case_type" IS DISTINCT FROM (NEW."federal_dn_case_type") OR OLD."federal_dn_judge_initials_assigned" IS DISTINCT FROM (NEW."federal_dn_judge_initials_assigned") OR OLD."federal_dn_judge_initials_referred" IS DISTINCT FROM (NEW."federal_dn_judge_initials_referred") OR OLD."federal_defendant_number" IS DISTINCT FROM (NEW."federal_defendant_number") OR OLD."pacer_case_id" IS DISTINCT FROM (NEW."pacer_case_id") OR OLD."cause" IS DISTINCT FROM (NEW."cause") OR OLD."nature_of_suit" IS DISTINCT FROM (NEW."nature_of_suit") OR OLD."jury_demand" IS DISTINCT FROM (NEW."jury_demand") OR OLD."jurisdiction_type" IS DISTINCT FROM (NEW."jurisdiction_type") OR OLD."appellate_fee_status" IS DISTINCT FROM (NEW."appellate_fee_status") OR OLD."appellate_case_type_information" IS DISTINCT FROM (NEW."appellate_case_type_information") OR OLD."mdl_status" IS DISTINCT FROM (NEW."mdl_status") OR OLD."filepath_local" IS DISTINCT FROM (NEW."filepath_local") OR OLD."filepath_ia" IS DISTINCT FROM (NEW."filepath_ia") OR OLD."filepath_ia_json" IS DISTINCT FROM (NEW."filepath_ia_json") OR OLD."ia_upload_failure_count" IS DISTINCT FROM (NEW."ia_upload_failure_count") OR OLD."ia_needs_upload" IS DISTINCT FROM (NEW."ia_needs_upload") OR OLD."ia_date_first_change" IS DISTINCT FROM (NEW."ia_date_first_change") OR OLD."date_blocked" IS DISTINCT FROM (NEW."date_blocked") OR OLD."blocked" IS DISTINCT FROM (NEW."blocked"))',
+ func='INSERT INTO "search_docketevent" ("appeal_from_id", "appeal_from_str", "appellate_case_type_information", "appellate_fee_status", "assigned_to_id", "assigned_to_str", "blocked", "case_name", "case_name_full", "case_name_short", "cause", "court_id", "date_argued", "date_blocked", "date_cert_denied", "date_cert_granted", "date_created", "date_filed", "date_last_filing", "date_last_index", "date_modified", "date_reargued", "date_reargument_denied", "date_terminated", "docket_number", "docket_number_core", "federal_defendant_number", "federal_dn_case_type", "federal_dn_judge_initials_assigned", "federal_dn_judge_initials_referred", "federal_dn_office_code", "filepath_ia", "filepath_ia_json", "filepath_local", "ia_date_first_change", "ia_needs_upload", "ia_upload_failure_count", "id", "idb_data_id", "jurisdiction_type", "jury_demand", "mdl_status", "nature_of_suit", "originating_court_information_id", "pacer_case_id", "panel_str", "parent_docket_id", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "referred_to_id", "referred_to_str", "slug", "source") VALUES (OLD."appeal_from_id", OLD."appeal_from_str", OLD."appellate_case_type_information", OLD."appellate_fee_status", OLD."assigned_to_id", OLD."assigned_to_str", OLD."blocked", OLD."case_name", OLD."case_name_full", OLD."case_name_short", OLD."cause", OLD."court_id", OLD."date_argued", OLD."date_blocked", OLD."date_cert_denied", OLD."date_cert_granted", OLD."date_created", OLD."date_filed", OLD."date_last_filing", OLD."date_last_index", OLD."date_modified", OLD."date_reargued", OLD."date_reargument_denied", OLD."date_terminated", OLD."docket_number", OLD."docket_number_core", OLD."federal_defendant_number", OLD."federal_dn_case_type", OLD."federal_dn_judge_initials_assigned", OLD."federal_dn_judge_initials_referred", OLD."federal_dn_office_code", OLD."filepath_ia", OLD."filepath_ia_json", OLD."filepath_local", OLD."ia_date_first_change", OLD."ia_needs_upload", OLD."ia_upload_failure_count", OLD."id", OLD."idb_data_id", OLD."jurisdiction_type", OLD."jury_demand", OLD."mdl_status", OLD."nature_of_suit", OLD."originating_court_information_id", OLD."pacer_case_id", OLD."panel_str", OLD."parent_docket_id", _pgh_attach_context(), NOW(), \'update_or_delete_snapshot\', OLD."id", OLD."referred_to_id", OLD."referred_to_str", OLD."slug", OLD."source"); RETURN NULL;',
+ hash="f2c9e18d74e58ec15e0f9d06a80edb4ae17347e8",
+ operation="UPDATE",
+ pgid="pgtrigger_update_or_delete_snapshot_update_7e039",
+ table="search_docket",
+ when="AFTER",
+ ),
+ ),
+ ),
+ pgtrigger.migrations.AddTrigger(
+ model_name="docket",
+ trigger=pgtrigger.compiler.Trigger(
+ name="update_or_delete_snapshot_delete",
+ sql=pgtrigger.compiler.UpsertTriggerSql(
+ func='INSERT INTO "search_docketevent" ("appeal_from_id", "appeal_from_str", "appellate_case_type_information", "appellate_fee_status", "assigned_to_id", "assigned_to_str", "blocked", "case_name", "case_name_full", "case_name_short", "cause", "court_id", "date_argued", "date_blocked", "date_cert_denied", "date_cert_granted", "date_created", "date_filed", "date_last_filing", "date_last_index", "date_modified", "date_reargued", "date_reargument_denied", "date_terminated", "docket_number", "docket_number_core", "federal_defendant_number", "federal_dn_case_type", "federal_dn_judge_initials_assigned", "federal_dn_judge_initials_referred", "federal_dn_office_code", "filepath_ia", "filepath_ia_json", "filepath_local", "ia_date_first_change", "ia_needs_upload", "ia_upload_failure_count", "id", "idb_data_id", "jurisdiction_type", "jury_demand", "mdl_status", "nature_of_suit", "originating_court_information_id", "pacer_case_id", "panel_str", "parent_docket_id", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "referred_to_id", "referred_to_str", "slug", "source") VALUES (OLD."appeal_from_id", OLD."appeal_from_str", OLD."appellate_case_type_information", OLD."appellate_fee_status", OLD."assigned_to_id", OLD."assigned_to_str", OLD."blocked", OLD."case_name", OLD."case_name_full", OLD."case_name_short", OLD."cause", OLD."court_id", OLD."date_argued", OLD."date_blocked", OLD."date_cert_denied", OLD."date_cert_granted", OLD."date_created", OLD."date_filed", OLD."date_last_filing", OLD."date_last_index", OLD."date_modified", OLD."date_reargued", OLD."date_reargument_denied", OLD."date_terminated", OLD."docket_number", OLD."docket_number_core", OLD."federal_defendant_number", OLD."federal_dn_case_type", OLD."federal_dn_judge_initials_assigned", OLD."federal_dn_judge_initials_referred", OLD."federal_dn_office_code", OLD."filepath_ia", OLD."filepath_ia_json", OLD."filepath_local", OLD."ia_date_first_change", OLD."ia_needs_upload", OLD."ia_upload_failure_count", OLD."id", OLD."idb_data_id", OLD."jurisdiction_type", OLD."jury_demand", OLD."mdl_status", OLD."nature_of_suit", OLD."originating_court_information_id", OLD."pacer_case_id", OLD."panel_str", OLD."parent_docket_id", _pgh_attach_context(), NOW(), \'update_or_delete_snapshot\', OLD."id", OLD."referred_to_id", OLD."referred_to_str", OLD."slug", OLD."source"); RETURN NULL;',
+ hash="a4b1625360e32dfb7392272ed99823a289ea336a",
+ operation="DELETE",
+ pgid="pgtrigger_update_or_delete_snapshot_delete_7294f",
+ table="search_docket",
+ when="AFTER",
+ ),
+ ),
+ ),
+ ]
diff --git a/cl/search/migrations/0032_update_docket_numbering_fields.sql b/cl/search/migrations/0032_update_docket_numbering_fields.sql
new file mode 100644
index 0000000000..77991068e5
--- /dev/null
+++ b/cl/search/migrations/0032_update_docket_numbering_fields.sql
@@ -0,0 +1,59 @@
+BEGIN;
+--
+-- Add field federal_defendant_number to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_defendant_number" smallint NULL;
+--
+-- Add field federal_dn_case_type to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_dn_case_type" varchar(6) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docket" ALTER COLUMN "federal_dn_case_type" DROP DEFAULT;
+--
+-- Add field federal_dn_judge_initials_assigned to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_dn_judge_initials_assigned" varchar(5) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docket" ALTER COLUMN "federal_dn_judge_initials_assigned" DROP DEFAULT;
+--
+-- Add field federal_dn_judge_initials_referred to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_dn_judge_initials_referred" varchar(5) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docket" ALTER COLUMN "federal_dn_judge_initials_referred" DROP DEFAULT;
+--
+-- Add field federal_dn_office_code to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_dn_office_code" varchar(3) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docket" ALTER COLUMN "federal_dn_office_code" DROP DEFAULT;
+--
+-- Add field parent_docket to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "parent_docket_id" integer NULL CONSTRAINT "search_docket_parent_docket_id_1a514426_fk_search_docket_id" REFERENCES "search_docket"("id") DEFERRABLE INITIALLY DEFERRED; SET CONSTRAINTS "search_docket_parent_docket_id_1a514426_fk_search_docket_id" IMMEDIATE;
+--
+-- Add field federal_defendant_number to docketevent
+--
+ALTER TABLE "search_docketevent" ADD COLUMN "federal_defendant_number" smallint NULL;
+--
+-- Add field federal_dn_case_type to docketevent
+--
+ALTER TABLE "search_docketevent" ADD COLUMN "federal_dn_case_type" varchar(6) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docketevent" ALTER COLUMN "federal_dn_case_type" DROP DEFAULT;
+--
+-- Add field federal_dn_judge_initials_assigned to docketevent
+--
+ALTER TABLE "search_docketevent" ADD COLUMN "federal_dn_judge_initials_assigned" varchar(5) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docketevent" ALTER COLUMN "federal_dn_judge_initials_assigned" DROP DEFAULT;
+--
+-- Add field federal_dn_judge_initials_referred to docketevent
+--
+ALTER TABLE "search_docketevent" ADD COLUMN "federal_dn_judge_initials_referred" varchar(5) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docketevent" ALTER COLUMN "federal_dn_judge_initials_referred" DROP DEFAULT;
+--
+-- Add field federal_dn_office_code to docketevent
+--
+ALTER TABLE "search_docketevent" ADD COLUMN "federal_dn_office_code" varchar(3) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docketevent" ALTER COLUMN "federal_dn_office_code" DROP DEFAULT;
+--
+-- Add field parent_docket to docketevent
+--
+CREATE INDEX "search_docket_parent_docket_id_1a514426" ON "search_docket" ("parent_docket_id");
+CREATE INDEX "search_docketevent_parent_docket_id_c7c9c9ad" ON "search_docketevent" ("parent_docket_id");
+COMMIT;
\ No newline at end of file
diff --git a/cl/search/migrations/0032_update_docket_numbering_fields_customers.sql b/cl/search/migrations/0032_update_docket_numbering_fields_customers.sql
new file mode 100644
index 0000000000..c6409e0fdb
--- /dev/null
+++ b/cl/search/migrations/0032_update_docket_numbering_fields_customers.sql
@@ -0,0 +1,34 @@
+BEGIN;
+--
+-- Add field federal_defendant_number to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_defendant_number" smallint NULL;
+--
+-- Add field federal_dn_case_type to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_dn_case_type" varchar(6) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docket" ALTER COLUMN "federal_dn_case_type" DROP DEFAULT;
+--
+-- Add field federal_dn_judge_initials_assigned to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_dn_judge_initials_assigned" varchar(5) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docket" ALTER COLUMN "federal_dn_judge_initials_assigned" DROP DEFAULT;
+--
+-- Add field federal_dn_judge_initials_referred to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_dn_judge_initials_referred" varchar(5) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docket" ALTER COLUMN "federal_dn_judge_initials_referred" DROP DEFAULT;
+--
+-- Add field federal_dn_office_code to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "federal_dn_office_code" varchar(3) DEFAULT '' NOT NULL;
+ALTER TABLE "search_docket" ALTER COLUMN "federal_dn_office_code" DROP DEFAULT;
+--
+-- Add field parent_docket to docket
+--
+ALTER TABLE "search_docket" ADD COLUMN "parent_docket_id" integer NULL CONSTRAINT "search_docket_parent_docket_id_1a514426_fk_search_docket_id" REFERENCES "search_docket"("id") DEFERRABLE INITIALLY DEFERRED; SET CONSTRAINTS "search_docket_parent_docket_id_1a514426_fk_search_docket_id" IMMEDIATE;
+--
+-- Create an index on the parent_docket_id column in the search_docket table
+--
+CREATE INDEX "search_docket_parent_docket_id_1a514426" ON "search_docket" ("parent_docket_id");
+COMMIT;
\ No newline at end of file
diff --git a/cl/search/migrations/0033_order_opinions.py b/cl/search/migrations/0033_order_opinions.py
new file mode 100644
index 0000000000..ce5ea91c13
--- /dev/null
+++ b/cl/search/migrations/0033_order_opinions.py
@@ -0,0 +1,72 @@
+# Generated by Django 5.0.7 on 2024-08-05 20:19
+
+import pgtrigger.compiler
+import pgtrigger.migrations
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ (
+ "people_db",
+ "0016_remove_abarating_update_or_delete_snapshot_update_and_more",
+ ),
+ ("search", "0032_update_docket_numbering_fields"),
+ ]
+
+ operations = [
+ pgtrigger.migrations.RemoveTrigger(
+ model_name="opinion",
+ name="update_or_delete_snapshot_delete",
+ ),
+ pgtrigger.migrations.RemoveTrigger(
+ model_name="opinion",
+ name="update_or_delete_snapshot_update",
+ ),
+ migrations.AddField(
+ model_name="opinion",
+ name="ordering_key",
+ field=models.IntegerField(blank=True, null=True),
+ ),
+ migrations.AddField(
+ model_name="opinionevent",
+ name="ordering_key",
+ field=models.IntegerField(blank=True, null=True),
+ ),
+ pgtrigger.migrations.AddTrigger(
+ model_name="opinion",
+ trigger=pgtrigger.compiler.Trigger(
+ name="update_or_delete_snapshot_update",
+ sql=pgtrigger.compiler.UpsertTriggerSql(
+ condition='WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."cluster_id" IS DISTINCT FROM (NEW."cluster_id") OR OLD."author_id" IS DISTINCT FROM (NEW."author_id") OR OLD."author_str" IS DISTINCT FROM (NEW."author_str") OR OLD."per_curiam" IS DISTINCT FROM (NEW."per_curiam") OR OLD."joined_by_str" IS DISTINCT FROM (NEW."joined_by_str") OR OLD."type" IS DISTINCT FROM (NEW."type") OR OLD."sha1" IS DISTINCT FROM (NEW."sha1") OR OLD."page_count" IS DISTINCT FROM (NEW."page_count") OR OLD."download_url" IS DISTINCT FROM (NEW."download_url") OR OLD."local_path" IS DISTINCT FROM (NEW."local_path") OR OLD."plain_text" IS DISTINCT FROM (NEW."plain_text") OR OLD."html" IS DISTINCT FROM (NEW."html") OR OLD."html_lawbox" IS DISTINCT FROM (NEW."html_lawbox") OR OLD."html_columbia" IS DISTINCT FROM (NEW."html_columbia") OR OLD."html_anon_2020" IS DISTINCT FROM (NEW."html_anon_2020") OR OLD."xml_harvard" IS DISTINCT FROM (NEW."xml_harvard") OR OLD."html_with_citations" IS DISTINCT FROM (NEW."html_with_citations") OR OLD."extracted_by_ocr" IS DISTINCT FROM (NEW."extracted_by_ocr") OR OLD."ordering_key" IS DISTINCT FROM (NEW."ordering_key"))',
+ func='INSERT INTO "search_opinionevent" ("author_id", "author_str", "cluster_id", "date_created", "date_modified", "download_url", "extracted_by_ocr", "html", "html_anon_2020", "html_columbia", "html_lawbox", "html_with_citations", "id", "joined_by_str", "local_path", "ordering_key", "page_count", "per_curiam", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "plain_text", "sha1", "type", "xml_harvard") VALUES (OLD."author_id", OLD."author_str", OLD."cluster_id", OLD."date_created", OLD."date_modified", OLD."download_url", OLD."extracted_by_ocr", OLD."html", OLD."html_anon_2020", OLD."html_columbia", OLD."html_lawbox", OLD."html_with_citations", OLD."id", OLD."joined_by_str", OLD."local_path", OLD."ordering_key", OLD."page_count", OLD."per_curiam", _pgh_attach_context(), NOW(), \'update_or_delete_snapshot\', OLD."id", OLD."plain_text", OLD."sha1", OLD."type", OLD."xml_harvard"); RETURN NULL;',
+ hash="7137855274503cc2c50a17729f82e150d2b7d872",
+ operation="UPDATE",
+ pgid="pgtrigger_update_or_delete_snapshot_update_67ecd",
+ table="search_opinion",
+ when="AFTER",
+ ),
+ ),
+ ),
+ pgtrigger.migrations.AddTrigger(
+ model_name="opinion",
+ trigger=pgtrigger.compiler.Trigger(
+ name="update_or_delete_snapshot_delete",
+ sql=pgtrigger.compiler.UpsertTriggerSql(
+ func='INSERT INTO "search_opinionevent" ("author_id", "author_str", "cluster_id", "date_created", "date_modified", "download_url", "extracted_by_ocr", "html", "html_anon_2020", "html_columbia", "html_lawbox", "html_with_citations", "id", "joined_by_str", "local_path", "ordering_key", "page_count", "per_curiam", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "plain_text", "sha1", "type", "xml_harvard") VALUES (OLD."author_id", OLD."author_str", OLD."cluster_id", OLD."date_created", OLD."date_modified", OLD."download_url", OLD."extracted_by_ocr", OLD."html", OLD."html_anon_2020", OLD."html_columbia", OLD."html_lawbox", OLD."html_with_citations", OLD."id", OLD."joined_by_str", OLD."local_path", OLD."ordering_key", OLD."page_count", OLD."per_curiam", _pgh_attach_context(), NOW(), \'update_or_delete_snapshot\', OLD."id", OLD."plain_text", OLD."sha1", OLD."type", OLD."xml_harvard"); RETURN NULL;',
+ hash="98fb52aa60fd8e89a83f8f7ac77ba5892739fb37",
+ operation="DELETE",
+ pgid="pgtrigger_update_or_delete_snapshot_delete_1f4fd",
+ table="search_opinion",
+ when="AFTER",
+ ),
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="opinion",
+ constraint=models.UniqueConstraint(
+ fields=("cluster_id", "ordering_key"),
+ name="unique_opinion_ordering_key",
+ ),
+ ),
+ ]
diff --git a/cl/search/migrations/0033_order_opinions.sql b/cl/search/migrations/0033_order_opinions.sql
new file mode 100644
index 0000000000..e2e07aee39
--- /dev/null
+++ b/cl/search/migrations/0033_order_opinions.sql
@@ -0,0 +1,14 @@
+BEGIN;
+--
+-- Add field ordering_key to opinion
+--
+ALTER TABLE "search_opinion" ADD COLUMN "ordering_key" integer NULL;
+--
+-- Add field ordering_key to opinionevent
+--
+ALTER TABLE "search_opinionevent" ADD COLUMN "ordering_key" integer NULL;
+--
+-- Create constraint unique_opinion_ordering_key on model opinion
+--
+ALTER TABLE "search_opinion" ADD CONSTRAINT "unique_opinion_ordering_key" UNIQUE ("cluster_id", "ordering_key");
+COMMIT;
diff --git a/cl/search/migrations/0033_order_opinions_customers.sql b/cl/search/migrations/0033_order_opinions_customers.sql
new file mode 100644
index 0000000000..e7158e3002
--- /dev/null
+++ b/cl/search/migrations/0033_order_opinions_customers.sql
@@ -0,0 +1,10 @@
+BEGIN;
+--
+-- Add field ordering_key to opinion
+--
+ALTER TABLE "search_opinion" ADD COLUMN "ordering_key" integer NULL;
+--
+-- Create constraint unique_opinion_ordering_key on model opinion
+--
+ALTER TABLE "search_opinion" ADD CONSTRAINT "unique_opinion_ordering_key" UNIQUE ("cluster_id", "ordering_key");
+COMMIT;
diff --git a/cl/search/migrations/0034_add_harvard_pdf_to_opinioncluster.py b/cl/search/migrations/0034_add_harvard_pdf_to_opinioncluster.py
new file mode 100644
index 0000000000..b73b7925c2
--- /dev/null
+++ b/cl/search/migrations/0034_add_harvard_pdf_to_opinioncluster.py
@@ -0,0 +1,73 @@
+# Generated by Django 5.0.7 on 2024-08-14 18:16
+
+import cl.lib.model_helpers
+import cl.lib.storage
+import pgtrigger.compiler
+import pgtrigger.migrations
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("search", "0033_order_opinions"),
+ ]
+
+ operations = [
+ pgtrigger.migrations.RemoveTrigger(
+ model_name="opinioncluster",
+ name="update_or_delete_snapshot_update",
+ ),
+ pgtrigger.migrations.RemoveTrigger(
+ model_name="opinioncluster",
+ name="update_or_delete_snapshot_delete",
+ ),
+ migrations.AddField(
+ model_name="opinioncluster",
+ name="filepath_pdf_harvard",
+ field=models.FileField(
+ blank=True,
+ help_text="The case PDF from the Caselaw Access Project for this cluster",
+ storage=cl.lib.storage.IncrementingAWSMediaStorage(),
+ upload_to=cl.lib.model_helpers.make_upload_path,
+ ),
+ ),
+ migrations.AddField(
+ model_name="opinionclusterevent",
+ name="filepath_pdf_harvard",
+ field=models.FileField(
+ blank=True,
+ help_text="The case PDF from the Caselaw Access Project for this cluster",
+ storage=cl.lib.storage.IncrementingAWSMediaStorage(),
+ upload_to=cl.lib.model_helpers.make_upload_path,
+ ),
+ ),
+ pgtrigger.migrations.AddTrigger(
+ model_name="opinioncluster",
+ trigger=pgtrigger.compiler.Trigger(
+ name="update_or_delete_snapshot_update",
+ sql=pgtrigger.compiler.UpsertTriggerSql(
+ condition='WHEN (OLD."id" IS DISTINCT FROM (NEW."id") OR OLD."date_created" IS DISTINCT FROM (NEW."date_created") OR OLD."docket_id" IS DISTINCT FROM (NEW."docket_id") OR OLD."judges" IS DISTINCT FROM (NEW."judges") OR OLD."date_filed" IS DISTINCT FROM (NEW."date_filed") OR OLD."date_filed_is_approximate" IS DISTINCT FROM (NEW."date_filed_is_approximate") OR OLD."slug" IS DISTINCT FROM (NEW."slug") OR OLD."case_name_short" IS DISTINCT FROM (NEW."case_name_short") OR OLD."case_name" IS DISTINCT FROM (NEW."case_name") OR OLD."case_name_full" IS DISTINCT FROM (NEW."case_name_full") OR OLD."scdb_id" IS DISTINCT FROM (NEW."scdb_id") OR OLD."scdb_decision_direction" IS DISTINCT FROM (NEW."scdb_decision_direction") OR OLD."scdb_votes_majority" IS DISTINCT FROM (NEW."scdb_votes_majority") OR OLD."scdb_votes_minority" IS DISTINCT FROM (NEW."scdb_votes_minority") OR OLD."source" IS DISTINCT FROM (NEW."source") OR OLD."procedural_history" IS DISTINCT FROM (NEW."procedural_history") OR OLD."attorneys" IS DISTINCT FROM (NEW."attorneys") OR OLD."nature_of_suit" IS DISTINCT FROM (NEW."nature_of_suit") OR OLD."posture" IS DISTINCT FROM (NEW."posture") OR OLD."syllabus" IS DISTINCT FROM (NEW."syllabus") OR OLD."headnotes" IS DISTINCT FROM (NEW."headnotes") OR OLD."summary" IS DISTINCT FROM (NEW."summary") OR OLD."disposition" IS DISTINCT FROM (NEW."disposition") OR OLD."history" IS DISTINCT FROM (NEW."history") OR OLD."other_dates" IS DISTINCT FROM (NEW."other_dates") OR OLD."cross_reference" IS DISTINCT FROM (NEW."cross_reference") OR OLD."correction" IS DISTINCT FROM (NEW."correction") OR OLD."citation_count" IS DISTINCT FROM (NEW."citation_count") OR OLD."precedential_status" IS DISTINCT FROM (NEW."precedential_status") OR OLD."date_blocked" IS DISTINCT FROM (NEW."date_blocked") OR OLD."blocked" IS DISTINCT FROM (NEW."blocked") OR OLD."filepath_json_harvard" IS DISTINCT FROM (NEW."filepath_json_harvard") OR OLD."filepath_pdf_harvard" IS DISTINCT FROM (NEW."filepath_pdf_harvard") OR OLD."arguments" IS DISTINCT FROM (NEW."arguments") OR OLD."headmatter" IS DISTINCT FROM (NEW."headmatter"))',
+ func='INSERT INTO "search_opinionclusterevent" ("arguments", "attorneys", "blocked", "case_name", "case_name_full", "case_name_short", "citation_count", "correction", "cross_reference", "date_blocked", "date_created", "date_filed", "date_filed_is_approximate", "date_modified", "disposition", "docket_id", "filepath_json_harvard", "filepath_pdf_harvard", "headmatter", "headnotes", "history", "id", "judges", "nature_of_suit", "other_dates", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "posture", "precedential_status", "procedural_history", "scdb_decision_direction", "scdb_id", "scdb_votes_majority", "scdb_votes_minority", "slug", "source", "summary", "syllabus") VALUES (OLD."arguments", OLD."attorneys", OLD."blocked", OLD."case_name", OLD."case_name_full", OLD."case_name_short", OLD."citation_count", OLD."correction", OLD."cross_reference", OLD."date_blocked", OLD."date_created", OLD."date_filed", OLD."date_filed_is_approximate", OLD."date_modified", OLD."disposition", OLD."docket_id", OLD."filepath_json_harvard", OLD."filepath_pdf_harvard", OLD."headmatter", OLD."headnotes", OLD."history", OLD."id", OLD."judges", OLD."nature_of_suit", OLD."other_dates", _pgh_attach_context(), NOW(), \'update_or_delete_snapshot\', OLD."id", OLD."posture", OLD."precedential_status", OLD."procedural_history", OLD."scdb_decision_direction", OLD."scdb_id", OLD."scdb_votes_majority", OLD."scdb_votes_minority", OLD."slug", OLD."source", OLD."summary", OLD."syllabus"); RETURN NULL;',
+ hash="bd5a29c929acce5171721fc7a4471ceb5d8c6f87",
+ operation="UPDATE",
+ pgid="pgtrigger_update_or_delete_snapshot_update_6a181",
+ table="search_opinioncluster",
+ when="AFTER",
+ ),
+ ),
+ ),
+ pgtrigger.migrations.AddTrigger(
+ model_name="opinioncluster",
+ trigger=pgtrigger.compiler.Trigger(
+ name="update_or_delete_snapshot_delete",
+ sql=pgtrigger.compiler.UpsertTriggerSql(
+ func='INSERT INTO "search_opinionclusterevent" ("arguments", "attorneys", "blocked", "case_name", "case_name_full", "case_name_short", "citation_count", "correction", "cross_reference", "date_blocked", "date_created", "date_filed", "date_filed_is_approximate", "date_modified", "disposition", "docket_id", "filepath_json_harvard", "filepath_pdf_harvard", "headmatter", "headnotes", "history", "id", "judges", "nature_of_suit", "other_dates", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "posture", "precedential_status", "procedural_history", "scdb_decision_direction", "scdb_id", "scdb_votes_majority", "scdb_votes_minority", "slug", "source", "summary", "syllabus") VALUES (OLD."arguments", OLD."attorneys", OLD."blocked", OLD."case_name", OLD."case_name_full", OLD."case_name_short", OLD."citation_count", OLD."correction", OLD."cross_reference", OLD."date_blocked", OLD."date_created", OLD."date_filed", OLD."date_filed_is_approximate", OLD."date_modified", OLD."disposition", OLD."docket_id", OLD."filepath_json_harvard", OLD."filepath_pdf_harvard", OLD."headmatter", OLD."headnotes", OLD."history", OLD."id", OLD."judges", OLD."nature_of_suit", OLD."other_dates", _pgh_attach_context(), NOW(), \'update_or_delete_snapshot\', OLD."id", OLD."posture", OLD."precedential_status", OLD."procedural_history", OLD."scdb_decision_direction", OLD."scdb_id", OLD."scdb_votes_majority", OLD."scdb_votes_minority", OLD."slug", OLD."source", OLD."summary", OLD."syllabus"); RETURN NULL;',
+ hash="aef7156b0ac4daa6c9fcc0c6ea2d981676bc6221",
+ operation="DELETE",
+ pgid="pgtrigger_update_or_delete_snapshot_delete_58fe8",
+ table="search_opinioncluster",
+ when="AFTER",
+ ),
+ ),
+ ),
+ ]
diff --git a/cl/search/migrations/0034_add_harvard_pdf_to_opinioncluster.sql b/cl/search/migrations/0034_add_harvard_pdf_to_opinioncluster.sql
new file mode 100644
index 0000000000..3f1cc19daf
--- /dev/null
+++ b/cl/search/migrations/0034_add_harvard_pdf_to_opinioncluster.sql
@@ -0,0 +1,6 @@
+BEGIN;
+ALTER TABLE "search_opinioncluster" ADD COLUMN "filepath_pdf_harvard" varchar(100) DEFAULT '' NOT NULL;
+ALTER TABLE "search_opinioncluster" ALTER COLUMN "filepath_pdf_harvard" DROP DEFAULT;
+ALTER TABLE "search_opinionclusterevent" ADD COLUMN "filepath_pdf_harvard" varchar(100) DEFAULT '' NOT NULL;
+ALTER TABLE "search_opinionclusterevent" ALTER COLUMN "filepath_pdf_harvard" DROP DEFAULT;
+COMMIT;
\ No newline at end of file
diff --git a/cl/search/migrations/0034_add_harvard_pdf_to_opinioncluster_customers.sql b/cl/search/migrations/0034_add_harvard_pdf_to_opinioncluster_customers.sql
new file mode 100644
index 0000000000..71248e195a
--- /dev/null
+++ b/cl/search/migrations/0034_add_harvard_pdf_to_opinioncluster_customers.sql
@@ -0,0 +1,4 @@
+BEGIN;
+ALTER TABLE "search_opinioncluster" ADD COLUMN "filepath_pdf_harvard" varchar(100) DEFAULT '' NOT NULL;
+ALTER TABLE "search_opinioncluster" ALTER COLUMN "filepath_pdf_harvard" DROP DEFAULT;
+COMMIT;
\ No newline at end of file
diff --git a/cl/search/models.py b/cl/search/models.py
index ef0f1e91e9..a0c808f3d3 100644
--- a/cl/search/models.py
+++ b/cl/search/models.py
@@ -28,6 +28,8 @@
from cl.lib import fields
from cl.lib.date_time import midnight_pt
from cl.lib.model_helpers import (
+ CSVExportMixin,
+ linkify_orig_docket_number,
make_docket_number_core,
make_recap_path,
make_upload_path,
@@ -327,6 +329,12 @@ class OriginatingCourtInformation(AbstractDateTimeModel):
null=True,
)
+ @property
+ def administrative_link(self):
+ return linkify_orig_docket_number(
+ self.docket.appeal_from_str, self.docket_number
+ )
+
def get_absolute_url(self) -> str:
return self.docket.get_absolute_url()
@@ -365,6 +373,17 @@ class Docket(AbstractDateTimeModel, DocketSources):
blank=True,
null=True,
)
+ parent_docket = models.ForeignKey(
+ "self",
+ help_text="In criminal cases (and some magistrate) PACER creates "
+ "a parent docket and one or more child dockets. Child dockets "
+ "contain docket information for each individual defendant "
+ "while parent dockets are a superset of all docket entries.",
+ on_delete=models.SET_NULL,
+ blank=True,
+ null=True,
+ related_name="child_dockets",
+ )
appeal_from_str = models.TextField(
help_text=(
"In appellate cases, this is the lower court or "
@@ -545,6 +564,44 @@ class Docket(AbstractDateTimeModel, DocketSources):
blank=True,
db_index=True,
)
+ federal_dn_office_code = models.CharField(
+ help_text="A one digit statistical code (either alphabetic or numeric) "
+ "of the office within the federal district. In this "
+ "example, 2:07-cv-34911-MJL, the 2 preceding "
+ "the : is the office code.",
+ max_length=3,
+ blank=True,
+ )
+ federal_dn_case_type = models.CharField(
+ help_text="Case type, e.g., civil (cv), magistrate (mj), criminal (cr), "
+ "petty offense (po), and miscellaneous (mc). These codes "
+ "can be upper case or lower case, and may vary in number of "
+ "characters.",
+ max_length=6,
+ blank=True,
+ )
+ federal_dn_judge_initials_assigned = models.CharField(
+ help_text="A typically three-letter upper cased abbreviation "
+ "of the judge's initials. In the example 2:07-cv-34911-MJL, "
+ "MJL is the judge's initials. Judge initials change if a "
+ "new judge takes over a case.",
+ max_length=5,
+ blank=True,
+ )
+ federal_dn_judge_initials_referred = models.CharField(
+ help_text="A typically three-letter upper cased abbreviation "
+ "of the judge's initials. In the example 2:07-cv-34911-MJL-GOG, "
+ "GOG is the magistrate judge initials.",
+ max_length=5,
+ blank=True,
+ )
+ federal_defendant_number = models.SmallIntegerField(
+ help_text="A unique number assigned to each defendant in a case, "
+ "typically found in pacer criminal cases as a -1, -2 after "
+ "the judge initials. Example: 1:14-cr-10363-RGS-1.",
+ null=True,
+ blank=True,
+ )
# Nullable for unique constraint requirements.
pacer_case_id = fields.CharNullField(
help_text="The case ID provided by PACER.",
@@ -1080,7 +1137,7 @@ class Meta:
@pghistory.track(AfterUpdateOrDeleteSnapshot())
-class DocketEntry(AbstractDateTimeModel):
+class DocketEntry(AbstractDateTimeModel, CSVExportMixin):
docket = models.ForeignKey(
Docket,
help_text=(
@@ -1180,7 +1237,10 @@ class Meta:
name="entry_number_idx",
condition=Q(entry_number=1),
),
- models.Index(fields=["recap_sequence_number", "entry_number"]),
+ models.Index(
+ fields=["recap_sequence_number", "entry_number"],
+ name="search_docketentry_recap_sequence_number_1c82e51988e2d89f_idx",
+ ),
]
ordering = ("recap_sequence_number", "entry_number")
permissions = (("has_recap_api_access", "Can work with RECAP API"),)
@@ -1201,6 +1261,27 @@ def datetime_filed(self) -> datetime | None:
)
return None
+ def get_csv_columns(self, get_column_name=False):
+ columns = [
+ "id",
+ "entry_number",
+ "date_filed",
+ "time_filed",
+ "pacer_sequence_number",
+ "recap_sequence_number",
+ "description",
+ ]
+ if get_column_name:
+ columns = [self.add_class_name(col) for col in columns]
+ return columns
+
+ def get_column_function(self):
+ """Get dict of attrs: fucntion to apply on field value if it needs
+ to be pre-processed before being add to csv
+
+ returns: dict -- > {attr1: function}"""
+ return {}
+
@pghistory.track(AfterUpdateOrDeleteSnapshot(), obj_field=None)
class DocketEntryTags(DocketEntry.tags.through):
@@ -1262,7 +1343,9 @@ class Meta:
@pghistory.track(AfterUpdateOrDeleteSnapshot())
-class RECAPDocument(AbstractPacerDocument, AbstractPDF, AbstractDateTimeModel):
+class RECAPDocument(
+ AbstractPacerDocument, AbstractPDF, AbstractDateTimeModel, CSVExportMixin
+):
"""The model for Docket Documents and Attachments."""
PACER_DOCUMENT = 1
@@ -1332,7 +1415,8 @@ class Meta:
"document_type",
"document_number",
"attachment_number",
- ]
+ ],
+ name="search_recapdocument_document_type_303cccac79571217_idx",
),
models.Index(
fields=["filepath_local"],
@@ -1408,11 +1492,11 @@ def pacer_url(self) -> str | None:
court_id = map_cl_to_pacer_id(court.pk)
if self.pacer_doc_id:
if self.pacer_doc_id.count("-") > 1:
- # It seems like loading the ACMS Download Page using links is not
- # possible. we've implemented a modal window that explains this
- # issue and guides users towards using the button to access the
- # docket report.
- return self.docket_entry.docket.pacer_docket_url
+ return (
+ f"https://{court_id}-showdoc.azurewebsites.us/docs/"
+ f"{self.docket_entry.docket.pacer_case_id}/"
+ f"{self.pacer_doc_id}"
+ )
elif court.jurisdiction == Court.FEDERAL_APPELLATE:
template = "https://ecf.%s.uscourts.gov/docs1/%s?caseId=%s"
else:
@@ -1680,6 +1764,53 @@ def as_search_dict(self, docket_metadata=None):
return normalize_search_dicts(out)
+ def get_csv_columns(self, get_column_name=False):
+ columns = [
+ "id",
+ "document_type",
+ "description",
+ "acms_document_guid",
+ "date_upload",
+ "document_number",
+ "attachment_number",
+ "pacer_doc_id",
+ "is_free_on_pacer",
+ "is_available",
+ "is_sealed",
+ "sha1",
+ "page_count",
+ "file_size",
+ "filepath_local",
+ "filepath_ia",
+ "ocr_status",
+ ]
+ if get_column_name:
+ columns = [self.add_class_name(col) for col in columns]
+ return columns
+
+ def _get_readable_document_type(self, *args, **kwargs):
+ return self.get_document_type_display()
+
+ def _get_readable_ocr_status(self, *args, **kwargs):
+ return self.get_ocr_status_display()
+
+ def _get_full_filepath_local(self, *args, **kwargs):
+ if self.filepath_local:
+ return f"https://storage.courtlistener.com/{self.filepath_local}"
+ return ""
+
+ def get_column_function(self):
+ """Get dict of attrs: function to apply on field value if it needs
+ to be pre-processed before being add to csv
+ If not functions returns empty dict
+
+ returns: dict -- > {attr1: function}"""
+ return {
+ "document_type": self._get_readable_document_type,
+ "ocr_status": self._get_readable_ocr_status,
+ "filepath_local": self._get_full_filepath_local,
+ }
+
@pghistory.track(AfterUpdateOrDeleteSnapshot(), obj_field=None)
class RECAPDocumentTags(RECAPDocument.tags.through):
@@ -2575,6 +2706,12 @@ class OpinionCluster(AbstractDateTimeModel):
blank=True,
db_index=True,
)
+ filepath_pdf_harvard = models.FileField(
+ help_text="The case PDF from the Caselaw Access Project for this cluster",
+ upload_to=make_upload_path,
+ storage=IncrementingAWSMediaStorage(),
+ blank=True,
+ )
arguments = models.TextField(
help_text="The attorney(s) and legal arguments presented as HTML text. "
"This is primarily seen in older opinions and can contain "
@@ -3096,9 +3233,15 @@ def get_absolute_url(self) -> str:
class Meta:
indexes = [
# To look up individual citations
- models.Index(fields=["volume", "reporter", "page"]),
+ models.Index(
+ fields=["volume", "reporter", "page"],
+ name="search_citation_volume_ae340b5b02e8912_idx",
+ ),
# To generate reporter volume lists
- models.Index(fields=["volume", "reporter"]),
+ models.Index(
+ fields=["volume", "reporter"],
+ name="search_citation_volume_251bc1d270a8abee_idx",
+ ),
]
unique_together = (("cluster", "volume", "reporter", "page"),)
@@ -3320,6 +3463,15 @@ class Opinion(AbstractDateTimeModel):
"sha1",
]
)
+ ordering_key = models.IntegerField(null=True, blank=True)
+
+ class Meta:
+ constraints = [
+ models.UniqueConstraint(
+ fields=["cluster_id", "ordering_key"],
+ name="unique_opinion_ordering_key",
+ )
+ ]
@property
def siblings(self) -> QuerySet:
@@ -3338,6 +3490,10 @@ def get_absolute_url(self) -> str:
def clean(self) -> None:
if self.type == "":
raise ValidationError("'type' is a required field.")
+ if isinstance(self.ordering_key, int) and self.ordering_key < 1:
+ raise ValidationError(
+ {"ordering_key": "Ordering key cannot be zero or negative"}
+ )
def save(
self,
@@ -3346,6 +3502,7 @@ def save(
*args: List,
**kwargs: Dict,
) -> None:
+ self.clean()
super().save(*args, **kwargs)
if index:
from cl.search.tasks import add_items_to_solr
diff --git a/cl/search/signals.py b/cl/search/signals.py
index 2d78f81c20..901588b9ff 100644
--- a/cl/search/signals.py
+++ b/cl/search/signals.py
@@ -280,7 +280,7 @@
"save": {
Docket: {
"self": {
- "case_name": ["caseName"],
+ "case_name": ["caseName", "party"],
"case_name_short": ["caseName"],
"case_name_full": ["case_name_full", "caseName"],
"docket_number": ["docketNumber"],
@@ -329,11 +329,11 @@
RECAPDocument: {
"self": {
"description": ["short_description"],
- "document_type": ["document_type"],
+ "document_type": ["document_type", "absolute_url"],
"document_number": ["document_number", "absolute_url"],
"pacer_doc_id": ["pacer_doc_id"],
"plain_text": ["plain_text"],
- "attachment_number": ["attachment_number"],
+ "attachment_number": ["attachment_number", "absolute_url"],
"is_available": ["is_available"],
"page_count": ["page_count"],
"filepath_local": ["filepath_local"],
@@ -364,6 +364,7 @@
"assigned_to_str": ["assignedTo"],
"referred_to_str": ["referredTo"],
"pacer_case_id": ["pacer_case_id"],
+ "slug": ["absolute_url"],
}
},
Person: {
diff --git a/cl/search/tasks.py b/cl/search/tasks.py
index df7d337f26..c5daeb5073 100644
--- a/cl/search/tasks.py
+++ b/cl/search/tasks.py
@@ -34,7 +34,10 @@
from cl.audio.models import Audio
from cl.celery_init import app
from cl.lib.elasticsearch_utils import build_daterange_query
-from cl.lib.search_index_utils import InvalidDocumentError
+from cl.lib.search_index_utils import (
+ InvalidDocumentError,
+ get_parties_from_case_name,
+)
from cl.people_db.models import Person, Position
from cl.search.documents import (
ES_CHILD_ID,
@@ -451,7 +454,19 @@ def document_fields_to_update(
if prepare_method:
field_value = prepare_method(main_instance)
else:
- field_value = getattr(related_instance, field)
+ if (
+ es_document == DocketDocument
+ and doc_field == "party"
+ ):
+ # Get party from docket case_name if no normalized
+ # parties are available.
+ if main_instance.parties.exists():
+ continue
+ field_value = get_parties_from_case_name(
+ main_instance.case_name
+ )
+ else:
+ field_value = getattr(related_instance, field)
fields_to_update[doc_field] = field_value
else:
# No fields_map is provided, extract field values only using the main
diff --git a/cl/search/templates/advanced.html b/cl/search/templates/advanced.html
index 7dd9d46588..bdbd5d1cc7 100644
--- a/cl/search/templates/advanced.html
+++ b/cl/search/templates/advanced.html
@@ -37,6 +37,7 @@
{% endblock %}
{% block footer-scripts %}
+
{% include "includes/date_picker.html" %}
{% include "includes/date_picker.html" %}
{% if alert_form.errors or request.GET.show_alert_modal %}