Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

maintenance/updates-may #176

Merged
merged 30 commits into from
Jun 26, 2024
Merged
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
3121c27
Detect django apps
jrcarretas Apr 12, 2024
956b914
Grants app cleanup
jrcarretas Apr 12, 2024
6077911
Include linting section in readme
jrcarretas Apr 12, 2024
03a7a9c
More linting fixes
jrcarretas Apr 12, 2024
1e74170
Linting fixes for users app and refactors
jrcarretas Apr 12, 2024
fc94f60
Merge pull request #505 from countable-web/grants-cleanup
jrcarretas Apr 12, 2024
e99f914
Cleaned up web app
jrcarretas Apr 12, 2024
3b943cc
Merge pull request #506 from countable-web/users-cleanup
jrcarretas Apr 12, 2024
fab5b07
Merge pull request #507 from countable-web/web-cleanup
jrcarretas Apr 12, 2024
252821f
Replace mapbox source with FPCC's (previously Countable's)
jrcarretas May 9, 2024
e234aaf
Merge pull request #508 from countable-web/update/mapbox-source
jrcarretas May 9, 2024
dd3d1ec
Cleaned up code in web app
jrcarretas May 10, 2024
4fe908c
Imports clean up
jrcarretas May 10, 2024
4435966
Import clean up
jrcarretas May 10, 2024
5ec8384
Import clean up
jrcarretas May 10, 2024
3d10bd9
Code clean up and fixed useless test
jrcarretas May 10, 2024
8b17339
Code clean up
jrcarretas May 10, 2024
f2d120e
Minor test improvements and cleaned up code
jrcarretas May 10, 2024
51e1e22
Refactors and clean up
jrcarretas May 10, 2024
338182f
Refactors and clean up
jrcarretas May 10, 2024
ba6128a
Minor clean up
jrcarretas May 10, 2024
8ee6917
Merge pull request #509 from countable-web/april-cleanup
jrcarretas May 10, 2024
356e92e
[update] changed language source
bomiku May 11, 2024
43fb936
[update] added hash for files to not cache files
bomiku May 11, 2024
94e916b
[update] if backslash exists, remove it
bomiku May 11, 2024
72a5922
[update] map button adjustments
bomiku May 11, 2024
bb8ee80
Merge pull request #510 from countable-web/updates/may
bomiku May 11, 2024
b1306b8
[update] icon size changed
bomiku May 11, 2024
c969936
Merge pull request #511 from countable-web/updates/may
bomiku May 11, 2024
57d43fe
Merge pull request #512 from countable-web/develop
jrcarretas Jun 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Grants app cleanup
jrcarretas committed Apr 12, 2024
commit 956b914f4e673b349a0efb324b1c4713c3d7b3ab
2 changes: 1 addition & 1 deletion web/grants/admin.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from django.contrib import admin

from .models import Grant, GrantCategory
from grants.models import Grant, GrantCategory


class GrantAdmin(admin.ModelAdmin):
2 changes: 1 addition & 1 deletion web/grants/apps.py
Original file line number Diff line number Diff line change
@@ -2,4 +2,4 @@


class GrantsConfig(AppConfig):
name = 'grants'
name = "grants"
Empty file.
39 changes: 1 addition & 38 deletions web/grants/management/commands/load_grants.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import pandas as pd

from django.core.management.base import BaseCommand

from grants.models import Grant, GrantCategory
from grants.utils import load_grants_from_spreadsheet


class Command(BaseCommand):
@@ -12,38 +10,3 @@ def handle(self, *args, **options):
def add_arguments(self, parser):
parser.add_argument("--file_name", type=str)
parser.add_argument("--test", type=bool)


def load_grants_from_spreadsheet(file_name, test=False):
# Convert excel into dataframe
grants_dataframe = pd.read_excel(file_name, sheet_name=0)

# Replace NaN values with empty string
grants_dataframe.fillna("", inplace=True)

for index, sheet_row in grants_dataframe.iterrows():
category_abbreviation = sheet_row["Grant"].strip().split(" ")[0]

grant_category = GrantCategory.objects.get(abbreviation=category_abbreviation)

if test:
print(sheet_row["Grant"])
continue
else:
print("Saving ", sheet_row["Grant"])

Grant.objects.create(
grant=sheet_row["Grant"],
language=sheet_row["Language"],
year=int(sheet_row["Year"]) if sheet_row["Year"] else None,
recipient=sheet_row["Recipient"],
community_affiliation=sheet_row["Community/Affiliation"],
title=sheet_row["Title"],
project_brief=sheet_row["Project Brief"],
amount=sheet_row["Amount"],
address=sheet_row["Address"],
city=sheet_row["City"],
province=sheet_row["Province"],
postal_code=sheet_row["Postal Code"],
grant_category=grant_category,
)
224 changes: 137 additions & 87 deletions web/grants/management/commands/load_initial_grants.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
import os
import csv
import pandas as pd

from django.core.management.base import BaseCommand
from django.contrib.gis.geos import Point

from web import dedruplify
from grants.models import Grant
from grants.constants import CATEGORY_ABBREVIATIONS, CATEGORY_NAMES, NEW_CATEGORIES
from grants.constants import CATEGORY_ABBREVIATIONS, NEW_CATEGORIES
from grants.utils import load_grants_from_spreadsheet


@@ -40,76 +39,102 @@ def load_grants_from_arts_db(self):

grant_data = {}

writer = csv.writer(open('old-arts-grants.csv', 'w'))
writer.writerow(["Year", "Grant", "Language", "Recipient", "Community/Affiliation", "Title", "Project Brief",
"Amount", "Address", "City", "Province", "Postal Code", "Status", "Category", "Node ID"])

for grant in grants:
grant_id = grant.get("nid", "")
grant_title = grant.get("title", "")

grant_data["year"] = self.get_year(grant_id)
grant_data["affiliation"] = self.get_affiliation(grant_id)
grant_data["project_brief"] = self.get_project_brief(grant_id)

grant_data["recipient"] = self.update_recipient(
self.get_recipient(grant_id), grant_title, grant_id)
grant_data["category"] = self.update_category(
self.get_category(grant_id))

location = self.get_location(grant_id)
grant_data["address"] = location.get(
"address", "") if location else ""
grant_data["city"] = location.get("city", "") if location else ""
grant_data["province"] = location.get(
"province", "") if location else ""
grant_data["postal_code"] = location.get(
"postal_code", "") if location else ""

updated_grant_title = self.update_grant_title(
grant_data["category"], grant_data["year"], grant_title)
grant_data["grant"] = updated_grant_title

writer.writerow([
f"{grant_data['year']}",
f"{grant_data['grant']}",
"", # For language
f"{grant_data['recipient']}",
f"{grant_data['affiliation']}",
f"{grant_title }", # For title
f"{grant_data['project_brief']}",
"", # For Amount
f"{grant_data['address']}",
f"{grant_data['city']}",
f"{grant_data['province']}",
f"{grant_data['postal_code']}",
"", # For status
f"{grant_data['category']}",
f"{grant_id}"
])

point = None
if location and \
location.get("latitude", None) and \
location.get("longitude", None):
point = Point(
float(location.get("longitude")),
float(location.get("latitude"))
with open("old-arts-grants.csv", "w", encoding="utf-8") as file:
writer = csv.writer(file)
writer.writerow(
[
"Year",
"Grant",
"Language",
"Recipient",
"Community/Affiliation",
"Title",
"Project Brief",
"Amount",
"Address",
"City",
"Province",
"Postal Code",
"Status",
"Category",
"Node ID",
]
)

for grant in grants:
grant_id = grant.get("nid", "")
grant_title = grant.get("title", "")

grant_data["year"] = self.get_year(grant_id)
grant_data["affiliation"] = self.get_affiliation(grant_id)
grant_data["project_brief"] = self.get_project_brief(grant_id)

grant_data["recipient"] = self.update_recipient(
self.get_recipient(grant_id), grant_title, grant_id
)
grant_data["category"] = self.update_category(
self.get_category(grant_id)
)

Grant.objects.create(
grant=grant_data['grant'],
year=int(grant_data["year"]) if grant_data["year"] else None,
recipient=grant_data["recipient"],
community_affiliation=grant_data["affiliation"],
project_brief=grant_data["project_brief"],
address=grant_data["address"],
city=grant_data["city"],
province=grant_data["province"],
postal_code=grant_data["postal_code"],
category=grant_data["category"],
point=point
)
location = self.get_location(grant_id)
grant_data["address"] = location.get("address", "") if location else ""
grant_data["city"] = location.get("city", "") if location else ""
grant_data["province"] = (
location.get("province", "") if location else ""
)
grant_data["postal_code"] = (
location.get("postal_code", "") if location else ""
)

updated_grant_title = self.update_grant_title(
grant_data["category"], grant_data["year"], grant_title
)
grant_data["grant"] = updated_grant_title

writer.writerow(
[
f"{grant_data['year']}",
f"{grant_data['grant']}",
"", # For language
f"{grant_data['recipient']}",
f"{grant_data['affiliation']}",
f"{grant_title }", # For title
f"{grant_data['project_brief']}",
"", # For Amount
f"{grant_data['address']}",
f"{grant_data['city']}",
f"{grant_data['province']}",
f"{grant_data['postal_code']}",
"", # For status
f"{grant_data['category']}",
f"{grant_id}",
]
)

point = None
if (
location
and location.get("latitude", None)
and location.get("longitude", None)
):
point = Point(
float(location.get("longitude")),
float(location.get("latitude")),
)

Grant.objects.create(
grant=grant_data["grant"],
year=int(grant_data["year"]) if grant_data["year"] else None,
recipient=grant_data["recipient"],
community_affiliation=grant_data["affiliation"],
project_brief=grant_data["project_brief"],
address=grant_data["address"],
city=grant_data["city"],
province=grant_data["province"],
postal_code=grant_data["postal_code"],
category=grant_data["category"],
point=point,
)

def update_recipient(self, recipient, title, grant_id):
is_new_grant = grant_id > 1190
@@ -144,7 +169,8 @@ def get_grants(self):
return self.query("SELECT nid, title FROM node WHERE type='grant';")

def get_year(self, grant_id):
year = self.query("""
year = self.query(
"""
SELECT
field_grant_award_year_value
FROM
@@ -155,28 +181,38 @@ def get_year(self, grant_id):
nid=entity_id
WHERE
nid=%s;
""" % grant_id)
"""
% grant_id
)
if year:
return year[0].get("field_grant_award_year_value") if year[0].get("field_grant_award_year_value") else ""
return (
year[0].get("field_grant_award_year_value")
if year[0].get("field_grant_award_year_value")
else ""
)

return ""

def get_affiliation(self, grant_id):
affiliation = self.query("""
affiliation = self.query(
"""
SELECT
field_grant_affiliation_value
FROM
field_data_field_grant_affiliation
where
entity_id=%s;
""" % grant_id)
"""
% grant_id
)
if affiliation:
return affiliation[0].get("field_grant_affiliation_value", "")

return ""

def get_recipient(self, grant_id):
recipient = self.query("""
recipient = self.query(
"""
SELECT
title
FROM
@@ -185,48 +221,57 @@ def get_recipient(self, grant_id):
SELECT
entity_id, title, type
FROM
field_data_field_grant_recipient, node
field_data_field_grant_recipient, node
WHERE
field_grant_recipient_nid=nid
) AS recipient_node
WHERE
field_data_field_grant_recipient.entity_id = recipient_node.entity_id
AND
recipient_node.entity_id=%s;
""" % grant_id)
"""
% grant_id
)
if recipient:
return recipient[0].get("title", "")

recipient = self.query("""
recipient = self.query(
"""
SELECT
field_grant_recipient_text_value
FROM
field_data_field_grant_recipient_text
WHERE
entity_id=%s;
""" % grant_id)
"""
% grant_id
)

if recipient:
return recipient[0].get("title", "")

return ""

def get_project_brief(self, grant_id):
project_brief = self.query("""
project_brief = self.query(
"""
SELECT
body_value
FROM
field_data_body
WHERE
entity_id=%s;
""" % grant_id)
"""
% grant_id
)
if project_brief:
return project_brief[0].get("body_value", "")

return ""

def get_location(self, grant_id):
location = self.query("""
location = self.query(
"""
SELECT
latitude, longitude, street, city, province, postal_code
FROM
@@ -237,21 +282,24 @@ def get_location(self, grant_id):
lid=field_shared_location_lid
WHERE
entity_id=%s;
""" % grant_id)
"""
% grant_id
)
if location:
return {
"latitude": location[0].get("latitude", ""),
"longitude": location[0].get("longitude", ""),
"address": location[0].get("street", ""),
"city": location[0].get("city", ""),
"province": location[0].get("province", ""),
"postal_code": location[0].get("postal_code", "")
"postal_code": location[0].get("postal_code", ""),
}

return None

def get_category(self, grant_id):
category = self.query("""
category = self.query(
"""
SELECT
name
FROM
@@ -262,7 +310,9 @@ def get_category(self, grant_id):
field_grant_category_tid=tid
where
entity_id=%s;
""" % grant_id)
"""
% grant_id
)
if category:
return category[0].get("name", "")

Original file line number Diff line number Diff line change
@@ -128,7 +128,6 @@ def get_values(model, start_id):
"&", search_value, all_values, model
)

value_list = []
for _id in set(possible_value_ids):
if model == "language":
obj = Language.objects.get(pk=_id)
45 changes: 18 additions & 27 deletions web/grants/models.py
Original file line number Diff line number Diff line change
@@ -4,21 +4,16 @@
from language.models import Language, Community, PlaceName


optional = {
'blank': True,
'null': True
}
optional = {"blank": True, "null": True}


class GrantCategory(models.Model):
name = models.TextField()
abbreviation = models.CharField(max_length=255)
order = models.IntegerField(default=None, **optional)
parent = models.ForeignKey(
'self',
on_delete=models.SET_NULL,
related_name='child_categories',
**optional)
"self", on_delete=models.SET_NULL, related_name="child_categories", **optional
)

def __str__(self):
return "{} ({})".format(self.name, self.abbreviation)
@@ -29,40 +24,36 @@ class Meta:

class Grant(models.Model):
grant = models.CharField(max_length=255, **optional)
year = models.IntegerField(
validators=[MinValueValidator(0)],
**optional)
year = models.IntegerField(validators=[MinValueValidator(0)], **optional)
language = models.CharField(max_length=255, **optional)
recipient = models.TextField(**optional)
community_affiliation = models.TextField(
verbose_name='Community/Affiliation',
**optional)
languages = models.ManyToManyField(
Language,
related_name='grants',
blank=True)
recipients = models.ManyToManyField(
PlaceName,
related_name='grants',
blank=True)
verbose_name="Community/Affiliation", **optional
)
languages = models.ManyToManyField(Language, related_name="grants", blank=True)
recipients = models.ManyToManyField(PlaceName, related_name="grants", blank=True)
communities_affiliations = models.ManyToManyField(
Community,
related_name='grants',
verbose_name='Communities/Affiliations',
blank=True)
related_name="grants",
verbose_name="Communities/Affiliations",
blank=True,
)
title = models.TextField(**optional)
project_brief = models.TextField(**optional)
amount = models.DecimalField(
validators=[MinValueValidator(0.0)],
default=0.0,
decimal_places=2,
max_digits=100,
**optional)
**optional
)
address = models.TextField(**optional)
city = models.CharField(max_length=255, **optional)
province = models.CharField(max_length=255, **optional)
postal_code = models.CharField(max_length=255, **optional)
grant_category = models.ForeignKey(GrantCategory, on_delete=models.SET_NULL, **optional)
grant_category = models.ForeignKey(
GrantCategory, on_delete=models.SET_NULL, **optional
)
point = models.PointField(null=True, default=None)
modified = models.DateTimeField("date modified", auto_now=True)
created = models.DateTimeField("date created", auto_now_add=True)
@@ -71,4 +62,4 @@ def __str__(self):
return self.grant

class Meta:
ordering = ['grant']
ordering = ["grant"]
22 changes: 15 additions & 7 deletions web/grants/serializers.py
Original file line number Diff line number Diff line change
@@ -5,21 +5,29 @@


class GrantSerializer(GeoFeatureModelSerializer):
category = serializers.CharField(
source="grant_category.name", allow_null=True)
category = serializers.CharField(source="grant_category.name", allow_null=True)
category_abbreviation = serializers.CharField(
source="grant_category.abbreviation", allow_null=True)
source="grant_category.abbreviation", allow_null=True
)

class Meta:
model = Grant
fields = ("id", "grant", "recipient", "project_brief", "category", "amount",
"category_abbreviation", "community_affiliation", "year")
fields = (
"id",
"grant",
"recipient",
"project_brief",
"category",
"amount",
"category_abbreviation",
"community_affiliation",
"year",
)
geo_field = "point"


class GrantDetailSerializer(serializers.ModelSerializer):
category = serializers.CharField(
source='grant_category.name', allow_null=True)
category = serializers.CharField(source="grant_category.name", allow_null=True)

class Meta:
model = Grant
2 changes: 0 additions & 2 deletions web/grants/tests.py
Original file line number Diff line number Diff line change
@@ -1,3 +1 @@
from django.test import TestCase

# Create your tests here.
41 changes: 40 additions & 1 deletion web/grants/utils.py
Original file line number Diff line number Diff line change
@@ -1 +1,40 @@
# Grants utilities
# Grants utilities
import pandas as pd


from grants.models import Grant, GrantCategory


def load_grants_from_spreadsheet(file_name, test=False):
# Convert excel into dataframe
grants_dataframe = pd.read_excel(file_name, sheet_name=0)

# Replace NaN values with empty string
grants_dataframe.fillna("", inplace=True)

for index, sheet_row in grants_dataframe.iterrows():
category_abbreviation = sheet_row["Grant"].strip().split(" ")[0]

grant_category = GrantCategory.objects.get(abbreviation=category_abbreviation)

if test:
print(sheet_row["Grant"])
continue
else:
print("Saving ", sheet_row["Grant"])

Grant.objects.create(
grant=sheet_row["Grant"],
language=sheet_row["Language"],
year=int(sheet_row["Year"]) if sheet_row["Year"] else None,
recipient=sheet_row["Recipient"],
community_affiliation=sheet_row["Community/Affiliation"],
title=sheet_row["Title"],
project_brief=sheet_row["Project Brief"],
amount=sheet_row["Amount"],
address=sheet_row["Address"],
city=sheet_row["City"],
province=sheet_row["Province"],
postal_code=sheet_row["Postal Code"],
grant_category=grant_category,
)
2 changes: 1 addition & 1 deletion web/grants/views/base.py
Original file line number Diff line number Diff line change
@@ -11,4 +11,4 @@ def get_serializer_class(self):
if hasattr(self, "detail_serializer_class"):
return self.detail_serializer_class

return super().get_serializer_class()
return super().get_serializer_class()
32 changes: 20 additions & 12 deletions web/grants/views/grant.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
from django.db.models import F
from django.utils.decorators import method_decorator
from django.views.decorators.cache import never_cache
from rest_framework import mixins, views
from rest_framework.response import Response
from rest_framework import mixins

from grants.models import Grant, GrantCategory
from grants.serializers import GrantSerializer, GrantDetailSerializer, GrantCategorySerializer
from grants.serializers import (
GrantSerializer,
GrantDetailSerializer,
GrantCategorySerializer,
)
from .base import BaseGenericViewSet


class GrantViewSet(mixins.ListModelMixin,
mixins.RetrieveModelMixin,
BaseGenericViewSet):
queryset = Grant.objects.filter(point__isnull=False, year__isnull=False).order_by('-year', 'recipient', 'grant')
class GrantViewSet(
mixins.ListModelMixin, mixins.RetrieveModelMixin, BaseGenericViewSet
):
queryset = Grant.objects.filter(point__isnull=False, year__isnull=False).order_by(
"-year", "recipient", "grant"
)
serializer_class = GrantSerializer
detail_serializer_class = GrantDetailSerializer
lookup_field = 'id'
lookup_field = "id"

def list(self, request, *args, **kwargs):
return super().list(request)
@@ -24,8 +27,13 @@ def list(self, request, *args, **kwargs):
class GrantCategoryViewSet(mixins.ListModelMixin, BaseGenericViewSet):
serializer_class = GrantCategorySerializer
queryset = GrantCategory.objects.all().order_by(
F('parent__id',).asc(nulls_first=True),
F('order',).asc(nulls_last=True))
F(
"parent__id",
).asc(nulls_first=True),
F(
"order",
).asc(nulls_last=True),
)

def list(self, request, *args, **kwargs):
return super().list(request)