Skip to content

Commit

Permalink
added update_item functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
@khoroshevskyi
khoroshevskyi committed Dec 21, 2022
1 parent 6e17ef3 commit 6658ad4
Show file tree
Hide file tree
Showing 5 changed files with 149 additions and 90 deletions.
39 changes: 16 additions & 23 deletions manual_tests.py
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,25 @@
# file for manual local tests

import sqlmodel
import pepdbagent
from pydantic import BaseModel
from typing import Optional


# class Annot(BaseModel):
# name: str
# surname: str
#
# class Model(BaseModel):
# person: str
# guest: Optional[str]
# annot: Annot
#
#
#
# ff = Model(**fff)
# print(ff.dict(exclude_none=True))
from peppy import Project


con = pepdbagent.Connection(user='postgres',
password='docker',
)
password='docker',
)

# proj = Project('/home/bnt4me/virginia/repos/pepdbagent/sample_pep/subtable1/project_config.yaml')

con.get_project(namespace="new", name="GSE220436", tag='raw')
# con.updload_project(namespace="test", name='sub', tag='f', project=proj, is_private=True)

dd = con.search.project(namespace="kk", search_str='de', admin=False)

print(dd)
ff = con.update_item(namespace="test", name="sub", tag='f', update_dict={'private': True,
'annot': {'description': "this is my description",
'something': 'else'},
'digest': "dfdfdf",

})

# res = con.search.project(namespace='kk',search_str='d', admin=True)
res = con.search.namespace('i', admin_nsp=('kk',))
print(res)
11 changes: 1 addition & 10 deletions pepdbagent/const.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,10 @@
DEFAULT_NAMESPACE = "_"
DEFAULT_TAG = "default"

# STATUS_KEY = "status"
DESCRIPTION_KEY = "description"
N_SAMPLES_KEY = "n_samples"
N_SAMPLES_KEY = "number_of_samples"
UPDATE_DATE_KEY = "last_update"
IS_PRIVATE_KEY = "is_private"
# DEFAULT_STATUS = "Unknown"

# BASE_ANNOTATION_DICT = {
# STATUS_KEY: DEFAULT_STATUS,
# DESCRIPTION_KEY: None,
# N_SAMPLES_KEY: None,
# UPDATE_DATE_KEY: None,
# }

from peppy.const import SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_DICT_KEY

Expand Down
33 changes: 29 additions & 4 deletions pepdbagent/models.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from typing import List, Optional, Union

from pydantic import BaseModel, Field, validator
from pydantic import BaseModel, Field, validator, Extra
from .pepannot import Annotation


class Model(BaseModel):
Expand Down Expand Up @@ -29,9 +30,9 @@ class NamespacesResponseModel(Model):


class NamespaceSearchResultModel(Model):
namespace = str
number_of_projects = int
number_of_samples = int
namespace: str
number_of_projects: int
number_of_samples: int


class ProjectSearchResultModel(Model):
Expand Down Expand Up @@ -66,6 +67,30 @@ class ProjectSearchModel(Model):
results: List[ProjectSearchResultModel]


class RawPEPModel(BaseModel):
name: str
description: str
_config: dict
_sample_dict: dict
_subsample_dict: Optional[list] = None

class Config:
extra = Extra.forbid


class UpdateModel(BaseModel):
name: Optional[str]
namespace: Optional[str]
tag: Optional[str]
digest: Optional[str]
project_value: Optional[RawPEPModel]
private: Optional[bool]
anno_info: Optional[Annotation] = Field(alias="annot")

class Config:
extra = Extra.forbid


class UploadResponse(Model):
registry_path: str
log_stage: str
Expand Down
152 changes: 101 additions & 51 deletions pepdbagent/pepdbagent.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import datetime
import json
from hashlib import md5
from typing import List, NoReturn, Union, Dict
from typing import List, NoReturn, Union, Tuple
from urllib.parse import urlparse

import coloredlogs
Expand All @@ -17,6 +17,7 @@
NamespacesResponseModel,
ProjectModel,
UploadResponse,
UpdateModel,
)
from .search import Search
from .const import *
Expand Down Expand Up @@ -96,7 +97,6 @@ def upload_project(
namespace: str = None,
name: str = None,
tag: str = None,
description: str = None,
is_private: bool = False,
overwrite: bool = False,
update_only: bool = False,
Expand All @@ -109,7 +109,6 @@ def upload_project(
:param namespace: namespace of the project (Default: 'other')
:param name: name of the project (Default: name is taken from the project object)
:param tag: tag (or version) of the project
:param description: description of the project
:param is_private: boolean value if the project should be visible just for user that creates it
:param overwrite: if project exists overwrite the project, otherwise upload it.
[Default: False - project won't be overwritten if it exists in db]
Expand All @@ -133,7 +132,7 @@ def upload_project(

# creating annotation:
proj_annot = Annotation(
description=description,
description=proj_dict.get('description'),
last_update=str(datetime.datetime.now()),
n_samples=len(project.samples),
)
Expand Down Expand Up @@ -305,7 +304,6 @@ def _update_project(
info="project does not exist!",
)


def update_item(
self,
update_dict: dict,
Expand All @@ -314,46 +312,98 @@ def update_item(
tag: str,
) -> UploadResponse:
"""
Update partial parts of the project record
:param update_dict: dict with update key->values. Dict structure:
{
name: Optional[str]
namespace: Optional[str]
tag: Optional[str]
digest: Optional[str]
project_value: dict
private: Optional[bool]
anno_info: dict
}
:param namespace: project namespace
:param name: project name
:param tag: project tag
:return: ResponseModel with information if project was updated
"""
cursor = self.pg_connection.cursor()

if self.project_exists(namespace=namespace, name=name, tag=tag):
try:

set_sql, set_values = self.__create_update_set(UpdateModel(**update_dict))
sql = f"""UPDATE {DB_TABLE_NAME}
{set_sql}
WHERE {NAMESPACE_COL} = %s and {NAME_COL} = %s and {TAG_COL} = %s;"""
_LOGGER.debug("Updating items...")
cursor.execute(
sql,
(*set_values, namespace, name, tag),
)
_LOGGER.info(f"Record '{namespace}/{name}:{tag}' was successfully updated!")
self._commit_to_database()

except Exception as err:
_LOGGER.error(f"Error while updating project! Error: {err}")
return UploadResponse(
registry_path=f"{namespace}/{name}:{tag}",
log_stage="update_item",
status="failure",
info=f"Error in executing SQL. {err}!",
)
else:
_LOGGER.error("Project does not exist! No project will be updated!")
return UploadResponse(
registry_path=f"{namespace}/{name}:{tag}",
log_stage="update_item",
status="failure",
info="Project does not exist!",
)

return UploadResponse(
registry_path=f"{namespace}/{name}:{tag}",
log_stage="update_item",
status="success",
info="Record was successfully updated!",
)

@staticmethod
def __create_update_set(update_info: UpdateModel) -> Tuple[str, tuple]:
"""
Create sql SET string by passing UpdateModel that later is converted to dict
:param update_info: UpdateModel (similar to database model)
:return: {sql_string (contains db keys) and updating values}
"""
_LOGGER.debug("Creating SET SQL string to update project")
sql_string = f"""SET """
sql_values = []

first = True
for key, val in update_info.dict(exclude_none=True).items():
if first:
sql_string = ''.join([sql_string, f"{key} = %s"])
first = False
else:
sql_string = ', '.join([sql_string, f"{key} = %s"])

if isinstance(val, dict):
input_val = json.dumps(val)
else:
input_val = val

sql_values.append(input_val)

# if not isinstance(val, dict):
# else:
# keys_str = ""
# for key1, val1 in val.items():
# keys_str = ", ".join([keys_str, f""" '{{{str(key1)}}}', %s"""])
# sql_values.append(f'"{val1}"')
# sql_string = ', '.join([sql_string, f"""{key} = jsonb_set({key} {keys_str})"""])

:param update_dict:
:param namespace:
:param name:
:param tag:
:return:
"""
"""
{digest: ...,
project: {
"""
pass
# cursor = self.pg_connection.cursor()
#
# if self.project_exists(namespace=namespace, name=name, tag=tag):
# try:
#
# # TODO: create here function that creates set ...
# set_sql = "SET {DIGEST_COL} = %s, {PROJ_COL}= %s, {ANNO_COL}= %s"
# set_sql_values = ["value1", "value2"]
# sql = f"""UPDATE {DB_TABLE_NAME}
# {set_sql}
# WHERE {NAMESPACE_COL} = %s and {NAME_COL} = %s and {TAG_COL} = %s;"""
# cursor.execute(
# sql,
# (
#
# ),
# )
# self._commit_to_database()
#
# _LOGGER.error("Project does not exist! No project will be updated!")
# except Exception:
# pass
# return UploadResponse(
# registry_path=f"{namespace}/{name}:{tag}",
# log_stage="update_project",
# status="failure",
# info="project does not exist!",
# )
return sql_string, tuple(sql_values)

def delete_project(
self,
Expand Down Expand Up @@ -572,21 +622,21 @@ def get_namespace_info(self, namespace: str, user: str = None):
projects:(id, name, tag, digest, description, n_samples)
"""
try:
sql_q = f"select {ID_COL}, {NAME_COL}, {TAG_COL}, {DIGEST_COL}, {ANNO_COL}, {PRIVATE_COL} from {DB_TABLE_NAME} where {NAMESPACE_COL} = %s"
sql_q = f"""select {ID_COL}, {NAME_COL}, {TAG_COL}, {DIGEST_COL}, {PRIVATE_COL}, {ANNO_COL}->>'{N_SAMPLES_KEY}', {PROJ_COL}->>'description'
from {DB_TABLE_NAME} where {NAMESPACE_COL} = %s"""
results = self._run_sql_fetchall(sql_q, namespace)

projects = []
for project_data in results:
annotation = Annotation(**project_data[4])
projects.append(
ProjectModel(
id=project_data[0],
name=project_data[1],
tag=project_data[2],
digest=project_data[3],
description=annotation.description,
number_of_samples=annotation.number_of_samples,
is_private=project_data[5],
description=project_data[6],
number_of_samples=project_data[5],
is_private=project_data[4],
)
)
namespace = NamespaceModel(
Expand Down Expand Up @@ -689,7 +739,7 @@ def get_project_annotation(
{NAMESPACE_COL},
{NAME_COL},
{TAG_COL},
{ANNO_COL}
{ANNO_COL},
from {DB_TABLE_NAME}
"""
if namespace is None:
Expand All @@ -712,7 +762,7 @@ def get_project_annotation(

annot = Annotation(
registry=f"{found_prj[0]}/{found_prj[1]}:{found_prj[2]}",
annotation_dict=found_prj[3],
**ANNO_COL
)

return annot
Expand Down
4 changes: 2 additions & 2 deletions pepdbagent/search.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ def _count_find_namespaces(
"""
count_sql = f"""
select COUNT(DISTINCT ({NAMESPACE_COL}))
from {DB_TABLE_NAME} where (({NAMESPACE_COL} ILIKE '%%{search_str}%%' and {PRIVATE_COL})
from {DB_TABLE_NAME} where (({NAMESPACE_COL} ILIKE '%%{search_str}%%' and {PRIVATE_COL} = false)
or ({NAMESPACE_COL} ILIKE '%%{search_str}%%' and {NAMESPACE_COL} in %s ))
"""
result = self.__run_sql_fetchall(count_sql, admin_nsp)
Expand Down Expand Up @@ -209,7 +209,7 @@ def _find_namespaces(
"""
count_sql = f"""
select {NAMESPACE_COL}, COUNT({NAME_COL}), SUM( ({ANNO_COL}->>'number_of_samples')::int)
from {DB_TABLE_NAME} where (({NAMESPACE_COL} ILIKE '%%{search_str}%%' and {PRIVATE_COL})
from {DB_TABLE_NAME} where (({NAMESPACE_COL} ILIKE '%%{search_str}%%' and {PRIVATE_COL} is false)
or ({NAMESPACE_COL} ILIKE '%%{search_str}%%' and {NAMESPACE_COL} in %s ))
GROUP BY {NAMESPACE_COL}
LIMIT {limit} OFFSET {offset};
Expand Down

0 comments on commit 6658ad4

Please sign in to comment.