From 0a57170d979b848f13c98fbdff5e1e6ae498de23 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 17 Mar 2023 14:52:30 +0100 Subject: [PATCH] prepare release (#112) * add error handling via email for attachment upload (#111) * update readme for attachment upload --- README.md | 6 +- .../register-attachment-dropbox.py | 191 ++++++++++++------ 2 files changed, 131 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 2222a036..c1b2f3be 100644 --- a/README.md +++ b/README.md @@ -212,11 +212,15 @@ Incoming structure overview: Metadata is expected to be denoted in line-separated key-value pairs, where key and value are separated by a '='. The following structure/pairs are expected: ``` -user= +user= info= barcode= type= ``` +If a university user name is provided and the registration of data fails, the user will receive an email containing the error. + +The info field must not contain line breaks, as each line in the metadata file must contain a key-value pair. + The code of the attachment sample is built from the project code followed by three zeroes, conforming to the regular expression "Q[A-Z0-9]{4}000", e.g. QABCD000. See code examples: diff --git a/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py b/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py index ae4615b2..8baef2e6 100755 --- a/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py +++ b/drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py @@ -22,81 +22,142 @@ # *Q[Project Code]^4000*.* # *Q[Project Code]^4E[Experiment Number]-000* # IMPORTANT: ONLY PROJECT LEVEL WORKING RIGHT NOW -ppattern = re.compile('Q\w{4}000') +#ppattern = re.compile('Q\w{4}000') #epattern = re.compile('Q\w{4}E[1-9][0-9]*') +# contains properties used to send emails, e.g. the mail server and the "from" address to use +import email_helper_qbic as email_helper +# provides functionality used to send emails, e.g. about registration errors +import etl_mailer + +class MetadataFormattingException(Exception): + "Thrown when metadata file cannot be successfully parsed." + pass + +class NoSamplesFoundForProjectException(Exception): + "Thrown when sample cannot be found in openBIS." + pass + +class CouldNotCreateException(Exception): + "Thrown when necessary experiment or sample could not be created." + pass + +def getInfoExperimentIdentifier(space, project): + experimentID = '/' + space + '/' + project + '/'+ project+'_INFO' + return experimentID + +def extractProjectCode(sampleCode): + return sampleCode[:5] + def process(transaction): + error = None + originalError = None context = transaction.getRegistrationContext().getPersistentMap() # Get the incoming path of the transaction incomingPath = transaction.getIncoming().getAbsolutePath() - key = context.get("RETRY_COUNT") - if (key == None): - key = 1 + try: + #read in the metadata file + for f in os.listdir(incomingPath): + if f == "metadata.txt": + metadata = open(os.path.join(incomingPath, f)) + fileInfo = dict() + for line in metadata: + try: + pair = line.strip().split('=') + fileInfo[pair[0]] = pair[1] + except IndexError as exception: + originalError = exception + error = MetadataFormattingException("Metadata file not correctly formatted. Check for additional line breaks.") + metadata.close() + try: + user = fileInfo["user"] + except: + user = None + secname = fileInfo["info"] + sampleCode = fileInfo["barcode"] + datasetType = fileInfo["type"] + else: + name = f - #read in the metadata file - for f in os.listdir(incomingPath): - if f == "metadata.txt": - metadata = open(os.path.join(incomingPath, f)) - fileInfo = dict(line.strip().split('=') for line in metadata) - metadata.close() - try: - user = fileInfo["user"] - except: - user = None - secname = fileInfo["info"] - code = fileInfo["barcode"] - datasetType = fileInfo["type"] - else: - name = f - - project = code[:5] - type = "INFORMATION" - if "Results" in datasetType: - type = "RESULT" - - if user: - transaction.setUserId(user) - - inputFile = os.path.join(incomingPath, name) - newname = urllib.unquote(name) - dataFile = os.path.join(incomingPath, newname) - print "renaming "+inputFile+" to "+dataFile - os.rename(inputFile, dataFile) - - search_service = transaction.getSearchService() - sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code)) - foundSamples = search_service.searchForSamples(sc) - sample = None - space = None - sa = None - attachmentReady = True - - if len(foundSamples) == 0: - attachmentReady = False + project = extractProjectCode(sampleCode) + type = "INFORMATION" + if "Results" in datasetType: + type = "RESULT" + if error: + raise error + if user: + transaction.setUserId(user) + + inputFile = os.path.join(incomingPath, name) + newname = urllib.unquote(name) + dataFile = os.path.join(incomingPath, newname) + print "renaming "+inputFile+" to "+dataFile + os.rename(inputFile, dataFile) + + search_service = transaction.getSearchService() sc = SearchCriteria() - sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, project+"ENTITY-1")) + sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode)) foundSamples = search_service.searchForSamples(sc) - sample = foundSamples[0] - sampleID = sample.getSampleIdentifier() - sa = transaction.getSampleForUpdate(sampleID) - space = sa.getSpace() - if not attachmentReady: - infoSampleID = "/"+space+"/"+code - sa = transaction.getSampleForUpdate(infoSampleID) - if not sa: - exp = transaction.createNewExperiment('/' + space + '/' + project + '/'+ project+'_INFO', "Q_PROJECT_DETAILS") - sa = transaction.createNewSample('/' + space + '/'+ code, "Q_ATTACHMENT_SAMPLE") - sa.setExperiment(exp) - info = None - - dataSet = transaction.createNewDataSet("Q_PROJECT_DATA") - dataSet.setMeasuredData(False) - dataSet.setPropertyValue("Q_SECONDARY_NAME", secname) - dataSet.setPropertyValue("Q_ATTACHMENT_TYPE", type) - dataSet.setSample(sa) - transaction.moveFile(dataFile, dataSet) + sample = None + space = None + sa = None + attachmentSampleFound = True + if len(foundSamples) == 0: + attachmentSampleFound = False + sc = SearchCriteria() + sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, project+"ENTITY-1")) + foundSamples = search_service.searchForSamples(sc) + try: + sample = foundSamples[0] + except IndexError as exception: + originalError = exception + error = NoSamplesFoundForProjectException("No sample could be found for this project.") + sampleID = sample.getSampleIdentifier() + sa = transaction.getSampleForUpdate(sampleID) + space = sa.getSpace() + if not attachmentSampleFound: + # fetch it by name + infoSampleID = "/"+space+"/"+sampleCode + sa = transaction.getSampleForUpdate(infoSampleID) + if not sa: + # create necessary objects if sample really doesn't exist + try: + experimentID = getInfoExperimentIdentifier(space, project) + exp = transaction.createNewExperiment(experimentID, "Q_PROJECT_DETAILS") + except Exception as exception: + originalError = exception + error = CouldNotCreateException("Experiment "+experimentID+" could not be created.") + try: + sa = transaction.createNewSample(infoSampleID, "Q_ATTACHMENT_SAMPLE") + except Exception as exception: + originalError = exception + error = CouldNotCreateException("Sample "+infoSampleID+" was not found and could not be created.") + sa.setExperiment(exp) + dataSet = transaction.createNewDataSet("Q_PROJECT_DATA") + dataSet.setMeasuredData(False) + dataSet.setPropertyValue("Q_SECONDARY_NAME", secname) + dataSet.setPropertyValue("Q_ATTACHMENT_TYPE", type) + dataSet.setSample(sa) + transaction.moveFile(dataFile, dataSet) + # catch other, unknown exceptions + except Exception as exception: + originalError = exception + if not error: + error = Exception("Unknown exception occured: "+str(exception)) + if originalError: + # if there is a problem sending the email, we log this and raise the original exception + try: + mailFactory = etl_mailer.EmailFactory() + etlMailer = etl_mailer.ETLMailer(email_helper.get_mail_host(), email_helper.get_mail_server(), email_helper.get_mail_from()) + + subject = "Error storing project attachment uploaded for "+project + content = mailFactory.formatRegistrationErrorContent(str(error)) + etlMailer.send_email([user], subject, content) + except Exception as mailException: + print "Could not send error email: "+str(mailException) + pass + raise originalError