Skip to content

Commit

Permalink
prepare release (#112)
Browse files Browse the repository at this point in the history 
* add error handling via email for attachment upload (#111)

* update readme for attachment upload
  • Loading branch information
@wow-such-code
wow-such-code authored Mar 17, 2023
1 parent df5b42d commit 0a57170
Show file tree
Hide file tree
Showing 2 changed files with 131 additions and 66 deletions.
6 changes: 5 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,11 +212,15 @@ Incoming structure overview:
Metadata is expected to be denoted in line-separated key-value pairs, where key and value are separated by a '='. The following structure/pairs are expected:

```
user=<the (optional) uploading user name>
user=<the (optional) uploading user name.>
info=<short info about the file>
barcode=<the sample code of the attachment sample>
type=<the type of attachment: information or results>
```
If a university user name is provided and the registration of data fails, the user will receive an email containing the error.

The info field must not contain line breaks, as each line in the metadata file must contain a key-value pair.

The code of the attachment sample is built from the project code followed by three zeroes, conforming to the regular expression "Q[A-Z0-9]{4}000", e.g. QABCD000.

See code examples:
Expand Down
191 changes: 126 additions & 65 deletions drop-boxes/register-attachments-dropbox/register-attachment-dropbox.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,81 +22,142 @@
# *Q[Project Code]^4000*.*
# *Q[Project Code]^4E[Experiment Number]-000*
# IMPORTANT: ONLY PROJECT LEVEL WORKING RIGHT NOW
ppattern = re.compile('Q\w{4}000')
#ppattern = re.compile('Q\w{4}000')
#epattern = re.compile('Q\w{4}E[1-9][0-9]*')

# contains properties used to send emails, e.g. the mail server and the "from" address to use
import email_helper_qbic as email_helper
# provides functionality used to send emails, e.g. about registration errors
import etl_mailer

class MetadataFormattingException(Exception):
"Thrown when metadata file cannot be successfully parsed."
pass

class NoSamplesFoundForProjectException(Exception):
"Thrown when sample cannot be found in openBIS."
pass

class CouldNotCreateException(Exception):
"Thrown when necessary experiment or sample could not be created."
pass

def getInfoExperimentIdentifier(space, project):
experimentID = '/' + space + '/' + project + '/'+ project+'_INFO'
return experimentID

def extractProjectCode(sampleCode):
return sampleCode[:5]

def process(transaction):
error = None
originalError = None
context = transaction.getRegistrationContext().getPersistentMap()

# Get the incoming path of the transaction
incomingPath = transaction.getIncoming().getAbsolutePath()

key = context.get("RETRY_COUNT")
if (key == None):
key = 1
try:
#read in the metadata file
for f in os.listdir(incomingPath):
if f == "metadata.txt":
metadata = open(os.path.join(incomingPath, f))
fileInfo = dict()
for line in metadata:
try:
pair = line.strip().split('=')
fileInfo[pair[0]] = pair[1]
except IndexError as exception:
originalError = exception
error = MetadataFormattingException("Metadata file not correctly formatted. Check for additional line breaks.")
metadata.close()
try:
user = fileInfo["user"]
except:
user = None
secname = fileInfo["info"]
sampleCode = fileInfo["barcode"]
datasetType = fileInfo["type"]
else:
name = f

#read in the metadata file
for f in os.listdir(incomingPath):
if f == "metadata.txt":
metadata = open(os.path.join(incomingPath, f))
fileInfo = dict(line.strip().split('=') for line in metadata)
metadata.close()
try:
user = fileInfo["user"]
except:
user = None
secname = fileInfo["info"]
code = fileInfo["barcode"]
datasetType = fileInfo["type"]
else:
name = f

project = code[:5]
type = "INFORMATION"
if "Results" in datasetType:
type = "RESULT"

if user:
transaction.setUserId(user)

inputFile = os.path.join(incomingPath, name)
newname = urllib.unquote(name)
dataFile = os.path.join(incomingPath, newname)
print "renaming "+inputFile+" to "+dataFile
os.rename(inputFile, dataFile)

search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, code))
foundSamples = search_service.searchForSamples(sc)
sample = None
space = None
sa = None
attachmentReady = True

if len(foundSamples) == 0:
attachmentReady = False
project = extractProjectCode(sampleCode)
type = "INFORMATION"
if "Results" in datasetType:
type = "RESULT"
if error:
raise error
if user:
transaction.setUserId(user)

inputFile = os.path.join(incomingPath, name)
newname = urllib.unquote(name)
dataFile = os.path.join(incomingPath, newname)
print "renaming "+inputFile+" to "+dataFile
os.rename(inputFile, dataFile)

search_service = transaction.getSearchService()
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, project+"ENTITY-1"))
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, sampleCode))
foundSamples = search_service.searchForSamples(sc)
sample = foundSamples[0]
sampleID = sample.getSampleIdentifier()
sa = transaction.getSampleForUpdate(sampleID)
space = sa.getSpace()
if not attachmentReady:
infoSampleID = "/"+space+"/"+code
sa = transaction.getSampleForUpdate(infoSampleID)
if not sa:
exp = transaction.createNewExperiment('/' + space + '/' + project + '/'+ project+'_INFO', "Q_PROJECT_DETAILS")
sa = transaction.createNewSample('/' + space + '/'+ code, "Q_ATTACHMENT_SAMPLE")
sa.setExperiment(exp)
info = None

dataSet = transaction.createNewDataSet("Q_PROJECT_DATA")
dataSet.setMeasuredData(False)
dataSet.setPropertyValue("Q_SECONDARY_NAME", secname)
dataSet.setPropertyValue("Q_ATTACHMENT_TYPE", type)
dataSet.setSample(sa)
transaction.moveFile(dataFile, dataSet)
sample = None
space = None
sa = None
attachmentSampleFound = True

if len(foundSamples) == 0:
attachmentSampleFound = False
sc = SearchCriteria()
sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, project+"ENTITY-1"))
foundSamples = search_service.searchForSamples(sc)
try:
sample = foundSamples[0]
except IndexError as exception:
originalError = exception
error = NoSamplesFoundForProjectException("No sample could be found for this project.")
sampleID = sample.getSampleIdentifier()
sa = transaction.getSampleForUpdate(sampleID)
space = sa.getSpace()
if not attachmentSampleFound:
# fetch it by name
infoSampleID = "/"+space+"/"+sampleCode
sa = transaction.getSampleForUpdate(infoSampleID)
if not sa:
# create necessary objects if sample really doesn't exist
try:
experimentID = getInfoExperimentIdentifier(space, project)
exp = transaction.createNewExperiment(experimentID, "Q_PROJECT_DETAILS")
except Exception as exception:
originalError = exception
error = CouldNotCreateException("Experiment "+experimentID+" could not be created.")
try:
sa = transaction.createNewSample(infoSampleID, "Q_ATTACHMENT_SAMPLE")
except Exception as exception:
originalError = exception
error = CouldNotCreateException("Sample "+infoSampleID+" was not found and could not be created.")
sa.setExperiment(exp)

dataSet = transaction.createNewDataSet("Q_PROJECT_DATA")
dataSet.setMeasuredData(False)
dataSet.setPropertyValue("Q_SECONDARY_NAME", secname)
dataSet.setPropertyValue("Q_ATTACHMENT_TYPE", type)
dataSet.setSample(sa)
transaction.moveFile(dataFile, dataSet)
# catch other, unknown exceptions
except Exception as exception:
originalError = exception
if not error:
error = Exception("Unknown exception occured: "+str(exception))
if originalError:
# if there is a problem sending the email, we log this and raise the original exception
try:
mailFactory = etl_mailer.EmailFactory()
etlMailer = etl_mailer.ETLMailer(email_helper.get_mail_host(), email_helper.get_mail_server(), email_helper.get_mail_from())

subject = "Error storing project attachment uploaded for "+project
content = mailFactory.formatRegistrationErrorContent(str(error))
etlMailer.send_email([user], subject, content)
except Exception as mailException:
print "Could not send error email: "+str(mailException)
pass
raise originalError

0 comments on commit 0a57170

Please sign in to comment.