Merge pull request #276 from AdityaSavara/tuningCorrector-updates

Tuning corrector updates

Author: AdityaSavara

MSRESOLVE.py

@@ -1289,6 +1289,25 @@ def CorrectionValuesObtain(ReferenceData):
         listOfNeededMolecules = ReferenceData.molecules #intentionally grab molecule list from the original data object fed in.
         ReferenceDataOriginalStandardTuning = copy.deepcopy(ReferenceData)  #This is mainly needed for G.referenceFileStandardTuning
         ReferenceDataStandardTuning = createReferenceDataObject ( G.referenceFileStandardTuning[0], G.referenceFileStandardTuning[1], AllMID_ObjectsDict=G.AllMID_ObjectsDict)
+        #TODO: Make the below block of code a function. This block is actually the copy of a block from tuning correction.
+        if (G.calculateUncertaintiesInConcentrations == True) and (type(G.referenceFileUncertainties) != type(None)):
+            if type(G.referenceFileUncertainties) == type(float(5)) or  type(G.referenceFileUncertainties) == type(int(5)) :
+                #TODO: Low priority. The below results in "nan" values. It could be better to change it to make zeros using a numpy "where" statement.
+                G.referenceFileUncertainties = float(G.referenceFileUncertainties) #Make sure we have a float.
+                #Get what we need.
+                provided_reference_patterns = ReferenceDataStandardTuning.provided_reference_patterns
+                provided_reference_patterns_without_masses = ReferenceDataStandardTuning.provided_reference_patterns[:,1:] #[:,0] is mass fragments, so we slice to remove those. 
+                #Make our variables ready.
+                absolute_standard_uncertainties = provided_reference_patterns*1.0 #First we make a copy.
+                absolute_standard_uncertainties_without_masses = (provided_reference_patterns_without_masses/provided_reference_patterns_without_masses)*G.referenceFileUncertainties #We do the division to get an array of ones. Then we multiply to get an array of the same size.
+                #now we populate our copy's non-mass area.
+                absolute_standard_uncertainties[:,1:] = absolute_standard_uncertainties_without_masses                                        
+                ReferenceDataStandardTuning.absolute_standard_uncertainties = absolute_standard_uncertainties
+                #We can't convert to relative uncertainties yet because the file may not be standardized yet.
+            else:
+                print("WARNING: Line 1325 of MSRESOLVE.py else statement has not been programmed. ReferenceDataStandardTuning currently only receives uncertainties if there is an integer in the referenceFileUncertainties.")
+                ReferenceDataStandardTuning.absolute_standard_uncertainties = ReferenceDataStandardTuning.provided_reference_patterns*1.0 #First we make a copy.
+                ReferenceDataStandardTuning.absolute_standard_uncertainties[:,1:] = 0
         listOfStandardTuningMoleculePatternsAvailable = copy.deepcopy(ReferenceDataStandardTuning.molecules)
         #get a list of molecules to remove from the ReferenceDataStandardTuning file.
         listOfMoleculesToRemove = []
@@ -1317,11 +1336,22 @@ def CorrectionValuesObtain(ReferenceData):
         ReferenceDataStandardTuning, addedReferenceSlice = extendReferencePattern(ReferenceDataStandardTuning, ReferenceDataOriginalStandardTuning)
         #This is the ReferenceDataObject to carry forward.
         ReferenceDataStandardTuning.exportReferencePattern("ExportedReferencePatternStandardForCorrectionValuesMixedStandardTuning.csv")
+        if G.createMixedTuningPattern == False:
+            listOfMoleculesToRemove = []
+            for moleculeName in ReferenceDataStandardTuning.molecules:
+                if moleculeName not in ReferenceData.molecules:
+                    listOfMoleculesToRemove.append(moleculeName)
+            ReferenceDataStandardTuning = ReferenceDataStandardTuning.removeMolecules(listOfMoleculesToRemove)
         #rearrange the molecules to the right order.
         ReferenceDataStandardTuning = rearrangeReferenceData(ReferenceData=ReferenceDataStandardTuning, desiredMoleculesOrder=ReferenceData.molecules)
         ReferenceDataStandardTuning.exportReferencePattern("ExportedReferencePatternStandardForCorrectionValuesMixedStandardTuningRearranged.csv")
         #move the pointer.
         ReferenceDataForCorrectionValues = ReferenceDataStandardTuning
+    #Before the below, ReferenceDataForCorrectionValues and ReferenceData must be made the same size and width. To do that, we will extend each to have any rows missing that the other has.
+    ReferenceDataMassFragments = ReferenceData.standardized_reference_patterns[:,0]
+    ReferenceDataForCorrectionValuesMassFragments = ReferenceDataForCorrectionValues.standardized_reference_patterns[:,0]
+    ReferenceData.extendMassFragments(ReferenceDataForCorrectionValuesMassFragments)
+    ReferenceDataForCorrectionValues.extendMassFragments(ReferenceDataMassFragments)   
     reference_width = len(ReferenceDataForCorrectionValues.standardized_reference_patterns[0,:])  #This is number of molecules plus 1 because of the mass fragments column.
     reference_height = len(ReferenceDataForCorrectionValues.standardized_reference_patterns[:,0]) #this is the number of mass fragments.
     correction_values_direct = ReferenceDataForCorrectionValues.standardized_reference_patterns*1.0 #just initializing as same size, but note that this has the mass fragments column.
@@ -3012,7 +3042,7 @@ def __init__(self, provided_reference_patterns, electronnumbers, molecules, mole
             self.moleculeIonizationType = ['unknown']* len(self.molecules)
         self.provided_mass_fragments = self.provided_reference_patterns[:,0]
         #clear ClearZeroRowsFromProvidedReferenceIntensities
-        self.ClearZeroRowsFromProvidedReferenceIntensities()
+        #self.ClearZeroRowsFromProvidedReferenceIntensities()  #commenting this out to avoid problems for now.
         #initialize the standardized_reference_patterns
         self.standardized_reference_patterns=StandardizeReferencePattern(self.provided_reference_patterns,len(self.molecules))
 
@@ -3189,7 +3219,19 @@ def ClearZeroRowsFromStandardizedReferenceIntensities(self):
             print("Warning: line 2897 was unable to update the relative uncertainties of a reference pattern.")
         self.ExportCollector("ClearZeroRowsFromStandardizedReferenceIntensities", use_provided_reference_patterns=False)
 
-        
+    def extendMassFragments(self, massFragmentsToExtendBy):
+        #This will add in rows of zeros to the standardized_reference_patterns and also the uncertainties arrays.
+        self.standardized_reference_patterns_mass_fragments = self.standardized_reference_patterns[:,0]*1.0
+        self.standardized_reference_patterns = DataFunctions.extendXYYYtoZYYY(self.standardized_reference_patterns, massFragmentsToExtendBy)
+        #Check if the absolute_standard_uncertainties is the right length. If it is, we will be extending it also.
+        if hasattr(self, "absolute_standard_uncertainties"):
+            print("line 3270", self.standardized_reference_patterns[:,0], numpy.shape(self.standardized_reference_patterns))
+            print("line 3270", self.absolute_standard_uncertainties[:,0], numpy.shape(self.absolute_standard_uncertainties))
+            if len(self.absolute_standard_uncertainties[:,0] ==  self.standardized_reference_patterns_mass_fragments):
+                self.absolute_standard_uncertainties = DataFunctions.extendXYYYtoZYYY(self.absolute_standard_uncertainties, massFragmentsToExtendBy)
+                #For the relative uncertainties, we will use the built in function.
+                print('line 3275', numpy.shape(self.absolute_standard_uncertainties), numpy.shape(self.standardized_reference_patterns))
+                self.update_relative_standard_uncertainties()
 
     #This class function converts the XYXY data to an XYYY format
     def FromXYXYtoXYYY(self):

UnitTests/BestMassFragmentChooser/bestMassFragCombinations.txt

@@ -1,3 +1,3 @@
 The setting of useExtentOfSLSUniqueSolvable was used. The best combination is the last in this file. 
-Time taken:1.1674611568450928
+Time taken:1.194232702255249
 [(28.0, 31.0, 44.0, 57.0), (28.0, 31.0, 45.0, 57.0), (28.0, 31.0, 46.0, 57.0), (28.0, 31.0, 50.0, 57.0), (28.0, 31.0, 51.0, 57.0), (28.0, 31.0, 53.0, 57.0), (28.0, 31.0, 54.0, 57.0), (28.0, 31.0, 55.0, 57.0), (28.0, 31.0, 57.0, 71.0), (28.0, 31.0, 57.0, 72.0)]

UnitTests/ConcentrationFinder/LogFile.txt

@@ -1,4 +1,4 @@
-Tue Nov  9 16:21:19 2021
+Tue Nov  9 20:23:42 2021
 
 referenceFileName = ['AcetaldehydeNISTRefMixed2_test_2.csv', 'AcetaldehydeNISTRefMixed2_test_2.csv'] 
 form  = ['xyyy', 'xyyy'] 
@@ -20,5 +20,5 @@ scaleRawDataFactor  = 1.0
 resolvedScaledConcentrationsOutputName  = ScaledConcentrations.csv 
 concentrationsOutputName = ResolvedConcentrations.csv 
 simulatedSignalsOutputName = SimulatedRawSignals.csv 
-Run Time 0.23048 seconds 
+Run Time 0.60301 seconds 
 ###################################################################### 

UnitTests/ExtractReferencePatternFromData/LogFile.txt

@@ -1,4 +1,4 @@
-Tue Nov  9 16:21:39 2021
+Tue Nov  9 20:24:02 2021
 
 referenceFileName = ['AcetaldehydeNISTRefMixed2.csv'] 
 form  = ['xyyy'] 
@@ -27,5 +27,5 @@ scaleRawDataFactor  = 1.0
 resolvedScaledConcentrationsOutputName  = ScaledConcentrations.csv 
 concentrationsOutputName = ResolvedConcentrations.csv 
 simulatedSignalsOutputName = SimulatedRawSignals.csv 
-Run Time 3.68472 seconds 
+Run Time 3.32647 seconds 
 ###################################################################### 

UnitTests/FinalOptimizer/LogFile.txt

@@ -1,4 +1,4 @@
-Tue Nov  9 16:21:45 2021
+Tue Nov  9 20:24:08 2021
 
 referenceFileName = ['ConvertedSpectra.csv'] 
 form  = ['xyyy'] 
@@ -20,5 +20,5 @@ scaleRawDataFactor  = 1.0
 resolvedScaledConcentrationsOutputName  = ScaledConcentrations.csv 
 concentrationsOutputName = ResolvedConcentrations.csv 
 simulatedSignalsOutputName = SimulatedRawSignals.csv 
-Run Time 4.42243 seconds 
+Run Time 4.42341 seconds 
 ###################################################################### 

UnitTests/IonizationFactors/LogFile.txt

@@ -1,4 +1,4 @@
-Tue Nov  9 16:21:54 2021
+Tue Nov  9 20:24:18 2021
 
 referenceFileName = ['AcetaldehydeNISTRefDefault.csv'] 
 form  = ['xyyy'] 
@@ -17,5 +17,5 @@ scaleRawDataFactor  = 1.0
 resolvedScaledConcentrationsOutputName  = ScaledConcentrations.csv 
 concentrationsOutputName = ResolvedConcentrations.csv 
 simulatedSignalsOutputName = SimulatedRawSignals.csv 
-Run Time 0.10617 seconds 
+Run Time 0.09788 seconds 
 ###################################################################### 

UnitTests/IterativeAnalysis/LogFile.txt

@@ -1,4 +1,4 @@
-Tue Nov  9 16:21:57 2021
+Tue Nov  9 20:24:21 2021
 
 referenceFileName = ['AcetaldehydeNISTRefMixed2.csv'] 
 form  = ['xyyy'] 
@@ -24,5 +24,5 @@ scaleRawDataFactor  = 1.0
 resolvedScaledConcentrationsOutputName  = ScaledConcentrations.csv 
 concentrationsOutputName = ResolvedConcentrations.csv 
 simulatedSignalsOutputName = SimulatedRawSignals.csv 
-Run Time 1.75765 seconds 
+Run Time 1.96085 seconds 
 ###################################################################### 

UnitTests/IterativeAnalysis/_iter_1/LogFile.txt

@@ -1,4 +1,4 @@
-Tue Nov  9 16:21:58 2021
+Tue Nov  9 20:24:22 2021
 
 referenceFileName = ['AcetaldehydeNISTRefMixed2_iter_1.csv'] 
 form  = ['xyyy'] 
@@ -26,5 +26,5 @@ scaleRawDataFactor  = 1.0
 resolvedScaledConcentrationsOutputName  = ScaledConcentrations.csv 
 concentrationsOutputName = ResolvedConcentrations.csv 
 simulatedSignalsOutputName = SimulatedRawSignals.csv 
-Run Time 0.44900 seconds 
+Run Time 0.47060 seconds 
 ###################################################################### 

UnitTests/IterativeAnalysis/_iter_1/UserInput_iter_1.py

@@ -112,8 +112,8 @@
 TotalConcentrationsOutputName = 'TotalConcentrations.csv'
 ExportAtEachStep = 'yes'
 generatePercentages = 'no'
-checkpoint = 3.0687719
-start = 3.0687716
+checkpoint = 3.3668313
+start = 3.3668311
 timeSinceLastCheckpoint = ''
 iterationNumber = 1
 

UnitTests/IterativeAnalysis/_iter_2/LogFile.txt

@@ -1,4 +1,4 @@
-Tue Nov  9 16:21:58 2021
+Tue Nov  9 20:24:22 2021
 
 referenceFileName = ['AcetaldehydeNISTRefMixed2_iter_2.csv'] 
 form  = ['xyyy'] 
@@ -21,5 +21,5 @@ scaleRawDataFactor  = 1.0
 resolvedScaledConcentrationsOutputName  = ScaledConcentrations.csv 
 concentrationsOutputName = ResolvedConcentrations.csv 
 simulatedSignalsOutputName = SimulatedRawSignals.csv 
-Run Time 0.34426 seconds 
+Run Time 0.35016 seconds 
 ###################################################################### 

UnitTests/IterativeAnalysis/_iter_2/UserInput_iter_2.py

@@ -112,9 +112,9 @@
 TotalConcentrationsOutputName = 'TotalConcentrations.csv'
 ExportAtEachStep = 'yes'
 generatePercentages = 'no'
-checkpoint = 3.5177747
-start = 3.0687716
-timeSinceLastCheckpoint = 0.2388444999999999
+checkpoint = 3.8374273
+start = 3.3668311
+timeSinceLastCheckpoint = 0.24987629999999994
 iterationNumber = 1
 
 __var_list__ = ['referenceFileNamesList','referenceFormsList','collectedFileName','referencePatternTimeRanges','ionizationDataFileName','iterativeAnalysis','iterationNumber','iterationSuffix','unusedMolecules','oldReferenceFileName','oldCollectedFileName','nextRefFileName','nextExpFileName','preProcessing','dataAnalysis','dataSimulation','grapher','stopAtGraphs','timeRangeLimit','timeRangeStart','timeRangeFinish','specificMolecules','chosenMoleculesNames','specificMassFragments','chosenMassFragments','moleculeLikelihoods','sensitivityValues','linearBaselineCorrectionSemiAutomatic','baselineType','massesToBackgroundCorrect','earlyBaselineTimes','lateBaselineTimes','backgroundMassFragment','backgroundSlopes','backgroundIntercepts','interpolateYorN','marginalChangeRestriction','ignorableDeltaYThreshold','dataLowerBound','dataUpperBound','dataRangeSpecifierYorN','signalOrConcentrationRange','csvFile','moleculesToRestrict','csvFileName','bruteIncrements','permutationNum','maxPermutations','scaleRawDataOption','scaleRawDataFactor','measuredReferenceYorN','referenceFileStandardTuning','createMixedTuningPattern','referenceFileExistingTuning','referenceFileDesiredTuning','referenceCorrectionCoefficients','referenceCorrectionCoefficients_cov','extractReferencePatternFromDataOption','rpcMoleculesToChange','rpcMoleculesToChangeMF','rpcTimeRanges','minimalReferenceValue','referenceValueThreshold','referenceSignificantFragmentThresholds','lowerBoundThresholdChooser','massesToLowerBoundThresholdFilter','lowerBoundThresholdPercentage','lowerBoundThresholdAbsolute','dataSmootherYorN','dataSmootherChoice','dataSmootherTimeRadius','dataSmootherPointRadius','dataSmootherHeadersToConfineTo','polynomialOrder','rawSignalThresholdMethod','rawSignalThresholdValue','sensitivityThresholdValue','rawSignalThresholdDivider','rawSignalThresholdLimit','rawSignalThresholdLimitPercent','negativeAnalyzerYorN','NegativeAnalyzerTopNContributors','NegativeAnalyzerBaseNumberOfGridIntervals','calculateUncertaintiesInConcentrations','referenceFileUncertainties','collectedFileUncertainties','referenceCorrectionCoefficientsUncertainties','referenceCorrectionCoefficientsIonizationUncertainties','answer','uniqueOrCommon','slsWeighting','slsFinish','slsUniquePositiveConcentrationsOnly','objectiveFunctionType','distinguished','fullBrute','SLSUniqueExport','implicitSLScorrection','finalOptimization','concentrationFinder','moleculesTSC_List','TSC_List_Type','moleculeSignalTSC_List','massNumberTSC_List','moleculeConcentrationTSC_List','unitsTSC','preProcessedDataOutputName','resolvedScaledConcentrationsOutputName','scaledConcentrationsPercentages','concentrationsOutputName','simulatedSignalsOutputName','TotalConcentrationsOutputName','ExportAtEachStep','generatePercentages','checkpoint','start','timeSinceLastCheckpoint','iterationNumber']

UnitTests/IterativeAnalysis/_iter_3/LogFile.txt

@@ -1,4 +1,4 @@
-Tue Nov  9 16:21:59 2021
+Tue Nov  9 20:24:23 2021
 
 referenceFileName = ['AcetaldehydeNISTRefMixed2_iter_3.csv'] 
 form  = ['xyyy'] 
@@ -21,5 +21,5 @@ scaleRawDataFactor  = 1.0
 resolvedScaledConcentrationsOutputName  = ScaledConcentrations.csv 
 concentrationsOutputName = ResolvedConcentrations.csv 
 simulatedSignalsOutputName = SimulatedRawSignals.csv 
-Run Time 0.36059 seconds 
+Run Time 0.36289 seconds 
 ###################################################################### 

UnitTests/IterativeAnalysis/_iter_3/UserInput_iter_3.py

@@ -112,9 +112,9 @@
 TotalConcentrationsOutputName = 'TotalConcentrations.csv'
 ExportAtEachStep = 'yes'
 generatePercentages = 'no'
-checkpoint = 3.9920898
-start = 3.64783
-timeSinceLastCheckpoint = 0.22920850000000037
+checkpoint = 4.2981038
+start = 3.9479398
+timeSinceLastCheckpoint = 0.22425789999999957
 iterationNumber = 2
 
 __var_list__ = ['referenceFileNamesList','referenceFormsList','collectedFileName','referencePatternTimeRanges','ionizationDataFileName','iterativeAnalysis','iterationNumber','iterationSuffix','unusedMolecules','oldReferenceFileName','oldCollectedFileName','nextRefFileName','nextExpFileName','preProcessing','dataAnalysis','dataSimulation','grapher','stopAtGraphs','timeRangeLimit','timeRangeStart','timeRangeFinish','specificMolecules','chosenMoleculesNames','specificMassFragments','chosenMassFragments','moleculeLikelihoods','sensitivityValues','linearBaselineCorrectionSemiAutomatic','baselineType','massesToBackgroundCorrect','earlyBaselineTimes','lateBaselineTimes','backgroundMassFragment','backgroundSlopes','backgroundIntercepts','interpolateYorN','marginalChangeRestriction','ignorableDeltaYThreshold','dataLowerBound','dataUpperBound','dataRangeSpecifierYorN','signalOrConcentrationRange','csvFile','moleculesToRestrict','csvFileName','bruteIncrements','permutationNum','maxPermutations','scaleRawDataOption','scaleRawDataFactor','measuredReferenceYorN','referenceFileStandardTuning','createMixedTuningPattern','referenceFileExistingTuning','referenceFileDesiredTuning','referenceCorrectionCoefficients','referenceCorrectionCoefficients_cov','extractReferencePatternFromDataOption','rpcMoleculesToChange','rpcMoleculesToChangeMF','rpcTimeRanges','minimalReferenceValue','referenceValueThreshold','referenceSignificantFragmentThresholds','lowerBoundThresholdChooser','massesToLowerBoundThresholdFilter','lowerBoundThresholdPercentage','lowerBoundThresholdAbsolute','dataSmootherYorN','dataSmootherChoice','dataSmootherTimeRadius','dataSmootherPointRadius','dataSmootherHeadersToConfineTo','polynomialOrder','rawSignalThresholdMethod','rawSignalThresholdValue','sensitivityThresholdValue','rawSignalThresholdDivider','rawSignalThresholdLimit','rawSignalThresholdLimitPercent','negativeAnalyzerYorN','NegativeAnalyzerTopNContributors','NegativeAnalyzerBaseNumberOfGridIntervals','calculateUncertaintiesInConcentrations','referenceFileUncertainties','collectedFileUncertainties','referenceCorrectionCoefficientsUncertainties','referenceCorrectionCoefficientsIonizationUncertainties','answer','uniqueOrCommon','slsWeighting','slsFinish','slsUniquePositiveConcentrationsOnly','objectiveFunctionType','distinguished','fullBrute','SLSUniqueExport','implicitSLScorrection','finalOptimization','concentrationFinder','moleculesTSC_List','TSC_List_Type','moleculeSignalTSC_List','massNumberTSC_List','moleculeConcentrationTSC_List','unitsTSC','preProcessedDataOutputName','resolvedScaledConcentrationsOutputName','scaledConcentrationsPercentages','concentrationsOutputName','simulatedSignalsOutputName','TotalConcentrationsOutputName','ExportAtEachStep','generatePercentages','checkpoint','start','timeSinceLastCheckpoint','iterationNumber']

UnitTests/IterativeAnalysis/_iter_4/LogFile.txt

@@ -1,4 +1,4 @@
-Tue Nov  9 16:21:59 2021
+Tue Nov  9 20:24:23 2021
 
 referenceFileName = ['AcetaldehydeNISTRefMixed2_iter_4.csv'] 
 form  = ['xyyy'] 
@@ -21,5 +21,5 @@ scaleRawDataFactor  = 1.0
 resolvedScaledConcentrationsOutputName  = ScaledConcentrations.csv 
 concentrationsOutputName = ResolvedConcentrations.csv 
 simulatedSignalsOutputName = SimulatedRawSignals.csv 
-Run Time 0.32445 seconds 
+Run Time 0.33896 seconds 
 ######################################################################