forked from opensbt/opensbt-fmnist
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathfitness_mnist.py
176 lines (145 loc) · 7.13 KB
/
fitness_mnist.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
from typing import Tuple
from opensbt.evaluation.fitness import *
from opensbt.evaluation.critical import *
from opensbt.simulation.simulator import SimulationOutput
class CriticalMNISTMulti(Critical):
def eval(self, vec_fit: np.ndarray, simout: SimulationOutput) -> bool:
return vec_fit[0] < -0.5 and \
vec_fit[1] < 0.2 #and \
#vec_fit[2] < -3 and vec_fit[2] > -4 # distance between -2 and -4
class CriticalMNISTEntropy(Critical):
def eval(self, vec_fit: np.ndarray, simout: SimulationOutput) -> bool:
return vec_fit[0] < -0.7 and vec_fit[1] > 0.5 and vec_fit[1] < 0.55
class CriticalMNISTConf(Critical):
''' Conf_diff_max < 0'''
def eval(self, vec_fit: np.ndarray, simout: SimulationOutput) -> bool:
return vec_fit[0] < 0
class CriticalMNISTConf_05(Critical):
def eval(self, vec_fit: np.ndarray, simout: SimulationOutput) -> bool:
return vec_fit[0] < -0.5
class CriticalMNISTConf_07(Critical):
def eval(self, vec_fit: np.ndarray, simout: SimulationOutput) -> bool:
return vec_fit[0] < -0.7
class CriticalMNISTConf_09(Critical):
def eval(self, vec_fit: np.ndarray, simout: SimulationOutput) -> bool:
return vec_fit[0] < -0.9
class CriticalMNISTConf_095(Critical):
def eval(self, vec_fit: np.ndarray, simout: SimulationOutput) -> bool:
return vec_fit[0] < -0.95
class CriticalMNIST(Critical):
''' Conf_diff_max < 0, Conf_missclassified_label > 0.5'''
def eval(self, vec_fit: np.ndarray, simout: SimulationOutput) -> bool:
# return vec_fit[0] < 0 and vec_fit[1] < 0.1 # Conf diff, Conf miss
# return vec_fit[0] < 0 and (vec_fit[1] == -7) # Conf diff, Predicted
# return vec_fit[0] < -0.2 and vec_fit[1] < -1 # Conf diff, Dist archive # Random value, need some method/explanation to choose it.
# return vec_fit[0] < 0 and (vec_fit[1] < -0.5) # Conf diff, maximize prediction as 8
return vec_fit[0] < -0.5 and vec_fit[1] < 0.2 # Conf diff, Dist archive # Random value, need some method/explanation to choose it.
# return vec_fit[0] < 0.5 and vec_fit[1] < 0.6 # Conf diff, Dist archive # Random value, need some method/explanation to choose it.
#return vec_fit[0] < 0.2 and vec_fit[1] > 0.5 and vec_fit[1] < 0.55 # Conf diff, Dist archive # Random value, need some method/explanation to choose it.
''' Fitness function for MNIST Problem'''
class FitnessMNIST(Fitness):
def __init__(self, diversify=False) -> None:
super().__init__()
self.diversify = diversify
@property
def min_or_max(self):
if self.diversify:
return "min", "min", "max"
else:
return "min", "min"
@property
def name(self):
#name = "Conf_diff_max", "Coverage" #"Distance_archive" #"Conf_miss" #"Predicted Label"
#name = "Confidence_Expected", "Luminosity" #"Distance_archive" #"Conf_miss" #"Predicted Label"
name = "Confidence_Expected", "Brightness", "Distance_Input" #"Distance_Archive" #"Distance_archive" #"Conf_miss" #"Predicted Label"
#return "Entropy_Signed", "Coverage" #, "Distance_Archive" #"Distance_archive" #"Conf_miss" #"Predicted Label"
if self.diversify:
name = name[0:3]
else:
name = name[0:2]
return name
# HACK. needed to add data parameter, as information is not sotred in simout
def eval(self, simout: SimulationOutput, **kwargs) -> Tuple[float]:
data = simout.otherParams["data"]
# Evalute fitness value of the classification ( = simulation)
# predicted_label = data["predicted_label"]
confidence = data["confidence"]
# predictions = data["predictions"]
# expected_label = data["expected_label"]
brightness = data["brightness"]
# distance_archive = data["distance_archive"]
# coverage = data["coverage"]
# move_distance = data["move_distance"]
# angle = data["angle"]
# entropy_signed = data["entropy_signed"]
# distance_test_input = data["distance_test_input"]
# coverage_rel = data["coverage_rel"]
''' Fitness 1 '''
# Assign fitness value to individual
# ff1: as in DeepHyperion paper, ff2: mis predicted label confidence (min)
ff1 = confidence #confidence if confidence > 0 else -0.1
#ff1 = predictions[expected_label]
#ff1 = entropy_signed
''' Fitness 2 '''
# ff2 = predicted_label
# ff2 = predictions[predicted_label] if expected_label != predicted_label else 0
# ff2 = predictions[8] # confidence to get an eight
# ff2 = distance_archive
ff2 = brightness
# ff2 = coverage
# ff2 = coverage_rel
#ff2 = angle
# ff3 = -distance_archive
# ff3 = - distance_test_input
# use novelty search distance
distance_archive = 0
if self.diversify:
if "algorithm" in kwargs:
algorithm = kwargs["algorithm"]
if not hasattr(algorithm, 'archive_novelty'):
distance_archive = 0
print("no archive novelty")
else:
_, distance_archive = algorithm.archive_novelty.closest_individual_from_vars(
kwargs["individual"])
print(f"archive size: {len(algorithm.archive_novelty)}")
print(f"distance_archive: {distance_archive}")
f_vector = (ff1, ff2, distance_archive)
else:
f_vector = (ff1, ff2)
return f_vector
''' Filtered fitness function for MNIST Problem'''
class FitnessMNISTFiltered(Fitness):
@property
def min_or_max(self):
return "min", "min"
@property
def name(self):
return "Conf_diff_max", "Coverage (neg)" #"Distance_archive" #"Conf_miss" #"Predicted Label"
# HACK. needed to add data parameter, as information is not sotred in simout
def eval(self, simout: SimulationOutput, **kwargs) -> Tuple[float]:
data = simout.otherParams["data"]
# Evalute fitness value of the classification ( = simulation)
predicted_label = data["predicted_label"]
confidence = data["confidence"]
predictions = data["predictions"]
expected_label = data["expected_label"]
#distance_archive = data["distance_archive"]
coverage = data["coverage"]
# Assign fitness value to individual
# ff1: as in DeepHyperion paper, ff2: mis predicted label confidence (min)
ff1 = confidence #confidence if confidence > 0 else -0.1
# ff2 = predicted_label
# ff2 = predictions[predicted_label] if expected_label != predicted_label else 0
#ff1 = predictions[expected_label]
ff2 = coverage
# ff2 = distance_archive
# predictions[predicted_label] if predicted_label != expected_label \
# else 0vec_fit
PENALTY_FF1 = 1
PENALTY_FF2 = 0
print("APPLYING MNIST FILTERED")
if CriticalMNIST().eval([ff1,ff2], simout=None):
return (ff1, ff2)
else:
return (PENALTY_FF1, PENALTY_FF2)