-
Notifications
You must be signed in to change notification settings - Fork 0
/
clean_train.py
210 lines (183 loc) · 8.54 KB
/
clean_train.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
"""
This tutorial shows how to generate some simple adversarial examples
and train a model using adversarial training using nothing but pure
TensorFlow.
It is very similar to mnist_tutorial_keras_tf.py, which does the same
thing but with a dependence on keras.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import numpy as np
import tensorflow as tf
from tensorflow.python.platform import flags
import logging
from cleverhans.utils_mnist import data_mnist
from cleverhans.utils_tf import model_train, model_eval
from cleverhans.attacks import FastGradientMethod
from cleverhans_tutorials.tutorial_models import make_basic_cnn
from cleverhans.utils import AccuracyReport, set_log_level
import os
FLAGS = flags.FLAGS
def mnist_tutorial(train_start=0, train_end=60000, test_start=0,
test_end=10000, nb_epochs=6, batch_size=128,
learning_rate=0.001,
clean_train=True,
testing=False,
backprop_through_attack=False,
nb_filters=64):
"""
MNIST cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param clean_train: if true, train on clean examples
:return: an AccuracyReport object
"""
nb_classes = 10
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(4264)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
sess = tf.Session()
# Get MNIST test data
X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
# Get MNISTnotMNIST data
# with np.load("MNISTnotMNIST.npz") as data:
# X_train, Y_train, X_test, Y_test = data['X_train'], data['Y_train'], data['X_test'], data['Y_test']
# Use label smoothing
# assert Y_train.shape[1] == 10
# label_smooth = .1
# Y_train = Y_train.clip(label_smooth / 9., 1. - label_smooth)
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
model_path = "./"
model_name = "clean_trained_mnist_model"
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'train_dir': model_path,
'filename': model_name
}
fgsm_params = {'eps': 0.3,
'clip_min': 0.,
'clip_max': 1.}
rng = np.random.RandomState([443, 224, 39])
if clean_train:
model = make_basic_cnn(nb_filters=nb_filters, nb_classes=nb_classes)
preds = model.get_probs(x)
def evaluate():
# Evaluate the accuracy of the MNIST model on legitimate test
# examples
eval_params = {'batch_size': batch_size}
acc = model_eval(
sess, x, y, preds, X_test, Y_test, args=eval_params)
report.clean_train_clean_eval = acc
assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate examples: %0.4f' % acc)
model_train(sess, x, y, preds, X_train, Y_train, evaluate=evaluate, save=True,
args=train_params, rng=rng)
# Calculate training error
if testing:
eval_params = {'batch_size': batch_size}
acc = model_eval(
sess, x, y, preds, X_train, Y_train, args=eval_params)
report.train_clean_train_clean_eval = acc
# Initialize the Fast Gradient Sign Method (FGSM) attack object and
# graph
fgsm = FastGradientMethod(model, sess=sess)
adv_x = fgsm.generate(x, **fgsm_params)
preds_adv = model.get_probs(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
eval_par = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds_adv, X_test, Y_test, args=eval_par)
print('Test accuracy on adversarial examples: %0.4f\n' % acc)
report.clean_train_adv_eval = acc
# Calculate training error
if testing:
eval_par = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds_adv, X_train,
Y_train, args=eval_par)
report.train_clean_train_adv_eval = acc
print("Repeating the process, using adversarial training")
# Redefine TF model graph
# model_2 = make_basic_cnn(nb_filters=nb_filters)
# preds_2 = model_2(x)
# fgsm2 = FastGradientMethod(model_2, sess=sess)
# adv_x_2 = fgsm2.generate(x, **fgsm_params)
# if not backprop_through_attack:
# # For the fgsm attack used in this tutorial, the attack has zero
# # gradient so enabling this flag does not change the gradient.
# # For some other attacks, enabling this flag increases the cost of
# # training, but gives the defender the ability to anticipate how
# # the atacker will change their strategy in response to updates to
# # the defender's parameters.
# adv_x_2 = tf.stop_gradient(adv_x_2)
# preds_2_adv = model_2(adv_x_2)
#
# def evaluate_2():
# # Accuracy of adversarially trained model on legitimate test inputs
# eval_params = {'batch_size': batch_size}
# accuracy = model_eval(sess, x, y, preds_2, X_test, Y_test,
# args=eval_params)
# print('Test accuracy on legitimate examples: %0.4f' % accuracy)
# report.adv_train_clean_eval = accuracy
#
# # Accuracy of the adversarially trained model on adversarial examples
# accuracy = model_eval(sess, x, y, preds_2_adv, X_test,
# Y_test, args=eval_params)
# print('Test accuracy on adversarial examples: %0.4f' % accuracy)
# report.adv_train_adv_eval = accuracy
#
# # Perform and evaluate adversarial training
# model_train(sess, x, y, preds_2, X_train, Y_train,
# predictions_adv=preds_2_adv, evaluate=evaluate_2,
# args=train_params, rng=rng)
#
# # Calculate training errors
# if testing:
# eval_params = {'batch_size': batch_size}
# accuracy = model_eval(sess, x, y, preds_2, X_train, Y_train,
# args=eval_params)
# report.train_adv_train_clean_eval = accuracy
# accuracy = model_eval(sess, x, y, preds_2_adv, X_train,
# Y_train, args=eval_params)
# report.train_adv_train_adv_eval = accuracy
return report
def main(argv=None):
mnist_tutorial(nb_epochs=FLAGS.nb_epochs, batch_size=FLAGS.batch_size,
learning_rate=FLAGS.learning_rate,
clean_train=FLAGS.clean_train,
backprop_through_attack=FLAGS.backprop_through_attack,
nb_filters=FLAGS.nb_filters)
if __name__ == '__main__':
flags.DEFINE_integer('nb_filters', 64, 'Model size multiplier')
flags.DEFINE_integer('nb_epochs', 8, 'Number of epochs to train model')
flags.DEFINE_integer('batch_size', 128, 'Size of training batches')
flags.DEFINE_float('learning_rate', 0.001, 'Learning rate for training')
flags.DEFINE_bool('clean_train', True, 'Train on clean examples')
flags.DEFINE_bool('backprop_through_attack', False,
('If True, backprop through adversarial example '
'construction process during adversarial training'))
tf.app.run()