-
Notifications
You must be signed in to change notification settings - Fork 0
/
GRU-attention.py
276 lines (209 loc) · 10.1 KB
/
GRU-attention.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
## RNN which classifies clauses into Semantic Clause Types
## Model variant: GRU + attention
## MB, February/March 2017
from __future__ import print_function
import numpy
import re, os
import random
import pandas
from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import GRU, LSTM
from keras.layers import Dense, Activation, Embedding, Bidirectional
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
from keras.layers.recurrent import SimpleRNN
from keras.layers.core import Masking, Dropout
from keras.callbacks import EarlyStopping
from keras.regularizers import l2, activity_l2
from keras.optimizers import Adagrad, Adam, Nadam
from keras.preprocessing.text import Tokenizer
# fix random seed for reproducibility
numpy.random.seed(7)
# load the dataset but only keep the top n words, zero the rest
top_words = 1000
path_train="" # set train path, data available at: https://github.com/annefried/sitent/tree/master/annotated_corpus
path_test="" # set test path, data available at: https://github.com/annefried/sitent/tree/master/annotated_corpus
embedding_path="" # set embedding path
emb_en="GoogleNews-vectors-negative300.txt"
path=os.listdir(path_train)
print("----------------------------LOADING DATA----------------------------")
### define and load train and test set ###
X=[] # texts
Y=[] # labels
for file in path_train:
if file.endswith(".csv"):
print (file)
op = open(path_train + file, "r")
thedata = pandas.read_csv(op, sep='\t', header='infer', names=None)
x = thedata['text'].astype(str)
y = thedata['gold_SitEntType'].astype(str)
X.extend(x.iloc[:].values)
Y.extend(y.iloc[:].values)
x=X
y=Y
x=numpy.asarray(x)
y=numpy.asarray(y)
path_test=os.listdir(path_test)
Xtest=[]
Ytest=[]
for file in path_test:
if file.endswith(".csv"):
print (file)
op = open(path_test + file, "r")
thedata = pandas.read_csv(op, sep='\t', header='infer', names=None)
xtest = thedata['text'].astype(str)
ytest = thedata['gold_SitEntType'].astype(str)
Xtest.extend(xtest.iloc[:].values)
Ytest.extend(ytest.iloc[:].values)
xtest=Xtest
ytest=Ytest
xtest=numpy.asarray(xtest)
ytest=numpy.asarray(ytest)
### Settings ###
tk = Tokenizer(nb_words=10000, lower=True, split=" ") #nb_words=number of most frequent words which the NN considers, lower = caseunsensitive, split=tokenisierer
tk.fit_on_texts(x)
x = tk.texts_to_sequences(x)
xtest = tk.texts_to_sequences(xtest)
max_len = 30 #number of words per clause that the NN considers
x = sequence.pad_sequences(x, maxlen=max_len) #cutting and zero padding
xtest = sequence.pad_sequences(xtest, maxlen=max_len)
max_features = 10000 #10000, equal to nb_words, size of one hot vector (sparse)
### Labels ###
def transform(label): #labels as one hot vectors
if label=="GENERIC_SENTENCE":
return [0, 0, 0, 0, 0, 0, 0, 1]
elif label=="EVENT":
return [0, 0, 0, 0, 0, 0, 1, 0]
elif label == "STATE":
return [0, 0, 0, 0, 0, 1, 0, 0]
elif label == "GENERALIZING_SENTENCE":
return [0, 0, 0, 0, 1, 0, 0, 0]
elif label == "REPORT":
return [0, 0, 0, 1, 0, 0, 0, 0]
elif label == "IMPERATIVE":
return [0, 0, 1, 0, 0, 0, 0, 0]
elif label == "QUESTION":
return [0, 1, 0, 0, 0, 0, 0, 0]
else:
return [1, 0, 0, 0, 0, 0, 0, 0]
y=[transform(label) for label in y]
ytest=[transform(label) for label in ytest]
print("---------------------------BUILDING MODEL---------------------------")
### Model ###
model = Sequential() #NNs framework
print ("Model:", model)
#use pretrained Embeddings
embeddings_index = {}
word_index = tk.word_index
f = open(os.path.join(embedding_path, emb_en)) #word2vec pre-trained Google News corpus (3 billion running words) word vector model (3 million 300-dimension English word vectors).))
print ("Embeddings:", f)
for line in f:
values = line.split()
word = values[0]
coefs = numpy.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
embedding_matrix = numpy.zeros((len(word_index) + 1, 300))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None: #words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
def get_H_n(X): #last output vector from lstm
ans = X[:, -1, :] #get last element from time dim
print (type(ans))
return ans
def get_Y(X, xmaxlen): #output vectors clause
print(type(X))
return X[:, :xmaxlen, :] #get first xmaxlen elem from time dim
def get_R(X): #weighted representation clause
Y, alpha = X[0], X[1]
ans = K.T.batched_dot(Y, alpha)
print(type(ans))
return ans
### Model ###
#1. Hidden Layer: Embeddings
main_input = Input(shape=(max_len,), dtype='int32', name='main_input')
emb = Embedding(output_dim=300, input_length=max_len, input_dim=29009, name='x', weights=[embedding_matrix])(main_input)
emb_drop_out = Dropout(0.8, name='dropout')(emb) # apply dropout to embeddings
bilstm = GRU(350, activation='tanh', return_sequences=True)(emb_drop_out)
bilstmstack = GRU(350, activation='tanh', return_sequences=True)(bilstm)
bilstmstack2 = GRU(350, activation='tanh', return_sequences=True)(bilstmstack)
bilstm_drop_out = Dropout(0.2)(bilstmstack2) # apply dropout to Bilstm
### GET M: Merged Outputs of two LSTMS (Rocktaeschel et al. 2016, p.3)
h_n = Lambda(get_H_n, output_shape=(350,), name="h_n")(bilstm_drop_out) # last output vector after merging two LSTMS above
Y = Lambda(get_Y, arguments={"xmaxlen": max_len}, name="Y", output_shape=(30, 350))(bilstm_drop_out) # output vector first LSTM
Whn = Dense(350, W_regularizer=l2(0.0001), name="Wh_n")(h_n) # product of weight vector and last output vector after merging 2 LSTMS above
Whn_x_e = RepeatVector(30, name="Wh_n_x_e")(Whn) # crossproduct of weight vector and last output vector after merging 2 LSTMS above times e (vector of 1s)
WY = TimeDistributed(Dense(350, W_regularizer=l2(0.0001)), name="WY")(Y) # product of weight vector and last output vector first LSTM
merged = merge([Whn_x_e, WY], name="merged", mode='sum') # sum Whn_x_e and WY
M = Activation('tanh', name="M")(merged) # apply tanh to sum of Whn_x_e and WY to get M
### GET alpha: attention weights (Rocktaeschel et al. 2016, p.3)
alpha_ = TimeDistributed(Dense(1, activation='linear'), name="alpha_")(M) # tim_dis applies a dense layer of shape 1 to every temporal slice of the input
flat_alpha = Flatten(name="flat_alpha")(alpha_) # flattens the input
alpha = Dense(max_len, activation='softmax', name="alpha")(flat_alpha) # vector of attention weights
### GET r: weighted representation of the premise (Rocktaeschel et al. 2016, p.3)
Y_trans = Permute((2, 1), name="y_trans")(Y) # transpose Y
r_ = merge([Y_trans, alpha], output_shape=(350, 1), name="r_", mode=get_R) # product of Y and alpha
r = Reshape((350,), name="r")(r_) # put r in the correct shape
### GET h_star: final sentence-pair representation, combination of r and h_n (Rocktaeschel et al. 2016, p.4)
Wr = Dense(350, W_regularizer=l2(0.0001))(r) # product of W and r
Wh = Dense(350, W_regularizer=l2(0.0001))(h_n) # product of W and h_n
merged = merge([Wr, Wh], mode='sum') # sum of Wr and Wh_n
h_star = Activation('tanh')(merged) # apply tanh to sum of Wr and Wh_n to get h_star
### Output Layer
out = Dense(8, activation='sigmoid')(h_star)
output = out
### Define input and output
model = Model(input=[main_input], output=output)
attention_extractor= Model(input=[main_input], output=alpha)
# try using different optimizers and different optimizer configs
adagrad=Adagrad(lr=0.05, epsilon=1e-08, decay=0.001)
model.compile(loss='categorical_crossentropy', optimizer='adagrad', metrics=['accuracy', 'fmeasure', 'precision', 'recall']) #
print ()
from collections import defaultdict
print('-----TRAINING MODEL-----')
dict1=tk.word_index
dict2={i:x for x,i in dict1.items()}
index_to_word=defaultdict(lambda: "", dict2)
print(index_to_word)
early_stopping = EarlyStopping(monitor='val_loss', patience=3)
conversion_dictionary={0: "other", 1:"question", 2:"imperative", 3:"report", 4:"generalizing", 5:"states", 6:"event", 7:"generic"}
print('-----TRAINING MODEL-----')
early_stopping = EarlyStopping(monitor='val_loss', patience=4)
model.fit(x, y, batch_size=100, nb_epoch=100, verbose=1, validation_split=0.2, callbacks=[early_stopping]) #schuffle = random picking from data
pred_y=model.predict(xtest)[0]
true_y=ytest
### produce files with weight vectors from attention mechanism for further analysis
res=attention_extractor.predict(x)
outputfile=open("weight_vectors_train.txt", "w")
for r1, s1, true_y in zip(res, x, y): #uses goldlabel of previous clause for prediction
liste=[]
for attention, word in zip(r1, s1):
outputfile.write(str(attention)+" "+str(word)+" "+index_to_word[word]+" "+"\n")
outputfile.write(conversion_dictionary[numpy.argmax(true_y)]+"\n")
outputfile.write("-"*100+"\n")
res=attention_extractor.predict(xtest)
outputfile=open("weight_vectors_test.txt", "w")
for r1, s1, true_y in zip(res, xtest, ytest): #uses goldlabel of previous clause for prediction
liste=[]
for attention, word in zip(r1, s1):
outputfile.write(str(attention)+" "+str(word)+" "+index_to_word[word]+" "+"\n")
outputfile.write(conversion_dictionary[numpy.argmax(true_y)]+"\n")
outputfile.write("-"*100+"\n")
### produce output file
outputfile=open("GRU+attention.txt", "w")
conversion_dictionary={0: "other", 1:"question", 2:"imperative", 3:"report", 4:"generalizing", 5:"states", 6:"event", 7:"generic"}
for pred, true in zip(pred_y, true_y):
outputfile.write(conversion_dictionary[numpy.argmax(pred)]+"\n")
outputfile.write(conversion_dictionary[numpy.argmax(true)]+"\n")
outputfile.write("-"*100+"\n")
outputfile.close()
score, acc, fmeasure, precision, recall = model.evaluate(xtest, ytest, batch_size=100)
print('-----RESULTS-----')
print('Test score:', score)
print('Test accuracy:', acc)
print('Test fmeasure:', fmeasure)
print('Test precision:', precision)
print('Test recall:', recall)