GRU-basic.py

## RNN which classifies clauses into Semantic Clause Types
## Model variant: Basic GRU
## MB, February/March 2017


from __future__ import print_function
import numpy
import re, os
import random
import pandas
from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import GRU, LSTM
from keras.layers import Dense, Activation, Embedding, Bidirectional
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
from keras.layers.recurrent import SimpleRNN
from keras.layers.core import Masking, Dropout
from keras.callbacks import EarlyStopping
from keras.regularizers import l2, activity_l2
from keras.optimizers import Adagrad, Adam, Nadam
from keras.preprocessing.text import Tokenizer

# fix random seed for reproducibility
numpy.random.seed(7)
# load the dataset but only keep the top n words, zero the rest
top_words = 1000

path_train="" # set train path, data available at: https://github.com/annefried/sitent/tree/master/annotated_corpus

path_test="" # set test path, data available at: https://github.com/annefried/sitent/tree/master/annotated_corpus

embedding_path="" # set embedding path

emb_en="GoogleNews-vectors-negative300.txt"


path=os.listdir(path_train)

print("----------------------------LOADING DATA----------------------------")

X=[] # texts
Y=[] # labels

### define and load train and test set ###

for file in path:
    
    if file.endswith(".csv"):
        print (file)
        op = open(path_train + file, "r")
        thedata = pandas.read_csv(op, sep='\t', header='infer', names=None)
        x = thedata['text'].astype(str)
        y = thedata['gold_SitEntType'].astype(str)
        X.extend(x.iloc[:].values)
        Y.extend(y.iloc[:].values)

x=X
y=Y
x=numpy.asarray(x) 
y=numpy.asarray(y) 

path=os.listdir(path_test)


Xtest=[]
Ytest=[]

for file in path:
    if file.endswith(".csv"):
        print (file)
        op = open(path_test + file, "r")
        thedata = pandas.read_csv(op, sep='\t', header='infer', names=None)
        xtest = thedata['text'].astype(str)
        ytest = thedata['gold_SitEntType'].astype(str)
        Xtest.extend(xtest.iloc[:].values)
        Ytest.extend(ytest.iloc[:].values)

xtest=Xtest
ytest=Ytest
xtest=numpy.asarray(xtest)
ytest=numpy.asarray(ytest)

### Settings ###

tk = Tokenizer(nb_words=10000, lower=True, split=" ") #nb_words= number of mfw which the network considers, lower = caseunsensitive, split=tokenisierer
tk.fit_on_texts(numpy.append(x,xtest))

x = tk.texts_to_sequences(x)
xtest = tk.texts_to_sequences(xtest)
max_len = 30 #number of words per clause that the NN considers
x = sequence.pad_sequences(x, maxlen=max_len) #zero padding
xtest = sequence.pad_sequences(xtest, maxlen=max_len)
max_features = 10000 #equal to nb_words, size of one hot vector (sparse)


print("---------------------------BUILDING MODEL---------------------------")

### Model ###

model = Sequential() # model's framework
print ("Model:", model)

#use pretrained Embeddings
embeddings_index = {}
word_index = tk.word_index
f = open(os.path.join(embedding_path, emb_en)) #word2vec pre-trained Google News corpus (3 billion running words) word vector model (3 million 300-dimension English word vectors).
print ("Embeddings:", f)

for line in f:
    values = line.split()
    word = values[0]
    coefs = numpy.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()

embedding_matrix = numpy.zeros((len(word_index) + 1, 300))
for word, i in word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None: #words not found in embedding index will be all-zeros.
        embedding_matrix[i] = embedding_vector

#1. Hidden Layer: Embeddings
model.add(Embedding(len(word_index) + 1, 300, input_length=max_len, dropout=0.4, weights=[embedding_matrix]))

W_regularizer=l2(0.001)
model.add(GRU(350, activation='tanh', return_sequences=True))
model.add(GRU(350, activation='tanh', return_sequences=True))
model.add(GRU(350, activation='tanh',  W_regularizer=W_regularizer))
model.add(Dropout(0.2))

#Ouput Layer
model.add(Dense(8)) # nb of labels (transforms cell size to that size)
model.add(Activation('sigmoid')) # activation function

adagrad=Adagrad(lr=0.05, epsilon=1e-08, decay=0.0011)
adam=Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.001)
nadam=Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004)

# compile model; use different optimizers and different optimizer configurations
model.compile(loss='categorical_crossentropy', optimizer='adagrad', metrics=['accuracy', 'fmeasure', 'precision', 'recall']) 

### Definition Labels ###

def transform(label):
    if label=="GENERIC_SENTENCE":
        return [0, 0, 0, 0, 0, 0, 0, 1]
    elif label=="EVENT":
        return [0, 0, 0, 0, 0, 0, 1, 0]
    elif label == "STATE":
        return [0, 0, 0, 0, 0, 1, 0, 0]
    elif label == "GENERALIZING_SENTENCE":
        return [0, 0, 0, 0, 1, 0, 0, 0]
    elif label == "REPORT":
        return [0, 0, 0, 1, 0, 0, 0, 0]
    elif label == "IMPERATIVE":
        return [0, 0, 1, 0, 0, 0, 0, 0]
    elif label == "QUESTION":
        return [0, 1, 0, 0, 0, 0, 0, 0]
    else:
        return [1, 0, 0, 0, 0, 0, 0, 0]


y=[transform(label) for label in y]
ytest=[transform(label) for label in ytest]
print (y)

print('-----TRAINING MODEL-----')

# early stopping to prevent from overfitting
early_stopping = EarlyStopping(monitor='val_loss', patience=3)

model.fit(x, y, batch_size=100, nb_epoch=50, verbose=1, validation_split=0.2, show_accuracy=True, shuffle=True, callbacks=[early_stopping]) #val_split: 20 prozent als dev set (von train)

pred_y=model.predict(xtest)[0]
true_y=ytest

### results ###

outputfile=open("predictions_basic_GRU.txt", "w")
conversion_dictionary={0: "other", 1:"question", 2:"imperative", 3:"report", 4:"generalizing", 5:"states", 6:"event", 7:"generic"}
for pred, true in zip(pred_y, true_y):
    outputfile.write(conversion_dictionary[numpy.argmax(pred)]+"\n") 
    outputfile.write(conversion_dictionary[numpy.argmax(true)]+"\n") 
    outputfile.write("-"*100+"\n")
outputfile.close()

score, acc, fmeasure, precision, recall = model.evaluate(xtest, ytest, batch_size=100)


print('-----RESULTS-----')

print('Test score:', score)
print('Test accuracy:', acc)
print('Test fmeasure:', fmeasure)
print('Test precision:', precision)
print('Test recall:', recall)