-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathenglish.py
171 lines (135 loc) · 5.85 KB
/
english.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
import os
import nltk
from nltk.parse.stanford import StanfordParser
from nltk.stem import WordNetLemmatizer
from nltk.stem import PorterStemmer
from nltk.tree import *
from conf import JAR_DIR
# from nltk.parse.corenlp import stanford
os.environ['STANFORD_PARSER'] = JAR_DIR
os.environ['STANFORD_MODELS'] = JAR_DIR
os.environ['JAVAHOME'] = "C:/Program Files/Java/jdk-12.0.1/bin/java.exe"
nltk.download('wordnet')
text = ""
def filter_stop_words(words):
stopwords_set = set(['a', 'an','am', 'The', 'is','for','to'])
# stopwords_set = set(stopwords.words("english"))
words = list(filter(lambda x: x not in stopwords_set, words))
return words
def lemmatize_tokens(token_list):
lemmatizer = WordNetLemmatizer()
ps = PorterStemmer()
lemmatized_words = []
for token in token_list:
#token=ps.stem(token)
token = lemmatizer.lemmatize(token)
lemmatized_words.append(lemmatizer.lemmatize(token,pos="v"))
return lemmatized_words
def label_parse_subtrees(parent_tree):
tree_traversal_flag = {}
for sub_tree in parent_tree.subtrees():
tree_traversal_flag[sub_tree.treeposition()] = 0
return tree_traversal_flag
def handle_noun_clause(i, tree_traversal_flag, modified_parse_tree, sub_tree):
# if clause is Noun clause and not traversed then insert them in new tree first
if tree_traversal_flag[sub_tree.treeposition()] == 0 and tree_traversal_flag[sub_tree.parent().treeposition()] == 0:
tree_traversal_flag[sub_tree.treeposition()] = 1
modified_parse_tree.insert(i, sub_tree)
i = i + 1
return i, modified_parse_tree
def handle_verb_prop_clause(i, tree_traversal_flag, modified_parse_tree, sub_tree):
# if clause is Verb clause or Proportion clause recursively check for Noun clause
for child_sub_tree in sub_tree.subtrees():
if child_sub_tree.label() == "NP" or child_sub_tree.label() == 'PRP':
if tree_traversal_flag[child_sub_tree.treeposition()] == 0 and tree_traversal_flag[child_sub_tree.parent().treeposition()] == 0:
tree_traversal_flag[child_sub_tree.treeposition()] = 1
modified_parse_tree.insert(i, child_sub_tree)
i = i + 1
return i, modified_parse_tree
def modify_tree_structure(parent_tree):
# Mark all subtrees position as 0
tree_traversal_flag = label_parse_subtrees(parent_tree)
# Initialize new parse tree
modified_parse_tree = Tree('ROOT', [])
i = 0
for sub_tree in parent_tree.subtrees():
if sub_tree.label() == "NP":
i, modified_parse_tree = handle_noun_clause(i, tree_traversal_flag, modified_parse_tree, sub_tree)
if sub_tree.label() == "VP" or sub_tree.label() == "PRP":
i, modified_parse_tree = handle_verb_prop_clause(i, tree_traversal_flag, modified_parse_tree, sub_tree)
# recursively check for omitted clauses to be inserted in tree
for sub_tree in parent_tree.subtrees():
for child_sub_tree in sub_tree.subtrees():
if len(child_sub_tree.leaves()) == 1: #check if subtree leads to some word
if tree_traversal_flag[child_sub_tree.treeposition()] == 0 and tree_traversal_flag[child_sub_tree.parent().treeposition()] == 0:
tree_traversal_flag[child_sub_tree.treeposition()] = 1
modified_parse_tree.insert(i, child_sub_tree)
i = i + 1
return modified_parse_tree
def convert_eng_to_isl(input_string):
if len(list(input_string.split(' '))) == 1:
return list(input_string.split(' '))
# Initializing stanford parser
parser = StanfordParser()
# Generates all possible parse trees sort by probability for the sentence
possible_parse_tree_list = [tree for tree in parser.parse(input_string.split())]
# Get most probable parse tree
parse_tree = possible_parse_tree_list[0]
<<<<<<< HEAD
#print(parse_tree)
=======
print(parse_tree)
>>>>>>> 8e43ad3c63987396256069fbf73746d9afc7e284
# Convert into tree data structure
parent_tree = ParentedTree.convert(parse_tree)
modified_parse_tree = modify_tree_structure(parent_tree)
parsed_sent = modified_parse_tree.leaves()
return parsed_sent
def pre_process(sentence):
words = list(sentence.split())
f = open('words.txt', 'r')
eligible_words = f.read()
f.close()
final_string = ""
for word in words:
if word not in eligible_words:
for letter in word:
final_string += " " + letter
else:
final_string += " " + word
return final_string
# DRIVER CODE
def isl(text):
input_string = text.capitalize()
<<<<<<< HEAD
#print(input_string)
isl_parsed_token_list = convert_eng_to_isl(input_string)
#print("isl parsed token list: ",isl_parsed_token_list)
# lemmatize tokens
lemmatized_isl_token_list = lemmatize_tokens(isl_parsed_token_list)
#print("LEMMATIZED WORDS",lemmatized_isl_token_list)
# remove stop words
filtered_isl_token_list = filter_stop_words(lemmatized_isl_token_list)
#print("filtered isl token list",filtered_isl_token_list)
=======
print(input_string)
isl_parsed_token_list = convert_eng_to_isl(input_string)
print("isl parsed token list: ",isl_parsed_token_list)
# lemmatize tokens
lemmatized_isl_token_list = lemmatize_tokens(isl_parsed_token_list)
print("LEMMATIZED WORDS",lemmatized_isl_token_list)
# remove stop words
filtered_isl_token_list = filter_stop_words(lemmatized_isl_token_list)
print("filtered isl token list",filtered_isl_token_list)
>>>>>>> 8e43ad3c63987396256069fbf73746d9afc7e284
isl_text_string = ""
for token in filtered_isl_token_list:
isl_text_string += token
isl_text_string += " "
isl_text_string = isl_text_string.lower()
<<<<<<< HEAD
#print(isl_text_string)
=======
print(isl_text_string)
>>>>>>> 8e43ad3c63987396256069fbf73746d9afc7e284
isl(text)