-
Notifications
You must be signed in to change notification settings - Fork 0
/
smt_installer.sh
555 lines (482 loc) · 16.5 KB
/
smt_installer.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
#!/bin/bash
############################################################################################
# STATISTICAL MACHINE TRANSLATION TOOLS INSTALLER #
# #
# -pranjan341@gmail.com #
# #
############################################################################################
#make sure this script wasn't started as 'sh smt_installer.sh' or similar.
if [ "x$BASH" = "x" ]; then
echo "\033[0;31m Should be started as 'bash smt_installer.sh'"
exit 1;
fi
# error function called when ever something important fails to execute.
error()
{
echo -e "\033[0;31m Oops! ERROR: " ${1} | tee -a $wdirect/smt_installer.log
echo ""
echo ""
echo -e '\033[0;31m Please check if you have a working internet connection and you are authorised to install programs in this system \e[0m' | tee -a $wdirect/smt_installer.log
kill "$!"
exit 1
}
# to check for a stable internet connection
chk_internet_connection()
{
ping -q -w 1 -c 1 `ip r | grep default | cut -d ' ' -f 3` 2> /dev/null && echo "Internet is working" || error "No internet"
}
# spinner animation while something runs in the background
spinner()
{
local i sp n
echo ' '
sp=' /-\|'
n=${#sp}
printf ' '
while sleep 0.1; do
printf "%s\b" "${sp:i++%n:1}"
done
}
# Update apt-get and install necessary packages
linux_packages_install()
{
echo 'Updating apt-get' | tee -a $wdirect/smt_installer.log
spinner &
sudo apt-get -y update || error "apt update error: cannot update "
echo 'Downloading and installing all required packages' | tee -a $wdirect/smt_installer.log
sudo apt-get install -y g++ git automake libtool zlib1g-dev libboost-all-dev libbz2-dev liblzma-dev libgoogle-perftools-dev python-dev graphviz imagemagick cmake build-essential subversion autoconf unzip || error "cannot download essential packages"
sudo apt-get install -y make
sudo apt-get install -y unzip
kill "$!"
lipdf = 1
echo "All packages downloded" | tee -a $wdirect/smt_installer.log
}
# download indic nlp library for indic languages from github and install the tokeniser to /usr/local/bin for easy access
indic_nlp_library_install()
{
if [ -d "indic_nlp_library/src" ]; then
echo -e '\033[0;32m indicnlp found, skipping installation \e[0m' | tee -a $wdirect/smt_installer.log
return 1
fi
echo "Downloading indic_nlp_library" | tee -a $wdirect/smt_installer.log
spinner &
git clone https://github.com/anoopkunchukuttan/indic_nlp_library.git || error "cannot download"
chmod +x indic_nlp_library/src/indicnlp/tokenize/indic_tokenize.py
sudo cp indic_nlp_library/src/indicnlp/tokenize/indic_tokenize.py /usr/local/bin/indic_tokenize.py
echo "Indic nlp library successfully installed" | tee -a $wdirect/smt_installer.log
kill "$!"
}
# manually install boost c++ libraries using the following method; use only if current way does not work.
# boost c++ libraries seems to be corrupted on ubuntu distributions 12.04 and older so there use this method
# The current (via apt-get) installs an older but compatible version of libboost-all-dev. But if latest version is required use this
old_boost_cpp_libraries_install()
{
spinner &
echo "Downloading boost_cpp_libraries" | tee -a $wdirect/smt_installer.log
wget https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz
echo 'unpacking boost' | tee -a $wdirect/smt_installer.log
tar zxvf boost_1_64_0.tar.gz
cd boost_1_64_0/
echo 'Setting up boost' | tee -a $wdirect/smt_installer.log
./bootstrap.sh || error
#boost install
echo 'Installing Boost cpp libraies' | tee -a $wdirect/smt_installer.log
./b2 || error
./b2 -j4 --prefix=${PWD} --libdir=${PWD}/lib64 --layout=system link=static install
echo "done" | tee -a $wdirect/smt_installer.log
#./b2 -j4 --prefix=${PWD} --libdir=${PWD}/lib64 --layout=system link=static install
cd ..
kill "$!"
}
# moses2 and moses-server require xmlrpc c++ libraries to install and work properly. Currently they are being installed via apt-get in line 197
# manually install xmlrpc libraries using the following method;use only if current way does not work.
old_xmlrpc_install()
{
echo "Downloading xmlrpc for moses2/moses_server " | tee -a $wdirect/smt_installer.log
wget -O xmlrpc-c-1.39.12.tgz https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.39.12/xmlrpc-c-1.39.12.tgz/download
tar -xvzf xmlrpc-c-1.39.12.tgz || error
cd xmlrpc-c-1.39.12
./configure --prefix=$wdirect/xmlrpc || error
make || error
make install || error
echo "xmlrpc installed " | tee -a $wdirect/smt_installer.log
cd ..
}
# installation for giza-pp
giza_pp_install()
{
cd $wdirect
spinner &
if [ -d "bin/" ]; then
echo -e '\033[0;32m giza-pp binaries found, skipping installation \e[0m' | tee -a $wdirect/smt_installer.log
kill "$!"
return 1
fi
sudo rm giza-pp/
echo "Downloading giza-pp" | tee -a $wdirect/smt_installer.log
git clone https://github.com/moses-smt/giza-pp.git
cd giza-pp
echo 'Compiling giza++' | tee -a $wdirect/smt_installer.log
make || error "problems in installing giza-pp"
cd ../
mkdir bin | tee -a $wdirect/smt_installer.log
echo 'Installing giza++' | tee -a $wdirect/smt_installer.log
cp giza-pp/GIZA++-v2/GIZA++ giza-pp/GIZA++-v2/snt2cooc.out giza-pp/GIZA++-v2/snt2plain.out giza-pp/GIZA++-v2/plain2snt.out giza-pp/mkcls-v2/mkcls bin
sudo cp giza-pp/GIZA++-v2/GIZA++ /usr/local/bin/GIZA++
sudo cp giza-pp/GIZA++-v2/snt2cooc.out /usr/local/bin/snt2cooc.out
sudo cp giza-pp/GIZA++-v2/snt2plain.out /usr/local/bin/snt2plain.out
sudo cp giza-pp/GIZA++-v2/plain2snt.out /usr/local/bin/plain2snt.out
sudo cp giza-pp/mkcls-v2/mkcls /usr/local/bin/mkcls
echo "giza-pp installed successfully" | tee -a $wdirect/smt_installer.log
kill "$!"
}
# manual installation for irstlm ver 5.8
irstlm_install()
{
spinner &
if [ -d "irstlm/bin" ]; then
echo -e '\033[0;32m irstlm binaries found, skipping installation \e[0m' | tee -a $wdirect/smt_installer.log
kill "$!"
return 1
fi
sudo rm irstlm/ irstlm-5.80.08/
echo "Downloading irstlm" | tee -a $wdirect/smt_installer.log
sudo apt-get install irstlm
wget -O irstlm.zip https://sourceforge.net/projects/irstlm/files/irstlm/irstlm-5.80/irstlm-5.80.08.zip/download
echo 'unpacking irstlm..' | tee -a $wdirect/smt_installer.log
unzip irstlm.zip || error "unable to unzip irstlm.zip"
cd irstlm-5.80.08/trunk || error "irstlm doesnot exist"
echo 'Setting up irstlm...' | tee -a $wdirect/smt_installer.log
./regenerate-makefiles.sh
./regenerate-makefiles.sh || error "fatal error while running ./regenerate-makefiles.sh in irstlm"
./configure --prefix=$wdirect/irstlm || error "fatal error in ./configure in irstlm"
echo 'compiling....' | tee -a $wdirect/smt_installer.log
make
echo 'Installing irstlm....' | tee -a $wdirect/smt_installer.log
make install || error "make install error in irstlm"
echo 'irstlm installed successfully' | tee -a $wdirect/smt_installer.log
kill "$!"
cd ../../
}
# installation for srilm
srilm_install()
{
spinner &
if [ -d "srilm/bin" ]; then
echo -e '\033[0;32m srilm binaries found, skipping installation \e[0m' | tee -a $wdirect/smt_installer.log
kill "$!"
return 1
fi
sudo rm srilm/
echo "Downloading srilm" | tee -a $wdirect/smt_installer.log
wget --no-check-certificate 'https://www.dropbox.com/s/mnfgpaw0oyh81gy/srilm%20%281%29.zip?dl=1' -O srilm.zip
echo 'unpacking srilm..' | tee -a $wdirect/smt_installer.log
unzip srilm.zip || error "cannot unzip srilm.zip"
cd srilm
echo 'Setting up srilm...' | tee -a $wdirect/smt_installer.log
make || error "make error"
echo 'Installing srilm....' | tee -a $wdirect/smt_installer.log
make World || error "make error"
echo 'srilm installed successfully' | tee -a $wdirect/smt_installer.log
kill "$!"
cd ..
}
# installation for xmlrpc using apt-get
xmlrpc_install()
{
echo "Downloading xmlrpc for moses2/moses_server " | tee -a $wdirect/smt_installer.log
sudo apt-get install -y libxmlrpc-c++8-dev libxmlrpc-c++8v5 libxmlrpc-core-c3 libxmlrpc-core-c3-dev xmlrpc-api-utils
echo "xmlrpc installed " | tee -a $wdirect/smt_installer.log
}
# installation for moses
moses_install()
{
cd $wdirect
spinner &
echo "Downloading mosesdecoder toolkit" | tee -a $wdirect/smt_installer.log
git clone https://github.com/moses-smt/mosesdecoder.git
echo 'Compiling and Installing Mosesdecoder with giza++ ; boost ; irstlm, etc.' | tee -a $wdirect/smt_installer.log
cd mosesdecoder/
install_moses="./bjam --with-giza-pp=$wdirect/bin ${1} -j4"
${install_moses} || {install_moses}
sudo cp bin/build_binary /usr/local/bin/build_binary
sudo cp bin/moses /usr/local/bin/moses
sudo cp scripts/recaser/train-truecaser.perl /usr/local/bin/train-truecaser.perl
sudo cp scripts/tokenizer/tokenizer.perl /usr/local/bin/tokenizer.perl
sudo cp scripts/recaser/truecase.perl /usr/local/bin/truecase.perl
sudo cp scripts/training/clean-corpus-n.perl /usr/local/bin/clean-corpus-n.perl
sudo cp scripts/training/mert-moses.pl /usr/local/bin/mert-moses.pl
echo "moses installed successfully" | tee -a $wdirect/smt_installer.log
kill "$!"
cd ..
}
# installation for moses; moses2/moses-server
moses2_install()
{
cd $wdirect
spinner &
if [ ! -d "mosesdecoder/" ]; then
echo "Downloading mosesdecoder toolkit" | tee -a $wdirect/smt_installer.log
git clone https://github.com/moses-smt/mosesdecoder.git
fi
xmlrpc_install || error
# uncomment to install xmlrpc manually
# old_xmlrpc_install
cd mosesdecoder
install_moses="./bjam --with-giza-pp=$wdirect/bin ${1} -j4"
${install_moses} || {install_moses}
sudo cp bin/build_binary /usr/local/bin/build_binary
sudo cp bin/moses /usr/local/bin/moses
sudo cp scripts/recaser/train-truecaser.perl /usr/local/bin/train-truecaser.perl
sudo cp scripts/tokenizer/tokenizer.perl /usr/local/bin/tokenizer.perl
sudo cp scripts/recaser/truecase.perl /usr/local/bin/truecase.perl
sudo cp scripts/training/clean-corpus-n.perl /usr/local/bin/clean-corpus-n.perl
sudo cp scripts/training/mert-moses.pl /usr/local/bin/mert-moses.pl
echo "moses2 installed successfully" | tee -a $wdirect/smt_installer.log
cd ..
kill "$!"
}
# test moses installation
test_moses()
{
spinner &
if [ ! -d "sample-models/" ]; then
echo "Downloding test cases" | tee -a $wdirect/smt_installer.log
wget http://www.statmt.org/moses/download/sample-models.tgz || chk_internet_connection 2> /dev/null
tar xzf sample-models.tgz || error
fi
echo 'Validating install' | tee -a $wdirect/smt_installer.log
cd sample-models
../mosesdecoder/bin/moses -f phrase-model/moses.ini < phrase-model/in > out || error
echo "moses installation successful" | tee -a $wdirect/smt_installer.log
notify-send "moses installation successful"
#feature yet to release
#cd ..
#clear
kill "$!"
}
# default installation
default_install()
{
echo "Installing mosesdecoder toolkit alongwith giza++, irstlm and boost c++ libraries " | tee -a $wdirect/smt_installer.log
#spinner &
irin="--with-irstlm=$wdirect/irstlm"
linux_packages_install
giza_pp_install
irstlm_install
moses_install $irin
test_moses
#kill "$!"
clear
cd $wdirect
cat smt_installer.log
#kill "$!"
exit
}
recompile_moses()
{
cd mosesdecoder/
spinner &
./bjam --with-giza-pp=$HOME/bin -j4 || error
kill "$!"
cd ..
}
# call if zenity not available
get_path()
{
echo "Please enter absolute path to installation directory in this format (without quotes)"
echo "'/home/xyz/path/here'"
read var1
echo $var1
}
trap "echo 'exiting installer';exit" 0 1 2 5 15
cat logo.txt
chk_internet_connection 2> /dev/null
echo "loading installer"
# install dialog
sudo apt-get install -y dialog zenity || chk_internet_connection 2> /dev/null
echo ''
echo -e '\033[0;32m Please choose installation directory \e[0m'
sleep 3
wdirect=$(zenity --file-selection --directory)
sleep 1
mkdir -p ${wdirect} || error "cannot create installation directory"
cd ${wdirect}
echo ""
echo "enter password"
sudo echo "installation directory"
pwd
sleep 2
rm ${wdirect}/smt_installer.log
echo ''
sleep 1
DIALOG_CANCEL=1
DIALOG_ESC=255
HEIGHT=0
WIDTH=0
advanced_options()
{
#choices=`/usr/bin/dialog --stdout --checklist "Choose additional softwares" 60 70 6 1 "mosesdecoder with boost and giza++" on 2 "moses2/moses-server and srilm with boost and giza++ " off 3 irstlm on 4 srilm off 5 indic_nlp_library off`
#spinner &
irin="--with-irstlm=$wdirect/irstlm"
while true; do
exec 3>&1
selection=$(dialog \
--clear \
--cancel-label "Exit" \
--checklist "Choose additional softwares" 100 100 6 \
1 indic_nlp_library off \
2 srilm off \
3 "irstlm (for moses on ubuntu 16.04 and below )" on \
4 "standalone irstlm, not installed with moses (ubuntu ver >= 17.10) " off \
5 "moses, moses2/moses-server, srilm with boost and giza++ " off \
6 "mosesdecoder with boost, giza++ and kenlm" on \
2>&1 1>&3)
exit_status=$?
exec 3>&-
case $exit_status in
$DIALOG_CANCEL)
clear
echo "Installer terminated."
exit
;;
$DIALOG_ESC)
clear
echo "Installer aborted." >&2
exit 1
;;
esac
if [ $? -eq 0 ]
then
for choice in $selection
do
echo "You chose: $choice"
#clear
case $choice in
0 )
#clear
echo "Installer terminated."
;;
1 )
echo 1
indic_nlp_library_install
#test_moses
;;
2 )
echo 2
if [[ $lipdf -eq 0 ]]; then
linux_packages_install
fi
lipdf=1
srilm_install
;;
3 )
echo 3
if [[ $lipdf -eq 0 ]]; then
linux_packages_install
fi
lipdf=1
irstlm_install
irflag=1
#test_moses
;;
4 )
echo 4
sudo apt-get install irstlm
;;
5 )
echo 5
# install linux packages only if they are not installed before
if [[ $lipdf -eq 0 ]]; then
linux_packages_install
fi
lipdf=1
# uncomment to install boost manually
#old_boost_cpp_libraries_install
giza_pp_install
srilm_install
if [[ $irflag -eq 1 ]]; then
moses2_install ${irin}
else
moses2_install
fi
test_moses
;;
6 )
echo 6
if [[ $lipdf -eq 0 ]]; then
linux_packages_install
fi
lipdf=1
# old_boost_cpp_libraries_install
giza_pp_install
if [[ $irflag -eq 1 ]]; then
moses_install ${irin}
else
moses_install
fi
test_moses
;;
esac
done
else echo cancel selected
fi
clear
cd $wdirect
cat smt_installer.log
#kill "$!"
exit
done
}
while true; do
exec 3>&1
selection=$(dialog \
--backtitle "Mosesdecoder toolkit installer" \
--title "Options" \
--clear \
--cancel-label "Exit" \
--menu "Please select installation method:" $HEIGHT $WIDTH 4 \
"1" "Default: install only moses with giza++ and irstlm" \
"2" "Advanced: Choose your configuration" \
"3" "Run bjam" \
2>&1 1>&3)
exit_status=$?
exec 3>&-
case $exit_status in
$DIALOG_CANCEL)
clear
echo "Installer terminated."
exit
;;
$DIALOG_ESC)
clear
echo "Installer aborted." >&2
exit 1
;;
esac
case $selection in
0 )
clear
echo "Installer terminated."
;;
1 )
default_install
;;
2 )
advanced_options
;;
3 )
recompile_moses
;;
esac
clear
cd $wdirect
cat smt_installer.log
#kill "$!"
exit
done
clear
cd $wdirect
cat smt_installer.log
#kill "$!"
exit