Skip to content

Commit

Permalink
real world
Browse files Browse the repository at this point in the history
  • Loading branch information
@david-yoon
david-yoon committed May 30, 2020
1 parent 51cbf56 commit 2e121fd
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 31 deletions.
57 changes: 27 additions & 30 deletions preprocessing/01D_raw_to_numpy_news-19.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -14,7 +14,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -49,22 +49,22 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# name_dataset = 'news-19_paragraph_swap-random-1m'\n",
"# name_dataset = 'headline_swap_news_v2'\n",
"name_dataset = 'headline_swap_news_v2.5'\n",
"# name_dataset = 'headline_swap_news_v2.5'\n",
"# name_dataset = 'headline_swap_news_v2.5_mf8'\n",
"# name_dataset = 'paragraph_swap_news_v2.5'\n",
"name_dataset = 'paragraph_swap_news_v2.5'\n",
"path_raw_data = '../data/raw/' + name_dataset + '/'\n",
"path_processed_data = '../data/' + name_dataset + '/whole/'"
]
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -99,17 +99,14 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2020-05-30T14:24:45.691755\n",
"100000 200000 300000 400000 500000 600000 700000 800000 900000 1000000 1100000 1200000 1300000 1347097 Done\n",
"2020-05-30T14:30:48.563996\n",
"voca size : 2988107\n"
"2020-05-30T21:44:39.779162\n"
]
}
],
Expand Down Expand Up @@ -432,16 +429,16 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"188095"
"196381"
]
},
"execution_count": 16,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -475,7 +472,7 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -485,7 +482,7 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -498,16 +495,16 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"188095"
"196381"
]
},
"execution_count": 19,
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -518,7 +515,7 @@
},
{
"cell_type": "code",
"execution_count": 20,
"execution_count": 9,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -737,7 +734,7 @@
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -1225,16 +1222,16 @@
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2020-05-30T19:24:03.243950\n",
"2020-05-30T21:46:32.079525\n",
"10000 20000 30000 40000 50000 60000 70000 80000 90000 100000 110000 120000 130000 140000 150000 160000 170000 178019 Done\n",
"2020-05-30T19:24:47.646848\n"
"2020-05-30T21:47:15.293565\n"
]
}
],
Expand Down Expand Up @@ -1274,15 +1271,15 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2020-05-30T19:24:47.656803\n",
"2020-05-30T19:25:19.275346\n"
"2020-05-30T21:47:15.302178\n",
"2020-05-30T21:47:47.505548\n"
]
}
],
Expand All @@ -1294,15 +1291,15 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2020-05-30T19:25:27.741204\n",
"2020-05-30T19:25:28.417751\n"
"2020-05-30T21:47:47.517646\n",
"2020-05-30T21:47:48.151477\n"
]
}
],
Expand Down
9 changes: 8 additions & 1 deletion src_whole/eval_AHDE.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,12 @@



CUDA_VISIBLE_DEVICES=1 python eval_AHDE.py --model_path "model_ahde-TEST" --batch_size 100 --encoder_size 170 --context_size 44 --encoderR_size 22 --num_layer 1 --hidden_dim 200 --num_layer_con 1 --hidden_dim_con 100 --embed_size 300 --corpus 'news-19_whole' --data_path '../data/headline_swap_news_v2.5/whole/'

# CUDA_VISIBLE_DEVICES=1 python eval_AHDE.py --model_path "model_ahde-head-v2.5-TEST" --batch_size 400 --encoder_size 170 --context_size 44 --encoderR_size 22 --num_layer 1 --hidden_dim 200 --num_layer_con 1 --hidden_dim_con 100 --embed_size 300 --corpus 'news-19_whole' --data_path '../data/headline_swap_news_v2.5/whole/'





CUDA_VISIBLE_DEVICES=1 python eval_AHDE.py --model_path "model_ahde-para-v2.5-TEST" --batch_size 400 --encoder_size 170 --context_size 44 --encoderR_size 22 --num_layer 1 --hidden_dim 200 --num_layer_con 1 --hidden_dim_con 100 --embed_size 300 --corpus 'news-19_whole' --data_path '../data/paragraph_swap_news_v2.5/whole/'

0 comments on commit 2e121fd

Please sign in to comment.