add zpop

shilad · Jun 24, 2020 · 464ee13 · 464ee13
1 parent ceea8f0
commit 464ee13
Show file tree

Hide file tree

Showing 147 changed files with 485 additions and 3 deletions.
diff --git a/.gitignore b/.gitignore
@@ -111,4 +111,3 @@ venv.bak/
 
 # Ignore experiment folder
 experiments/
-data/
diff --git a/bin/run-joint.sh b/bin/run-joint.sh
@@ -19,7 +19,9 @@ do
   exp_dir=$(prepare_experiment_dir $topic ${exp_id})
   # Write down parameters that'll be on the html file
   write_experiment_params ${exp_dir} weight 1
-
+  python -m cartograph.draw.zpop_creator \
+            --experiment ${exp_dir} \
+            --popularity_score data/${topic}/popularity_score.csv
   # Step 2: run UMAP
   python -m cartograph.xy_embed.umap_embed \
           --map_directory ${exp_dir} \
@@ -47,13 +49,18 @@ do
           --cluster_groups /key_phrases_cluster_groups.csv \
           --output_file /key_phrases_top_labels.csv \
           --label_source key_phrases \
-          --num_top_labels ${labels_num} # number of top keyphrases labels 
+          --num_top_labels ${labels_num} # number of top keyphrases labels
 
    #Step 5: Fetch hierarchical categories from key phrases
   python -m cartograph.h_cat_fetcher \
           --experiment ${exp_dir} \
           --isSumInKeyPhrase ${isSumInKeyPhrase}
 
+  #Step 5.5 generate zpop
+  python -m cartograph.draw.zpop_creator \
+            --experiment ${exp_dir} \
+            --popularity_score data/${topic}/popularity_score.csv
+
   # Step 6
   python -m cartograph.user_study_label \
           --experiment ${exp_dir} \

diff --git a/cartograph/draw/zpop_creator.py b/cartograph/draw/zpop_creator.py
@@ -0,0 +1,47 @@
+# Transform popularity score from wikipedia to a score that looks good on cartograph
+# Author: Rock Pang,
+# Reference: https://github.com/shilad/cartograph/blob/develop-simple/cartograph/CalculateZPop.py
+
+import pandas as pd
+import numpy as np
+import argparse
+
+
+def log4(x):
+    return np.log2(x) / np.log2(4)
+
+
+def main(experiment_directory, popularity_score_df, new_xy_embeddings_df):
+    assert(popularity_score_df.shape[0] != 0) # check if popularity score df is not empty
+    new_rows = []
+    pop_dic = {}
+    for row in popularity_score_df.itertuples():
+        pop_dic[row.article_id] = row.popularity_score
+
+    for row in new_xy_embeddings_df.itertuples():
+        val = 0
+        if row.article_id in pop_dic:
+            val = pop_dic[row.article_id]
+        new_rows.append({"article_id":row.article_id, "popularity_score": val})
+
+    df = pd.DataFrame(new_rows)
+
+    sorted_score = df.sort_values(by='popularity_score', ascending=False)
+
+    sorted_score['zpop'] = log4(np.arange(sorted_score.shape[0]) / 2.0 + 1.0)
+    sorted_score = sorted_score.drop("popularity_score", axis=1)
+    sorted_score.to_csv(experiment_directory + "/zpop_score.csv", index=False)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--experiment', required=True)
+    parser.add_argument('--popularity_score', required=True)
+    args = parser.parse_args()
+
+    experiment_directory = args.experiment
+    popularity_score = args.popularity_score
+    new_xy_embedding = experiment_directory + "/new_xy_embeddings.csv"
+
+
+    main(experiment_directory, pd.read_csv(popularity_score), pd.read_csv(new_xy_embedding))
diff --git a/data/article_vectors.csv b/data/article_vectors.csv
diff --git a/data/education/article_hierarchical_categories.csv b/data/education/article_hierarchical_categories.csv
diff --git a/data/education/article_keyphrases.csv b/data/education/article_keyphrases.csv
diff --git a/data/education/article_keywords.csv b/data/education/article_keywords.csv
diff --git a/data/education/article_labels_combined.csv b/data/education/article_labels_combined.csv
diff --git a/data/education/article_lda_labels.csv b/data/education/article_lda_labels.csv
diff --git a/data/education/article_links.csv b/data/education/article_links.csv
diff --git a/data/education/article_text.csv b/data/education/article_text.csv
diff --git a/data/education/article_text_gloss.csv b/data/education/article_text_gloss.csv
diff --git a/data/education/article_topic_distribution.csv b/data/education/article_topic_distribution.csv
diff --git a/data/education/article_vectors.csv b/data/education/article_vectors.csv
diff --git a/data/education/combined_label_names.csv b/data/education/combined_label_names.csv
diff --git a/data/education/domain_concept.csv b/data/education/domain_concept.csv
diff --git a/data/education/hierarchical_category_names.csv b/data/education/hierarchical_category_names.csv
diff --git a/data/education/keyphrases_names.csv b/data/education/keyphrases_names.csv
diff --git a/data/education/keyword_names.csv b/data/education/keyword_names.csv
diff --git a/data/education/labels/LDA_labels/LDA_labels.csv b/data/education/labels/LDA_labels/LDA_labels.csv
diff --git a/data/education/lda_label_names.csv b/data/education/lda_label_names.csv
diff --git a/data/education/link_names.csv b/data/education/link_names.csv
diff --git a/data/education/popularity_score.csv b/data/education/popularity_score.csv
diff --git a/data/education/topic_model b/data/education/topic_model
diff --git a/data/education/topic_model.expElogbeta.npy b/data/education/topic_model.expElogbeta.npy
diff --git a/data/education/topic_model.id2word b/data/education/topic_model.id2word
diff --git a/data/education/topic_model.state b/data/education/topic_model.state
diff --git a/data/food/article_categories.csv b/data/food/article_categories.csv
diff --git a/data/food/article_hierarchical_categories.csv b/data/food/article_hierarchical_categories.csv
diff --git a/data/food/article_keyphrases.csv b/data/food/article_keyphrases.csv
diff --git a/data/food/article_keywords.csv b/data/food/article_keywords.csv
diff --git a/data/food/article_labels_combined.csv b/data/food/article_labels_combined.csv
diff --git a/data/food/article_lda_labels.csv b/data/food/article_lda_labels.csv
diff --git a/data/food/article_links.csv b/data/food/article_links.csv
diff --git a/data/food/article_text.csv b/data/food/article_text.csv
diff --git a/data/food/article_text_gloss.csv b/data/food/article_text_gloss.csv
diff --git a/data/food/article_vectors.csv b/data/food/article_vectors.csv
diff --git a/data/food/category_names.csv b/data/food/category_names.csv
diff --git a/data/food/combined_label_names.csv b/data/food/combined_label_names.csv
diff --git a/data/food/domain_concept.csv b/data/food/domain_concept.csv
diff --git a/data/food/hierarchical_category_names.csv b/data/food/hierarchical_category_names.csv
diff --git a/data/food/keyphrases_names.csv b/data/food/keyphrases_names.csv
diff --git a/data/food/keyword_names.csv b/data/food/keyword_names.csv
diff --git a/data/food/lda_label_names.csv b/data/food/lda_label_names.csv
diff --git a/data/food/link_names.csv b/data/food/link_names.csv
diff --git a/data/food/popularity_score.csv b/data/food/popularity_score.csv
diff --git a/data/food/topic/article_topic_distribution.csv b/data/food/topic/article_topic_distribution.csv
diff --git a/data/food/topic/topic_label_distribution.csv b/data/food/topic/topic_label_distribution.csv
diff --git a/data/food/vanilla_vectors.csv b/data/food/vanilla_vectors.csv
diff --git a/data/georgraphy/article_hierarchical_categories.csv b/data/georgraphy/article_hierarchical_categories.csv
diff --git a/data/georgraphy/article_keyphrases.csv b/data/georgraphy/article_keyphrases.csv
diff --git a/data/georgraphy/article_keywords.csv b/data/georgraphy/article_keywords.csv
diff --git a/data/georgraphy/article_lda_labels.csv b/data/georgraphy/article_lda_labels.csv
diff --git a/data/georgraphy/article_links.csv b/data/georgraphy/article_links.csv
diff --git a/data/georgraphy/article_text b/data/georgraphy/article_text
diff --git a/data/georgraphy/article_text_gloss.csv b/data/georgraphy/article_text_gloss.csv
diff --git a/data/georgraphy/article_topic_distribution.csv b/data/georgraphy/article_topic_distribution.csv
diff --git a/data/georgraphy/article_vectors.csv b/data/georgraphy/article_vectors.csv
diff --git a/data/georgraphy/domain_concept.csv b/data/georgraphy/domain_concept.csv
diff --git a/data/georgraphy/hierarchical_category_names.csv b/data/georgraphy/hierarchical_category_names.csv
diff --git a/data/georgraphy/keyphrases_names.csv b/data/georgraphy/keyphrases_names.csv
diff --git a/data/georgraphy/keyword_names.csv b/data/georgraphy/keyword_names.csv
diff --git a/data/georgraphy/labels/LDA_labels/LDA_labels.csv b/data/georgraphy/labels/LDA_labels/LDA_labels.csv
diff --git a/data/georgraphy/lda_label_names.csv b/data/georgraphy/lda_label_names.csv
diff --git a/data/georgraphy/link_names.csv b/data/georgraphy/link_names.csv
diff --git a/data/georgraphy/popularity_score.csv b/data/georgraphy/popularity_score.csv
diff --git a/data/georgraphy/topic_model b/data/georgraphy/topic_model
diff --git a/data/georgraphy/topic_model.expElogbeta.npy b/data/georgraphy/topic_model.expElogbeta.npy
diff --git a/data/georgraphy/topic_model.id2word b/data/georgraphy/topic_model.id2word
diff --git a/data/georgraphy/topic_model.state b/data/georgraphy/topic_model.state
diff --git a/data/h_cat_from_top_labels.pkl b/data/h_cat_from_top_labels.pkl
diff --git a/data/h_cat_from_top_labels_one_level.pkl b/data/h_cat_from_top_labels_one_level.pkl
diff --git a/data/internet/article_categories.csv b/data/internet/article_categories.csv
diff --git a/data/internet/article_hierarchical_categories.csv b/data/internet/article_hierarchical_categories.csv
diff --git a/data/internet/article_keyphrases.csv b/data/internet/article_keyphrases.csv
diff --git a/data/internet/article_keywords.csv b/data/internet/article_keywords.csv
diff --git a/data/internet/article_labels_combined.csv b/data/internet/article_labels_combined.csv
diff --git a/data/internet/article_lda_labels.csv b/data/internet/article_lda_labels.csv
diff --git a/data/internet/article_links.csv b/data/internet/article_links.csv
diff --git a/data/internet/article_text.csv b/data/internet/article_text.csv
diff --git a/data/internet/article_text_summary_don_forget_delete.csv b/data/internet/article_text_summary_don_forget_delete.csv
diff --git a/data/internet/article_vectors.csv b/data/internet/article_vectors.csv
diff --git a/data/internet/category_names.csv b/data/internet/category_names.csv
Original file line number	Diff line number	Diff line change
Expand Up		@@ -111,4 +111,3 @@ venv.bak/

		# Ignore experiment folder
		experiments/
		data/