forked from hengchao0248/ccf2016_sougou
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathtfidf_lr_stack.py
86 lines (69 loc) · 2.58 KB
/
tfidf_lr_stack.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
# tfidf-lr stack for education/age/gender
import pandas as pd
import numpy as np
import jieba
import pickle
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import KFold
from datetime import datetime
import cfg
# -----------------------myfunc-----------------------
def myAcc(y_true, y_pred):
y_pred = np.argmax(y_pred, axis=1)
return np.mean(y_true == y_pred)
# -----------------------load data--------------------
df_all = pd.read_csv(cfg.data_path + 'all_v2.csv', encoding='utf8', nrows=200000)
ys = {}
for label in ['Education', 'age', 'gender']:
ys[label] = np.array(df_all[label])
class Tokenizer(object):
def __init__(self):
self.n = 0
def __call__(self, line):
tokens = []
for query in line.split('\t'):
words = [word for word in jieba.cut(query)]
for gram in [1, 2]:
for i in range(len(words) - gram + 1):
tokens += ["_*_".join(words[i:i+gram])]
if np.random.rand() < 0.00001:
print(line)
print('='*20)
print(tokens)
self.n += 1
if self.n % 10000 == 0:
print(self.n)
return tokens
tfv = TfidfVectorizer(tokenizer=Tokenizer(), min_df=3, max_df=0.95, sublinear_tf=True)
X_sp = tfv.fit_transform(df_all['query'])
pickle.dump(X_sp, open(cfg.data_path + 'tfidf_10W.feat', 'wb'))
df_stack = pd.DataFrame(index=range(len(df_all)))
# -----------------------stack for education/age/gender------------------
for lb in ['Education', 'age', 'gender']:
print(lb)
TR = 100000
num_class = len(pd.value_counts(ys[lb]))
n = 5
X = X_sp[:TR]
y = ys[lb][:TR]
X_te = X_sp[TR:]
y_te = ys[lb][TR:]
stack = np.zeros((X.shape[0], num_class))
stack_te = np.zeros((X_te.shape[0], num_class))
for i, (tr, va) in enumerate(KFold(len(y), n_folds=n)):
print('%s stack:%d/%d' % (str(datetime.now()), i+1, n))
clf = LogisticRegression(C=3)
clf.fit(X[tr], y[tr])
y_pred_va = clf.predict_proba(X[va])
y_pred_te = clf.predict_proba(X_te)
print('va acc:', myAcc(y[va], y_pred_va))
print('te acc:', myAcc(y_te, y_pred_te))
stack[va] += y_pred_va
stack_te += y_pred_te
stack_te /= n
stack_all = np.vstack([stack, stack_te])
for i in range(stack_all.shape[1]):
df_stack['tfidf_{}_{}'.format(lb, i)] = stack_all[:, i]
df_stack.to_csv(cfg.data_path + 'tfidf_stack_20W.csv', index=None, encoding='utf8')
print(datetime.now(), 'save tfidf stack done!')