Grid Seach Classifier with Three Models and Ensembling All

########## Grid search classification with three models and Then ensembling to produce ensembled model##########

################### grid Serch Cv Code for classification with three models###################

# Cross validate model with Kfold stratified cross val
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold
kfold = StratifiedKFold(n_splits=10)

from sklearn.ensemble import ExtraTreesClassifier
ExtC = ExtraTreesClassifier()


## Search grid for optimal parameters
ex_param_grid = {"max_depth":  [n for n in range(9, 14)],  
              "max_features": [1, 3, 10],
              "min_samples_split": [n for n in range(4, 11)],
              "min_samples_leaf": [n for n in range(2, 5)],
              "bootstrap": [False],
              "n_estimators" :[n for n in range(10, 60, 10)],
              "criterion": ["gini"]}


gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)

gsExtC.fit(X_train,y_train)

ExtC_best = gsExtC.best_estimator_

# Best score
gsExtC.best_score_

 # RFC Parameters tunning 
from sklearn.ensemble import RandomForestClassifier

### Random Forest #########
RFC = RandomForestClassifier()



## Search grid for optimal parameters
rf_param_grid = {"max_depth":  [n for n in range(9, 14)],  
              "max_features": [1, 3, 10],
              "min_samples_split": [n for n in range(4, 11)],
              "min_samples_leaf": [n for n in range(2, 5)],
              "bootstrap": [False],
              "n_estimators" :[n for n in range(10, 60, 10)],
              "criterion": ["gini"]}


gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)

gsRFC.fit(X_train,y_train)

RFC_best = gsRFC.best_estimator_

# Best score
gsRFC.best_score_


#### Adaboost

# Adaboost
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier

DTC = DecisionTreeClassifier()

adaDTC = AdaBoostClassifier(DTC, random_state=7)

ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
              "base_estimator__splitter" :   ["best", "random"],
              "algorithm" : ["SAMME","SAMME.R"],
              "n_estimators" :[30],
              "learning_rate":  [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}

gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)

gsadaDTC.fit(X_train,y_train)

ada_best = gsadaDTC.best_estimator_

gsadaDTC.best_score_

##### SVC########

### SVC classifier
from sklearn.svm import SVC

SVMC = SVC(probability=True)
svc_param_grid = {'kernel': ['rbf'], 
                  'gamma': [ 0.001, 0.01, 0.1, 1],
                  'C': [1, 10, 50, 100,200,300, 1000]}

gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)

gsSVMC.fit(X_train,y_train)

SVMC_best = gsSVMC.best_estimator_

# Best score
gsSVMC.best_score_

##### GB cassifier

# Gradient boosting tunning
from sklearn.ensemble import GradientBoostingClassifier

GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
              'n_estimators' : [n for n in range(10, 60, 10)],
              'learning_rate': [0.1, 0.05, 0.01],
              'max_depth':  [n for n in range(9, 14)],  
              'min_samples_leaf': [n for n in range(2, 5)],
              'max_features': [0.3, 0.1] 
              }

gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)

gsGBC.fit(X_train,y_train)

GBC_best = gsGBC.best_estimator_

# Best score
gsGBC.best_score_

#### Ensembling#######

from sklearn.ensemble import VotingClassifier

votingC = VotingClassifier(estimators=[('rfc', RFC_best), ('extc', ExtC_best),('svm',SVMC_best),
('gbc',GBC_best)], voting='soft', n_jobs=4)

votingC = votingC.fit(X_train, y_train)
test_Survived = pd.Series(votingC.predict(test), name="Survived")

Submission = pd.concat([IDtest,test_Survived],axis=1)
Submission.to_csv("submission.csv",index=False)


################### Bayesina Parameter Tuning###################################################################
from skopt import BayesSearchCV
# Bayesian Optimization
opt = BayesSearchCV(
    estimator = XGBRegressor(
        n_jobs = -1,
        tree_method='approx'
    ),
    search_spaces = {
        'learning_rate': (0.01, 1.0, 'log-uniform'),
        'min_child_weight': (0, 20),
        'max_depth': (3, 5),
#         'max_delta_step': (0, 20),
        'subsample': (0.1, 1.0, 'uniform'),
        'colsample_bytree': (0.1, 1.0, 'uniform'),
        'colsample_bylevel': (0.1, 1.0, 'uniform'),
        'reg_lambda': (1e-3, 1000, 'log-uniform'),
        'reg_alpha': (1e-3, 1.0, 'log-uniform'),
#         'gamma': (1e-9, 0.5, 'log-uniform'),
        'base_score':((1e-4, 0.5, 'log-uniform'))
    },    
    scoring = scoring,
    cv =3,
    n_jobs = -1,
    refit = True,
    random_state = 42
)

opt.fit(X_train_scaled,y_train)

opt.best_params_


#Tuned Model
xgb=XGBRegressor(random_state=51,n_jobs=-1,**opt.best_params_)
xgb.fit(X_train_scaled,y_train)
y_train_pred=xgb.predict(X_train_scaled)
y_val_pred=xgb.predict(X_val_scaled)
y_pred_xgb=np.append(y_train_pred,y_val_pred)
print('Train RMSE:',np.sqrt(mean_squared_error(y_train,y_train_pred)))
print('Test RMSE:',np.sqrt(mean_squared_error(y_val,y_val_pred)))


#cross-val-score
xgb=XGBRegressor(random_state=50,**opt.best_params_)
scores=cross_val_score(xgb,X_train_scaled,y_train,n_jobs=-1,cv=3,scoring=scoring)
print('RMSE: ',np.sqrt(-scores).mean())