惠城网站建设有哪些,网站公司怎么找客户,厦门专业网站营销,微盟做一个小程序大概多少钱做数据分析许久了, 简单写写比赛的数据分析项目思路一 使用逻辑回归/随机森林等对kaggle比赛项目 给出泰坦尼克号上的乘客的信息, 预测乘客是否幸存进行简单的数据分析过程, 使用的工具是Jupyter Notebook项目提供了两份数据,分别是titanic_train.csv(训练集,用来构…做数据分析许久了, 简单写写比赛的数据分析项目思路一 使用逻辑回归/随机森林等对kaggle比赛项目 给出泰坦尼克号上的乘客的信息, 预测乘客是否幸存进行简单的数据分析过程, 使用的工具是Jupyter Notebook项目提供了两份数据,分别是titanic_train.csv(训练集,用来构建模型)和test(测试集,用来对模型准确度进行测试)读取并观察数据import numpy as npimport pandas as pddata pd.read_csv(data/titanic_train.csv)# 观察数据结构信息print(data.info())print(data.describe())# 查看前两行数据data.head(2)通过观察可知,训练集共891行,并且Age, Cabin, Embarked字段存在缺失值:PassengerId -- 类似于编号,每个人对应一个Id, 具有唯一性Survived -- 是否幸存, 1表示幸存者, 0则表示否Pclass -- 船舱等级, 1为一等舱, 2为二等舱, 3为三等舱Name -- 姓名Sex -- 性别, female女性, male男性Age -- 年龄(缺失值个数:177,缺失占比:19.8%)SibSp -- 同船配偶以及兄弟姐妹的人数Parch -- 同船父母或者子女的人数Ticket -- 船票号Fare -- 船票的票价Cabin -- 舱位号码(缺失值个数:687,缺失占比:77%)Embarked -- 登船港口(缺失值个数:2,占比:0.2%)特征分析在进行数据建模前必须找出特征因素,找出与幸存有关的特征,特征的好坏直接决定了模型的可靠程度.PassengerId仅仅是标识船员的id和票号Ticket,理论上与是否幸存无关,此特征先行放弃.Cabin船位号码缺失值较多,暂时不考虑.下面继续分析其他特征值域幸存者的关联首先对特征的缺失值进行补充:#对年龄的缺失值取中位数#对登船港口缺失值取S(S港口登船人员最多)data[Age] data[Age].fillna(data[Age].median())data[Embarked] data[Embarked].fillna(S)对特征值进行转换,以便利于建模#将性别转换为0,1data.loc[data[Sex] male, Sex] 0data.loc[data[Sex] female, Sex] 1#将登船港口改为0,1,2data.loc[data[Embarked] S, Embarked] 0data.loc[data[Embarked] C, Embarked] 1data.loc[data[Embarked] Q, Embarked] 2使用data.loc[data[Embarked] S, Embarked] 0首先运用SelectKBest对特征进行一个简单的分析import numpy as npfrom sklearn.feature_selection import SelectKBest, f_classifimport matplotlib.pyplot as pltpredictors [Pclass, Sex, Age, SibSp, Parch, Fare, Embarked]selector SelectKBest(f_classif, k5)selector.fit(data[predictors], data[Survived])scores -np.log10(selector.pvalues_)plt.bar(range(len(predictors)), scores)plt.xticks(range(len(predictors)), predictors, rotationvertical)plt.show()上述可知,Pclass, Sex, Fare, Embarked这几个特征相对比较重要,那我们首先用这几个特征进行建模线性回归算法from sklearn.linear_model import LinearRegressionfrom sklearn.model_selection import KFoldpredictors [Pclass, Sex, Fare, Embarked]# 线性回归 算法alg LinearRegression()kf KFold(n_splits3,random_state 1)predictions []for train,test in kf.split(data):train_predictors (data[predictors].iloc[train,:])train_target data[Survived].iloc[train]alg.fit(train_predictors, train_target)test_predictions alg.predict(data[predictors].iloc[test,:])predictions.append(test_predictions)predictions np.concatenate(predictions, axis0)predictions[predictions 0.5] 1predictions[predictions 0.5] 0accuracy sum(predictions[predictions data[Survived]]) / len(predictions)print(accuracy)0.2615039281705948(结果太差,可忽略)逻辑回归 算法from sklearn.model_selection import cross_val_scorefrom sklearn.linear_model import LogisticRegressionfrom sklearn.model_selection import train_test_splitX data[predictors]y data[Survived]X_train, X_test, y_train, y_test train_test_split(X, y, test_size0.33)alg LogisticRegression(random_state1)alg.fit(X_train, y_train)scores cross_val_score(alg, X_train, y_train, cv3)print(scores.mean())alg.score(X_test,y_test)0.78346801346801340.7762711864406779结果一般,再试试随机森林from sklearn.model_selection import cross_val_score, KFoldfrom sklearn.ensemble import RandomForestClassifierpredictors [Pclass, Sex, Fare, Embarked]alg RandomForestClassifier(random_state1, n_estimators10, min_samples_split2, min_samples_leaf1)alg.fit(X_train, y_train)kf KFold(n_splits3, random_state1)scores cross_val_score(alg, X_train, y_train, cvkf)print(scores.mean())alg.score(X_test,y_test)0.78858941170498950.7864406779661017alg RandomForestClassifier(random_state1, n_estimators100, min_samples_split2, min_samples_leaf1)alg.fit(X_train, y_train)kf KFold(n_splits3, random_state1)scores cross_val_score(alg, X_train, y_train, cvkf)print(scores.mean())alg.score(X_test,y_test)0.79527265959426750.8067796610169492随机森林参数随意取得这只是简单的特征提取和建模分析,后续可以进一步提取特征值,进行更多的建模分析二 使用Lasso/随机森林/SVM等对天池比赛项目 工业蒸汽量预测建模算法进行简单的数据分析过程, 使用的工具是Jupyter Notebookimport warningsimport numpy as npimport pandas as pdimport matplotlib.pyplot as pltimport seaborn as snswarnings.filterwarnings(ignore)train pd.read_table(data/zhengqi_train.txt)test pd.read_table(data/zhengqi_test.txt)train_x train.drop([target],axis1)train_y train[target]data pd.concat([train_x,test])test.head()from sklearn.feature_selection import SelectKBest, f_classiffigsize 15,8figure plt.subplots(figsizefigsize)selector SelectKBest(f_classif, k5)selector.fit(train_x,train_y)scores -np.log10(selector.pvalues_)plt.bar(range(38), scores)plt.xticks(range(38),train_x.columns)# plt.figure(figsize(5,5))plt.show()fig plt.subplots(figsize(30,20))j 1for cols in data.columns:plt.subplot(5,8,j)sns.distplot(train[cols])sns.distplot(test[cols])j1从上面数据来看,特征V5,V9,V11,V17,V22,V28训练集和测试集分布不均,删除类似特征,并且V14对目标值影响较小,所以也删除此特征.# 删除无用特征并进行模型尝试data.drop([V5,V9,V11,V14,V17,V22,V28],axis1,inplaceTrue)#数据分割from sklearn.model_selection import train_test_splitx_train,x_test,y_train,y_test train_test_split(train_x,train_y, test_size 0.3,random_state 0)建模from sklearn.linear_model import Lasso,LinearRegressionfrom sklearn.svm import SVRfrom sklearn.ensemble import RandomForestRegressorimport lightgbm as lgbfrom sklearn.model_selection import KFold, cross_val_scorefrom sklearn.metrics import mean_squared_errordef kfold_scores(alg,x_train, y_train):kf KFold(n_splits 5, random_state 1, shuffleFalse)predict_y []for kf_train,kf_test in kf.split(x_train):alg.fit(x_train.iloc[kf_train],y_train.iloc[kf_train])y_pred_train alg.predict(x_train.iloc[kf_test])mse mean_squared_error(y_train.iloc[kf_test],y_pred_train)predict_y.append(mse)print(交叉验证集MSE均值为 %s % (np.mean(predict_y)))y_pred_test alg.predict(x_test)mse mean_squared_error(y_test, y_pred_test)print(测试集MSE为 %s % mse)alg Lasso(alpha 0.002)mse_mean kfold_scores(alg,x_train,y_train)交叉验证集MSE均值为 0.11694436512799282测试集MSE为 0.11029985071315368alg RandomForestRegressor()mse_mean kfold_scores(alg,x_train,y_train)交叉验证集MSE均值为 0.1404184691171128测试集MSE为 0.1467028448096886其他算法代入也如上即可,后续优化空间:特征值选择,目标值处理,异常值剔除等!alg lgb.LGBMRegressor()mse_mean kfold_scores(alg,x_train,y_train)y alg.predict(test)k y.tolist()with open(data/data.txt,w) as f:for i in k:f.write(str(i) \r\n)f.close()将test的结果输出到txt文件
