关于spark逻辑回归测试qqf

这个代码是使用scala写的，直接粘贴复制到idea中，数据源我不提供，我想按照mysql的格式应该能够自己手动制造一些，这个暂时看动手能力了

贴一份sbt文件清单

ThisBuild / version := "0.1.0-SNAPSHOT"ThisBuild / scalaVersion := "2.11.7"
libraryDependencies += "org.apache.spark" % "spark-core_2.11" % "2.4.8"
// https://mvnrepository.com/artifact/org.apache.spark/spark-mllib
libraryDependencies += "org.apache.spark" %% "spark-mllib" % "2.4.8"
// https://mvnrepository.com/artifact/org.apache.spark/spark-sql
libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.8"
// https://mvnrepository.com/artifact/org.apache.spark/spark-hive
libraryDependencies += "org.apache.spark" %% "spark-hive" % "2.4.8"
// https://mvnrepository.com/artifact/org.apache.spark/spark-streaming
libraryDependencies += "org.apache.spark" %% "spark-streaming" % "2.4.8"https://mvnrepository.com/artifact/org.apache.hive/hive-jdbc
//libraryDependencies += "org.apache.hive" % "hive-jdbc" % "3.1.2"https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch-spark
//libraryDependencies += "org.elasticsearch" %% "elasticsearch-spark" % "2.4.5"
// https://mvnrepository.com/artifact/mysql/mysql-connector-java
libraryDependencies += "mysql" % "mysql-connector-java" % "5.1.47"
// https://mvnrepository.com/artifact/com.crealytics/spark-excel
libraryDependencies += "com.crealytics" %% "spark-excel" % "0.13.5"// https://mvnrepository.com/artifact/org.apache.xmlbeans/xmlbeans
libraryDependencies += "org.apache.xmlbeans" % "xmlbeans" % "3.1.0"// https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml-schemas
libraryDependencies += "org.apache.poi" % "poi-ooxml-schemas" % "4.1.2"
// https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml
libraryDependencies += "org.apache.poi" % "poi-ooxml" % "4.1.2"
// https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml
libraryDependencies += "org.apache.poi" % "poi" % "4.1.2"
lazy val root = (project in file(".")).settings(name := "jixun")

object 逻辑回归 {

def main(args: Array[String]):Unit = {

val spark =

SparkSession.builder()

.appName("MysqlSupport")

.master("local[2]")

.getOrCreate()

lrxunlian(spark)

}

def lrxunlian(spark: SparkSession):Unit = {

val url ="jdbc:mysql://127.0.0.1:3306/park?user=root&password=root&characterEncoding=UTF-8&useSSL=false"

val prop =new Properties()

val df = spark.read.jdbc(url, "hangkongshuju1", prop)

println(df.count())

println(df.rdd.partitions.size)

df.createOrReplaceTempView("t_score")

// sql("select * from t_score where score=98").show()

// df.filter("parameter like '%升%'").filter("sale like '%在售%'").show(10)

var afterdata = df

.filter("BP_SUM != '.'")

.filter("BP_SUM is not null")

.filter("BP_SUM != '-'")

.filter("WEIGHTED_SEG_KM != '.'")

.filter("WEIGHTED_SEG_KM != '-'")

.filter("WEIGHTED_SEG_KM is not null")

.filter("AGE is not null")

.filter("AGE != '-'")

.filter("AGE != '.'")

.filter("id < 100")

.select("id", "BP_SUM", "WEIGHTED_SEG_KM", "AGE", "FLIGHT_COUNT")

.withColumn("AGE", col("AGE").cast("int"))

.withColumn("WEIGHTED_SEG_KM", col("WEIGHTED_SEG_KM").cast("int"))

.withColumn("FLIGHT_COUNT", col("FLIGHT_COUNT").cast("int"))

.withColumn("BP_SUM", col("BP_SUM").cast("int"))

afterdata.show(true)

afterdata.printSchema()

// VectorAssembler 是一个转换器

val assembler =new VectorAssembler()

.setInputCols(Array("AGE", "BP_SUM", "WEIGHTED_SEG_KM"))

.setOutputCol("features")

val dataset = assembler.transform(afterdata)

//拆分成训练集和测试集

val Array(train, test) = dataset.randomSplit(Array(0.8, 0.2), 1234L)

// 设置线性回归参数

val lr1 =new LinearRegression()

.setLabelCol("FLIGHT_COUNT")

.setFeaturesCol("features")

.setFitIntercept(true)// 是否有w0截距

// 设置正则化参数

val lrmodel2 = lr1.setMaxIter(10)// 最大迭代次数

.setRegParam(0.3)

.setElasticNetParam(0.8)

val model = lrmodel2.fit(train)

val result = model.transform(test)

// result.show()

// var insdbdata = result.toDF("1", "2", "3", "4", "5", "6", "7")

// insdbdata.foreach(

// println(_)

// )

insdatabase(result)

// insdbdata.show()

}

def insdatabase(result:DataFrame):Unit = {

val conn: Connection = DriverManager.getConnection(

"jdbc:mysql://localhost:3306/park?characterEncoding=UTF-8&useSSL=false", "root", "root")

result.show()

result.collect().foreach(

row => {

println(row(0), row(1), row(2), row(3), row(4), row(6))

// val sql1 = "INSERT INTO qichexiaoliangbiao (riqi, ctqcxiaoliang, gnsczzdangji,qiyoujiage,rmbdaikuanlilv," +

// "qczongchanl,xnyxiaol,gonglulichengs,qczhengchegupiaozs,xfzxinxinzs) " +

// "VALUES ('" + row(0) + "','" + row(1) + "','" + row(2) + "','" + row(3) + "','" + row(4) + "','" + row(5) + "','" + row(6) + "','" + row(7) + "','" + row(8) + "','" + row(9) + "');"

// println(sql1)

// val ps1 = conn.createStatement()

// ps1.execute(sql1)

val sql ="INSERT INTO xunlianshuju (BP_SUM, WEIGHTED_SEG_KM, AGE ,FLIGHT_COUNT,prediction) " +

"VALUES (?,?,?,?,?);"

val ps: PreparedStatement = conn.prepareStatement(sql)

println(sql)

ps.setString(1, row(1).toString)

ps.setString(2, row(2).toString)

ps.setString(3, row(3).toString)

ps.setString(4, row(4).toString)

ps.setString(5, row(6).toString)

ps.execute()

}

)

}

关于spark逻辑回归测试qqf相关推荐

spark 逻辑回归算法案例_黄美灵的Spark ML机器学习实战
原标题:黄美灵的Spark ML机器学习实战本课程主要讲解基于Spark 2.x的ML,ML是相比MLlib更高级的机器学习库,相比MLlib更加高效.快捷:ML实现了常用的机器学习,如:聚类.分类 ...
基于逻辑回归/决策树/随机森林/多层感知分类器/xgboost/朴素贝叶斯分类的资讯多分类性能对比
在上一篇(https://blog.csdn.net/baymax_007/article/details/82748544)中,利用逻辑回归实现资讯多分类.本文在之前基础上,又引入决策树.随机森林. ...
吴恩达机器学习之线性逻辑回归实现部分
C++实现 "linear_regression.h" //二分类逻辑回归模型 struct elem_log {double y;double* x; //用数组传入自变量数据( ...
逻辑回归预测泰坦尼克号乘客生存率
逻辑回归预测泰坦尼克号乘客生存率描述 RMS泰坦尼克号的沉没是历史上最臭名昭着的沉船之一.1912年4月15日,在她的处女航中,泰坦尼克号在与冰山相撞后沉没,在2224名乘客和机组人员中造成1502 ...
基于Spark的机器学习实践 (七) - 回归算法
0 相关源码 1 回归分析概述 1.1 回归分析介绍 ◆ 回归与分类类似,只不过回归的预测结果是连续的,而分类的预测结果是离散的 ◆ 如此,使得很多回归与分类的模型可以经过改动而通用 ◆ 因此对于回归 ...
逻辑回归梯度下降法详解
引言逻辑回归常用于预测疾病发生的概率,例如因变量是是否恶性肿瘤,自变量是肿瘤的大小.位置.硬度.患者性别.年龄.职业等等(很多文章里举了这个例子,但现代医学发达,可以通过病理检查,即获取标本放到显微 ...
【机器学习】逻辑回归原理及代码
大家好,我是机器侠~ 1 Linear Regression(线性回归) 在了解逻辑回归之前,我们先简单介绍一下Linear Regression(线性回归). 线性回归是利用连续性的变量来预估实际数 ...
大数据利器之神兵淬炼谱:VMware-Linux-Hadoop-Spark-Anaconda-Pycharm琅琊榜
目录大数据神器谱 VMware简介安装注意 Linux简介安装介绍 Hadoop简介安装介绍 Spark简介安装介绍 Pycharm简介安装介绍 Anaconda简介安装介绍安装步骤 ...
泰坦尼克号任务-模型建立和评估
机器学习相关知识理解标签和特征的含义以周志华老师在<机器学习>中判断好瓜的问题为例--给你一个西瓜,如何判断出一个它是不是正熟的好瓜?对于人类来说,根据以前的经验,我们首先会从西瓜这个 ...

关于spark逻辑回归测试qqf

关于spark逻辑回归测试qqf相关推荐

最新文章

热门文章