Изменения
Нет описания правки
libraryDependencies += "org.apache.spark" %% "spark-core" % "2.4.0"
libraryDependencies += "org.apache.spark" %% "spark-mllib" % "2.4.0" % "runtime"
Пример линейной регрессии c применением org.apache.spark.ml.regression.LinearRegression<ref>[https://spark.apache.org/docs/latest/ml-classification-regression.html#linear-regression Spark ML, LinearRegression]</ref>:
val training = spark.read.format("libsvm")
.load("linear_regression.txt")
libraryDependencies += "com.github.haifengl" %% "smile-scala" % "1.5.2"
Пример ридж и лассо регрессии c применением smile.regression<ref>[https://haifengl.github.io/smile/regression.html Smile, Regression]</ref>:
import smile.data.{AttributeDataset, NumericAttribute}
import smile.read
import smile.regression.{LASSO, RidgeRegression, lasso, ridge}
val data: AttributeDataset = read.table("regression.txt", delimiter = " ", response = Some((new NumericAttribute("class"), 0)))
val x: Array[Array[Double]] = data.x()
println(ridgeRegression)
println(lassoRegression)
===Логистическая регрессия===
Sbt зависимость:
libraryDependencies += "org.apache.spark" %% "spark-core" % "2.4.0"
libraryDependencies += "org.apache.spark" %% "spark-mllib" % "2.4.0" % "runtime"
Пример логистической регрессии c применением spark.mllib.classification<ref>[https://spark.apache.org/docs/2.3.1/mllib-linear-methods.html#logistic-regression Spark, Logistic Regression]</ref>:
import org.apache.spark.mllib.classification.{LogisticRegressionModel, LogisticRegressionWithLBFGS}
import org.apache.spark.mllib.evaluation.MulticlassMetrics
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.util.MLUtils
val data = MLUtils.loadLibSVMFile(sc, "logisticRegresion.txt")
val splits = data.randomSplit(Array(0.6, 0.4), seed = 11L)
val training = splits(0).cache()
val test = splits(1)
val model = new LogisticRegressionWithLBFGS()
.setNumClasses(10)
.run(training)
val predictionAndLabels = test.map { case LabeledPoint(label, features) =>
val prediction = model.predict(features)
(prediction, label)
}
val metrics = new MulticlassMetrics(predictionAndLabels)
val accuracy = metrics.accuracy
println(accuracy)
== Примечания ==
<references/>
