fix for Spark 2.1.0

Author: jli05

build.sbt

@@ -43,14 +43,7 @@ libraryDependencies ++= Seq(
 )
 
 
-//{
-//  val defaultHadoopVersion = "[2.6.0,)"
-//  val hadoopVersion =
-//    scala.util.Properties.envOrElse("SPARK_HADOOP_VERSION", defaultHadoopVersion)
-//  libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion
-//}
-
-mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
+assemblyMergeStrategy in assembly ~= (old =>
   {
     case PathList("javax", "servlet", xs @ _*)               => MergeStrategy.first
     case PathList(ps @ _*) if ps.last endsWith ".html"       => MergeStrategy.first
@@ -62,12 +55,12 @@ mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
     case m if m.toLowerCase.startsWith("meta-inf/services/") => MergeStrategy.filterDistinctLines
     case _ => MergeStrategy.first
   }
-}
+)
 
 
 spName := "FurongHuang/spectrallda-tensorspark"
 
-sparkVersion := "2.0.2"
+sparkVersion := "2.1.0"
 
 sparkComponents += "mllib"
 

src/main/scala/edu/uci/eecs/spectralLDA/algorithm/ALS.scala

@@ -120,7 +120,7 @@ class ALS(dimK: Int,
                                  B: DenseMatrix[Double],
                                  C: DenseMatrix[Double]): (DenseMatrix[Double], DenseVector[Double]) = {
     val updatedA = unfoldedM3 * TensorOps.krprod(C, B) * TensorOps.to_invert(C, B)
-    val lambda = norm(updatedA(::, *)).toDenseVector
+    val lambda = norm(updatedA(::, *)).t.toDenseVector
     (AlgebraUtil.matrixNormalization(updatedA), lambda)
   }
 }

src/main/scala/edu/uci/eecs/spectralLDA/algorithm/TensorLDA.scala

@@ -6,7 +6,7 @@ package edu.uci.eecs.spectralLDA.algorithm
  * Created by Furong Huang on 11/2/15.
  */
 import edu.uci.eecs.spectralLDA.datamoments.DataCumulant
-import breeze.linalg.{DenseMatrix, DenseVector, SparseVector, argtopk, diag, max, min}
+import breeze.linalg.{DenseMatrix, DenseVector, SparseVector, argsort, diag, max, min}
 import breeze.numerics._
 import breeze.stats.distributions.{Rand, RandBasis}
 import edu.uci.eecs.spectralLDA.utils.NonNegativeAdjustment
@@ -70,7 +70,7 @@ class TensorLDA(dimK: Int,
     val topicWordMatrixUnordered: DenseMatrix[Double] = unwhiteningMatrix * nu * diag(lambda)
 
     // re-arrange alpha and topicWordMatrix in descending order of alpha
-    val idx = argtopk(alphaUnordered, dimK)
+    val idx = argsort(alphaUnordered).reverse.take(dimK)
     val alpha = alphaUnordered(idx).toDenseVector
     val topicWordMatrix = topicWordMatrixUnordered(::, idx).toDenseMatrix
 

src/main/scala/edu/uci/eecs/spectralLDA/algorithm/TensorLDAModel.scala

@@ -106,7 +106,8 @@ class TensorLDAModel(val topicWordDistribution: DenseMatrix[Double],
     val smoothedBeta: DenseMatrix[Double] = topicWordDistribution * (1 - smoothing)
     smoothedBeta += DenseMatrix.ones[Double](vocabSize, k) * (smoothing / vocabSize)
 
-    assert(sum(smoothedBeta(::, *)).toDenseVector.forall(a => abs(a - 1) <= 1e-10))
+    assert(sum(smoothedBeta(::, *)).t.toDenseVector
+      .forall(a => abs(a - 1) <= 1e-10))
     assert(smoothing < 1e-4 || smoothedBeta.forall(_ > 1e-10))
 
     smoothedBeta
@@ -134,4 +135,4 @@ private[algorithm] object TensorLDAModel {
     val a = max(x)
     a + log(sum(exp(x :- a)))
   }
-}
+}

src/main/scala/edu/uci/eecs/spectralLDA/utils/AlgebraUtil.scala

@@ -13,7 +13,7 @@ object AlgebraUtil {
 
   def matrixNormalization(B: DenseMatrix[Double]): DenseMatrix[Double] = {
     val A: DenseMatrix[Double] = B.copy
-    val colNorms: DenseVector[Double] = norm(A(::, *)).toDenseVector
+    val colNorms: DenseVector[Double] = norm(A(::, *)).t.toDenseVector
 
     for (i <- 0 until A.cols optimized) {
       A(::, i) :/= colNorms(i)

src/main/scala/edu/uci/eecs/spectralLDA/utils/TensorOps.scala

@@ -10,12 +10,12 @@ import scala.reflect.ClassTag
 object TensorOps {
   /** Complex matrix norm */
   def matrixNorm(m: DenseMatrix[Complex]): Double = {
-    norm(norm(m(::, *)).toDenseVector)
+    norm(norm(m(::, *)).t.toDenseVector)
   }
 
   /** Double matrix norm */
   def dmatrixNorm(m: DenseMatrix[Double]): Double = {
-    norm(norm(m(::, *)).toDenseVector)
+    norm(norm(m(::, *)).t.toDenseVector)
   }
 
   /** Unfold 3rd-order tensor */
@@ -89,4 +89,4 @@ object TensorOps {
     }
     prod
   }
-}
+}

src/test/scala/edu/uci/eecs/spectralLDA/algorithm/TensorLDATest.scala

@@ -66,7 +66,7 @@ class TensorLDATest extends FlatSpec with Matchers {
 
     // Rearrange the elements/columns of fitted_alpha and fitted_beta
     // to the order of initial alpha and beta
-    val idx = argtopk(fitted_alpha, 3)
+    val idx = argsort(fitted_alpha).reverse.take(3)
     val sorted_beta = fitted_beta(::, idx).toDenseMatrix
     // if one vector is all negative, multiply it by -1 to turn it positive
     for (j <- 0 until sorted_beta.cols) {
@@ -79,7 +79,7 @@ class TensorLDATest extends FlatSpec with Matchers {
     val diff_beta: DenseMatrix[Double] = sorted_beta - allTokenDistributions
     val diff_alpha: DenseVector[Double] = sorted_alpha - alpha
 
-    val norm_diff_beta = norm(norm(diff_beta(::, *)).toDenseVector)
+    val norm_diff_beta = norm(norm(diff_beta(::, *)).t.toDenseVector)
     val norm_diff_alpha = norm(diff_alpha)
 
     info(s"Expecting alpha: $alpha")
@@ -107,7 +107,7 @@ class TensorLDATest extends FlatSpec with Matchers {
 
     val s = sum(allTokenDistributions(::, *))
     val normalisedAllTokenDistributions: DenseMatrix[Double] =
-      allTokenDistributions * diag(1.0 / s.toDenseVector)
+      allTokenDistributions * diag(1.0 / s.t.toDenseVector)
 
     val documents = simulateLDAData(
       alpha,
@@ -130,7 +130,7 @@ class TensorLDATest extends FlatSpec with Matchers {
 
     // Rearrange the elements/columns of fitted_alpha and fitted_beta
     // to the order of initial alpha and beta
-    val idx = argtopk(fitted_alpha, dimK)
+    val idx = argsort(fitted_alpha).reverse.take(dimK)
     val sorted_beta = fitted_beta(::, idx).toDenseMatrix
     val sorted_alpha = fitted_alpha(idx).toDenseVector
 
@@ -140,7 +140,7 @@ class TensorLDATest extends FlatSpec with Matchers {
     val diff_beta: DenseMatrix[Double] = sorted_beta - expected_beta
     val diff_alpha: DenseVector[Double] = sorted_alpha - expected_alpha
 
-    val norm_diff_beta = norm(norm(diff_beta(::, *)).toDenseVector)
+    val norm_diff_beta = norm(norm(diff_beta(::, *)).t.toDenseVector)
     val norm_diff_alpha = norm(diff_alpha)
 
     info(s"Expecting alpha: $expected_alpha")
@@ -154,4 +154,4 @@ class TensorLDATest extends FlatSpec with Matchers {
     norm_diff_beta should be <= 0.025
     norm_diff_alpha should be <= 3.5
   }
-}
+}

src/test/scala/edu/uci/eecs/spectralLDA/utils/RandNLATest.scala

@@ -45,7 +45,7 @@ class RandNLATest extends FlatSpec with Matchers {
     val expectedResult = m2 * g
 
     val diff: DenseMatrix[Double] = result - expectedResult
-    val normDiff: Double = norm(norm(diff(::, *)).toDenseVector)
+    val normDiff: Double = norm(norm(diff(::, *)).t.toDenseVector)
     normDiff should be <= 1e-8
   }
 
@@ -59,7 +59,7 @@ class RandNLATest extends FlatSpec with Matchers {
     val alpha: DenseVector[Double] = DenseVector[Double](25.0, 20.0, 15.0, 10.0, 5.0)
     val beta: DenseMatrix[Double] = DenseMatrix.rand(n, k, Uniform(0.0, 1.0))
 
-    val norms = norm(beta(::, *)).toDenseVector
+    val norms = norm(beta(::, *)).t.toDenseVector
     for (j <- 0 until k) {
       beta(::, j) /= norms(j)
     }
@@ -72,7 +72,7 @@ class RandNLATest extends FlatSpec with Matchers {
     val (s: DenseVector[Double], u: DenseMatrix[Double]) = RandNLA.decomp2(a * q, q)
 
     val diff_a = u * diag(s) * u.t - a
-    val norm_diff_a = norm(norm(diff_a(::, *)).toDenseVector)
+    val norm_diff_a = norm(norm(diff_a(::, *)).t.toDenseVector)
 
     norm_diff_a should be <= 1e-8
   }

src/test/scala/edu/uci/eecs/spectralLDA/utils/TensorOpsTest.scala

@@ -42,6 +42,6 @@ class TensorOpsTest extends FlatSpec with Matchers {
     val expected = expectedRankOneTensor3d(x, y, z)
 
     val diff = tensor - expected
-    norm(norm(diff(::, *)).toDenseVector) should be <= 1e-8
+    norm(norm(diff(::, *)).t.toDenseVector) should be <= 1e-8
   }
-}
+}