spark MLlib之零 构建通用的解析矩阵程序

在使用spark MLlib时，有时候需要使用到一些基础的矩阵（向量），例如：全零矩阵，全一矩阵；以及矩阵之间的运算操作。这里整理了一些常用的矩阵操作方法：

矩阵：

package utils

import java.util.Random

/**

 * 密集矩阵，用于封装模型参数

 */

class DenseMatrix(rowNum: Int, columnNum: Int) extends Serializable{

  var matrix = Array.ofDim[Double](rowNum, columnNum)

  def rows(): Int = {

    rowNum

  }

  def columns(): Int = {

    columnNum

  }

  def apply(i: Int): Array[Double] = {

    matrix(i)

  }

  /**

   * 构造0矩阵

   */

  def zeros(): DenseMatrix = {

    for (i <- 0 until rowNum) {

      for (j <- 0 until columnNum) {

        matrix(i)(j) = 0

      }

    }

    this

  }

  /**

   * 随机初始化矩阵的值

   */

  def rand(): DenseMatrix = {

    val rand = new Random(42)

    for (i <- 0 until rowNum) {

      for (j <- 0 until columnNum) {

        matrix(i)(j) = rand.nextDouble

      }

    }

    this

  }

  def set(i: Int, j: Int, value: Double) {

    matrix(i)(j) = value

  }

  def get(i: Int, j: Int): Double = {

    matrix(i)(j)

  }

  def +(scalar: Double): DenseMatrix = {

    for (i <- 0 until rowNum) yield {

      for (j <- 0 until columnNum) yield {

        matrix(i)(j) += scalar

      }

    }

    this

  }

  def -(scalar: Double): DenseMatrix = {

    this - scalar

  }

  def +(other: DenseMatrix): DenseMatrix = {

    for (i <- 0 until rowNum) yield {

      for (j <- 0 until columnNum) yield {

        matrix(i)(j) += other(i)(j)

      }

    }

    this

  }

  def -(other: DenseMatrix): DenseMatrix = {

    this + (other * (-1))

  }

  def *(scalar: Double): DenseMatrix = {

    for (i <- 0 until rowNum) yield {

      for (j <- 0 until columnNum) yield {

        matrix(i)(j) *= scalar

      }

    }

    this

  }

}

object DenseMatrix {

  def main(args: Array[String]): Unit = {}

}

向量：

package utils

import scala.collection.mutable.HashMap

import org.apache.spark.util.Vector

/**

 * 定义一个基于HashMap的稀疏向量

 */

class SparserVector(dimNum: Int) {

  var elements = new HashMap[Int, Double]

  def insert(index: Int, value: Double) {

    elements += index -> value;

  }

  def *(scale: Double): Vector = {

    var x = new Array[Double](dimNum)

    elements.keySet.foreach(k => x(k) = scale * elements.get(k).get);

    Vector(x)

  }

}

object SparserVector {

  def main(args: Array[String]): Unit = {}

}

亿速云「云服务器」，即开即用、新一代英特尔至强铂金CPU、三副本存储NVMe SSD云盘，价格低至29元/月。点击查看>>