I am getting the correct rows but they are out of order. Is this a bug or am
I doing something wrong?
public class CoordinateMatrixDemo {
public static void main(String[] args) {
//boiler plate needed to run locally
SparkConf conf = new SparkConf().setAppName("Simple
Application").setMaster("local");
JavaSparkContext sc = new JavaSparkContext(conf);
SparkSession spark = SparkSession
.builder()
.appName("CoordinateMatrix")
.getOrCreate()
.newSession();
run(sc,"Data/sparsematrix.txt");
}
private static void run(JavaSparkContext sc, String file) {
//Read coordinate matrix from text or database
JavaRDD<String> fileA = sc.textFile(file);
//map text file with coordinate data (sparse matrix) to
JavaRDD<MatrixEntry>
JavaRDD<MatrixEntry> matrixA = fileA.map(new Function<String,
MatrixEntry>() {
public MatrixEntry call(String x){
String[] indeceValue = x.split(",");
long i = Long.parseLong(indeceValue[0]);
long j = Long.parseLong(indeceValue[1]);
double value =
Double.parseDouble(indeceValue[2]);
return new MatrixEntry(i, j, value );
}
});
//coordinate matrix from sparse data
CoordinateMatrix cooMatrixA = new
CoordinateMatrix(matrixA.rdd());
//create block matrix
BlockMatrix matA = cooMatrixA.toBlockMatrix();
//create block matrix after matrix multiplication (square
matrix)
BlockMatrix ata = matA.transpose().multiply(matA);
//print out the original dense matrix
System.out.println(matA.toLocalMatrix().toString());
//print out the transpose of the dense matrix
System.out.println(matA.transpose().toLocalMatrix().toString());
//print out the square matrix (after multiplication)
System.out.println(ata.toLocalMatrix().toString());
JavaRDD<MatrixEntry> entries =
ata.toCoordinateMatrix().entries().toJavaRDD();
//QR decomposition DEMO
// Convert it to an IndexRowMatrix whose rows are sparse
vectors.
IndexedRowMatrix indexedRowMatrix =
cooMatrixA.toIndexedRowMatrix();
// Drop its row indices.
RowMatrix rowMat = indexedRowMatrix.toRowMatrix();
// QR decomposition
*QRDecomposition<RowMatrix, Matrix> result =
rowMat.tallSkinnyQR(true);*
*System.out.println("Q: " + result.Q().toBreeze().toString());*
System.out.println("R: " + result.R().toString());
Vector[] collectPartitions = (Vector[])
result.Q().rows().collect();
System.out.println("Q factor is:");
for (Vector vector : collectPartitions) {
System.out.println("\t" + vector);
}
//compute Qt
//need to compute d = Qt*b where b is the experimental
//Then solve for d using Gaussian elimination
//Extract Q values and create matrix
//TODO:! The array will be HUGE
String Qm = result.Q().toBreeze().toString();
String[] Qmatrix = Qm.split("\\s+");
int rows = (int)result.Q().numRows();
int cols = (int)result.Q().numCols();
try {
PrintWriter pw = new PrintWriter("Data/qMatrix.txt");
pw.write(Qm);
pw.close();
PrintWriter pw1 = new PrintWriter("Data/qMatrix1.txt");
//write coordinate matrix to file
int k = 0;
for(int i = 0; i < (int)result.Q().numRows();i++){
for(int j = 0; j <
(int)result.Q().numCols();j++){
pw1.println(i + "," + j + "," +
Qmatrix[k]);
k++;
}
}
pw1.close();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//Read coordinate matrix from text or database
JavaRDD<String> fileQ = sc.textFile("Data/qMatrix1.txt");
//somehow map with coordinate data (sparse matrix) to
JavaRDD<MatrixEntry>
JavaRDD<MatrixEntry> matrixQ = fileQ.map(new Function<String,
MatrixEntry>() {
long row = 0;
long col = 0;
public MatrixEntry call(String x){
String[] indeceValue = x.split(",");
long i = Long.parseLong(indeceValue[0]);
long j = Long.parseLong(indeceValue[1]);
double value =
Double.parseDouble(indeceValue[2]);
return new MatrixEntry(i, j, value );
}
});
//coordinate matrix from sparse data
CoordinateMatrix cooMatrixQ = new
CoordinateMatrix(matrixQ.rdd());
//create block matrix
BlockMatrix matQ = cooMatrixQ.toBlockMatrix();
//create block matrix after matrix multiplication (square
matrix)
BlockMatrix Qt = matQ.transpose();
//print out the original dense matrix
System.out.println("Matrix Q!!!! " +
matQ.toLocalMatrix().toString());
//create vector from coordinate matrix (text or database)
//Read coordinate matrix from text or database
JavaRDD<String> fileB = sc.textFile("Data/sparsevector.txt");
//map text file with coordinate data (sparse matrix) to
JavaRDD<MatrixEntry>
JavaRDD<MatrixEntry> matrixV = fileB.map(new Function<String,
MatrixEntry>() {
public MatrixEntry call(String x){
String[] indeceValue = x.split(",");
long i = Long.parseLong(indeceValue[0]);
long j = Long.parseLong(indeceValue[1]);
double value =
Double.parseDouble(indeceValue[2]);
return new MatrixEntry(i, j, value );
}
});
//coordinate matrix from sparse data
CoordinateMatrix cooMatrixV = new
CoordinateMatrix(matrixV.rdd());
//create block matrix
BlockMatrix matB = cooMatrixV.toBlockMatrix();
BlockMatrix matD = Qt.multiply(matB);
//we have A and b (Can solve Ax = b now)
//b is just a m x 1 matrix
Matrix v = matB.toLocalMatrix();
System.out.println(v);
DenseMatrix r = (DenseMatrix) result.R();
System.out.println(r);
double[] A = r.toArray();
double[] b = v.toArray();
double[] d = matD.toLocalMatrix().toArray();
double[][] m = new double[r.numRows()][r.numCols()];
int k = 0;
for(int i = 0; i < r.numRows();i++){
for(int j = 0;j < r.numCols();j++ ){
m[j][i] = A[k];
k++;
}
}
GaussianElimination ge = new GaussianElimination();
double[] resultV =ge.lsolve(m, d);
for(int i = 0; i < 3;i++){
System.out.println(resultV[i]);
}
//NONE of these commands work!
//JavaRDD<String> output = entries.map(new
Function<MatrixEntry, String>()
{
// public String call(MatrixEntry e) {
// return String.format("%d,%d,%s", e.i(), e.j(),
e.value());
// }
//});
//entries.saveAsTextFile("Data/output1");
//output.saveAsTextFile("Data/output1");
//ata.toCoordinateMatrix().entries().toJavaRDD().saveAsTextFile("Data/Output1");
}
}
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/TallSkinnyQR-tp28039.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe e-mail: user-unsubscr...@spark.apache.org
