SOC

from pyspark import SparkConf, SparkContext
from pyspark.sql import SQLContext
from operator import add
import sys

# based on
# SELECT SUM(salary)
# FROM employees
# WHERE salary > 1000
# GROUP by deptname

## Constants

APP_NAME = " sql test"

##OTHER FUNCTIONS/CLASSES

## Main functionality

def main(sc,filename):
    sqlContext = SQLContext(sc)
    datCtx = sqlContext.read.json(filename)
    datCtx.registerTempTable("datTbl")
    res = sqlContext.sql("SELECT department, SUM(salary) FROM datTbl
WHERE salary > 1 GROUP by department")
    res.show()

if __name__ == "__main__":
    # Configure Spark
    conf = SparkConf().setAppName(APP_NAME)
    conf = conf.setMaster("local[*]")
    sc   = SparkContext(conf=conf)
    filename = sys.argv[1]
    # Execute Main functionality
    main(sc, filename)

# simple
sqlContext = SQLContext(sc)
datCtx = sqlContext.read.json(filename)
datCtx.collect()

# simple stats
sqlContext = SQLContext(sc)
datCtx = sqlContext.read.json(filename)
datCtx.describe()

>>> df.describe().show()
+-------+------------------+
|summary|               age|
+-------+------------------+
|  count|                 2|
|   mean|               3.5|
| stddev|2.1213203435596424|
|    min|                 2|
|    max|                 5|
+-------+------------------+
>>> df.describe(['age', 'name']).show()
+-------+------------------+-----+
|summary|               age| name|
+-------+------------------+-----+
|  count|                 2|    2|
|   mean|               3.5| null|
| stddev|2.1213203435596424| null|
|    min|                 2|Alice|
|    max|                 5|  Bob|
+-------+------------------+-----+

import numpy as np

# https://en.wikipedia.org/wiki/Positive-definite_matrix

# symmetric matrix: a square matrix that is equal to its transpose.
# Formally, matrix A is symmetric if A = A^T

# Positive-definite matrix: a symmetric n × n real matrix M is said to be
# positive definite if the scalar z^T M z is positive for every non-zero
# column vector z of n real numbers.


id2x2 = np.matrix([[1., 0.],
                   [0., 1.]])

m = np.matrix([[2., -1., 0.]
               [-1., 2., -1.],
               [0., -1., 2.]])