org.apache.spark.sql.api.r

Class SQLUtils

Object

org.apache.spark.sql.api.r.SQLUtils

public class SQLUtils
extends Object

Constructor Summary

Constructors

Constructor and Description
SQLUtils()

Method Summary

Methods

Modifier and Type	Method and Description
static Dataset<Row>	createDF(RDD<byte[]> rdd, StructType schema, SparkSession sparkSession)
static StructField	createStructField(String name, String dataType, boolean nullable)
static StructType	createStructType(scala.collection.Seq<StructField> fields)
static Dataset<Row>	dapply(Dataset<Row> df, byte[] func, byte[] packageNames, Object[] broadcastVars, StructType schema) The helper function for dapply() on R side.
static Object[][]	dfToCols(Dataset<Row> df)
static JavaRDD<byte[]>	dfToRowRDD(Dataset<Row> df)
static Dataset<Row>	gapply(RelationalGroupedDataset gd, byte[] func, byte[] packageNames, Object[] broadcastVars, StructType schema) The helper function for gapply() on R side.
static JavaSparkContext	getJavaSparkContext(SparkSession spark)
static SparkSession	getOrCreateSparkSession(JavaSparkContext jsc, java.util.Map<Object,Object> sparkConfigMap, boolean enableHiveSupport)
static java.util.Map<String,String>	getSessionConf(SparkSession spark)
static DataType	getSQLDataType(String dataType)
static String[]	getTableNames(SparkSession sparkSession, String databaseName)
static Dataset<Row>	getTables(SparkSession sparkSession, String databaseName)
static Dataset<Row>	loadDF(SparkSession sparkSession, String source, java.util.Map<String,String> options)
static Dataset<Row>	loadDF(SparkSession sparkSession, String source, StructType schema, java.util.Map<String,String> options)
static Object	readSqlObject(java.io.DataInputStream dis, char dataType)
static SaveMode	saveMode(String mode)
static StructType	SERIALIZED_R_DATA_SCHEMA()
static void	setSparkContextSessionConf(SparkSession spark, java.util.Map<Object,Object> sparkConfigMap)
static boolean	writeSqlObject(java.io.DataOutputStream dos, Object obj)

Methods inherited from class Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

SQLUtils

public SQLUtils()

Method Detail

getOrCreateSparkSession

public static SparkSession getOrCreateSparkSession(JavaSparkContext jsc,
                                   java.util.Map<Object,Object> sparkConfigMap,
                                   boolean enableHiveSupport)

setSparkContextSessionConf

public static void setSparkContextSessionConf(SparkSession spark,
                              java.util.Map<Object,Object> sparkConfigMap)

getSessionConf

public static java.util.Map<String,String> getSessionConf(SparkSession spark)

getJavaSparkContext

public static JavaSparkContext getJavaSparkContext(SparkSession spark)

createStructType

public static StructType createStructType(scala.collection.Seq<StructField> fields)

getSQLDataType

public static DataType getSQLDataType(String dataType)

createStructField

public static StructField createStructField(String name,
                            String dataType,
                            boolean nullable)

createDF

public static Dataset<Row> createDF(RDD<byte[]> rdd,
                    StructType schema,
                    SparkSession sparkSession)

dfToRowRDD

public static JavaRDD<byte[]> dfToRowRDD(Dataset<Row> df)

SERIALIZED_R_DATA_SCHEMA

public static StructType SERIALIZED_R_DATA_SCHEMA()

dapply

public static Dataset<Row> dapply(Dataset<Row> df,
                  byte[] func,
                  byte[] packageNames,
                  Object[] broadcastVars,
                  StructType schema)

The helper function for dapply() on R side.

Parameters:

df - (undocumented)

func - (undocumented)

packageNames - (undocumented)

broadcastVars - (undocumented)

schema - (undocumented)

Returns:

(undocumented)

gapply

public static Dataset<Row> gapply(RelationalGroupedDataset gd,
                  byte[] func,
                  byte[] packageNames,
                  Object[] broadcastVars,
                  StructType schema)

The helper function for gapply() on R side.

Parameters:

gd - (undocumented)

func - (undocumented)

packageNames - (undocumented)

broadcastVars - (undocumented)

schema - (undocumented)

Returns:

(undocumented)

dfToCols

public static Object[][] dfToCols(Dataset<Row> df)

saveMode

public static SaveMode saveMode(String mode)

loadDF

public static Dataset<Row> loadDF(SparkSession sparkSession,
                  String source,
                  java.util.Map<String,String> options)

loadDF

public static Dataset<Row> loadDF(SparkSession sparkSession,
                  String source,
                  StructType schema,
                  java.util.Map<String,String> options)

readSqlObject

public static Object readSqlObject(java.io.DataInputStream dis,
                   char dataType)

writeSqlObject

public static boolean writeSqlObject(java.io.DataOutputStream dos,
                     Object obj)

getTables

public static Dataset<Row> getTables(SparkSession sparkSession,
                     String databaseName)

getTableNames

public static String[] getTableNames(SparkSession sparkSession,
                     String databaseName)