Up

sparktk module

# vim: set encoding=utf-8

#  Copyright (c) 2016 Intel Corporation 
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import sys
import os

try:
    import pyspark
except ImportError:
    SPARK_HOME="SPARK_HOME"
    if SPARK_HOME in os.environ:
        spark_home = os.environ.get(SPARK_HOME)
        pyspark_path = "%s/python" % spark_home
        sys.path.insert(1, pyspark_path)
    else:
        raise Exception("Required Environment variable %s not set" % SPARK_HOME)
    del SPARK_HOME

from tkcontext import TkContext
from sparkconf import create_sc

import dtypes
from sparktk.loggers import loggers
from sparktk.frame.ops.inspect import inspect_settings


(Note: This is documentation for the complete sparktk python package. For general usage, see the main Python APIs documentation)

Module variables

var pyspark_path

var spark_home

Sub-modules

sparktk.arguments

Type checking for arguments, including the implicit argument

sparktk.dtypes

definitions for Data Types

sparktk.loggers

Logging - simple helpers for now

sparktk.sparkconf

Sets up Spark Context