convert pandas dataframe to spark dataframe
import pandas as pd from pyspark.sql import SparkSession filename = <'path to file'> spark = SparkSession.build.appName('pandasToSpark').getOrCreate() # Assuming file is csv pandas_df = pd.read_csv(filename) spark_df = spark.CreateDataFrame(pandas_df)