connect to spark cluster
from pyspark.conf import SparkConf
from pyspark.sql import SparkSession, Catalog
spark_conf = SparkConf()
spark_conf.setAll([
('spark.master', 'spark://<master-ip>:7077'),
('spark.app.name', 'myApp'),
('spark.submit.deployMode', 'client'),
('spark.ui.showConsoleProgress', 'true'),
('spark.eventLog.enabled', 'false'),
('spark.logConf', 'false'),
('spark.driver.bindAddress', '0.0.0.0'),
('spark.driver.host', '<master-ip>')
])
spark = SparkSession.builder.config(conf=spark_conf).getOrCreate()