spark to pandas
pandas_df = some_df.toPandas()
python spark dataframe
# import pyspark class Row from module sql from pyspark.sql import * # Create Example Data - Departments and Employees # Create the Departments department1 = Row(id='123456', name='Computer Science') department2 = Row(id='789012', name='Mechanical Engineering') department3 = Row(id='345678', name='Theater and Drama') department4 = Row(id='901234', name='Indoor Recreation') # Create the Employees Employee = Row("firstName", "lastName", "email", "salary") employee1 = Employee('michael', 'armbrust', 'no-reply@berkeley.edu', 100000) employee2 = Employee('xiangrui', 'meng', 'no-reply@stanford.edu', 120000) employee3 = Employee('matei', None, 'no-reply@waterloo.edu', 140000) employee4 = Employee(None, 'wendell', 'no-reply@berkeley.edu', 160000) employee5 = Employee('michael', 'jackson', 'no-reply@neverla.nd', 80000) # Create the DepartmentWithEmployees instances from Departments and Employees departmentWithEmployees1 = Row(department=department1, employees=[employee1, employee2]) departmentWithEmployees2 = Row(department=department2, employees=[employee3, employee4]) departmentWithEmployees3 = Row(department=department3, employees=[employee5, employee4]) departmentWithEmployees4 = Row(department=department4, employees=[employee2, employee3]) print(department1) print(employee2) print(departmentWithEmployees1.employees[0].email)
Copyright © 2021 Codeinu
Forgot your account's password or having trouble logging into your Account? Don't worry, we'll help you to get back your account. Enter your email address and we'll send you a recovery link to reset your password. If you are experiencing problems resetting your password contact us
