remove duplicates python
mylist = ["a", "b", "a", "c", "c"]
mylist = list(dict.fromkeys(mylist))remove duplicates python
mylist = ["a", "b", "a", "c", "c"]
mylist = list(dict.fromkeys(mylist))remove duplicates function python
def remove_dupiclates(list_):
new_list = []
for a in list_:
if a not in new_list:
new_list.append(a)
return new_listReturn a new DataFrame with duplicate rows removed
# Return a new DataFrame with duplicate rows removed
from pyspark.sql import Row
df = sc.parallelize([
Row(name='Alice', age=5, height=80),
Row(name='Alice', age=5, height=80),
Row(name='Alice', age=10, height=80)]).toDF()
df.dropDuplicates().show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# | 5| 80|Alice|
# | 10| 80|Alice|
# +---+------+-----+
df.dropDuplicates(['name', 'height']).show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# | 5| 80|Alice|
# +---+------+-----+Pandas drop duplicates
df = df.drop_duplicates(subset = ["b"])Copyright © 2021 Codeinu
Forgot your account's password or having trouble logging into your Account? Don't worry, we'll help you to get back your account. Enter your email address and we'll send you a recovery link to reset your password. If you are experiencing problems resetting your password contact us
