Answers for "remove the duplciates based on row in pandas . delete the rows where all the values are same"

remove row if all are the same value pandas

print (df['S'] != df['T'])
0    False
1     True
2     True
3     True
4    False
5     True
6     True
7     True
8    False
dtype: bool

df = df[df['S'] != df['T']]
print (df)
   S  T  W           U
1  A  B  0  Undirected
2  A  C  1  Undirected
3  B  A  0  Undirected
5  B  C  1  Undirected
6  C  A  1  Undirected
7  C  B  1  Undirected

Return a new DataFrame with duplicate rows removed

# Return a new DataFrame with duplicate rows removed

from pyspark.sql import Row
df = sc.parallelize([
  Row(name='Alice', age=5, height=80),
  Row(name='Alice', age=5, height=80),
  Row(name='Alice', age=10, height=80)]).toDF()
df.dropDuplicates().show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# |  5|    80|Alice|
# | 10|    80|Alice|
# +---+------+-----+

df.dropDuplicates(['name', 'height']).show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# |  5|    80|Alice|
# +---+------+-----+