spark.shuffle.manager

Table of Contents

org.apache.spark.shuffle.sort.SortShuffleManager

(org.apache.spark.shuffle.sort.SortShuffleManager)[https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala]

org.apache.spark.shuffle.celeborn.SparkShuffleManager

--conf spark.shuffle.manager=org.apache.spark.shuffle.celeborn.SparkShuffleManager \
--conf spark.shuffle.service.enabled=false

org.apache.spark.shuffle.celeborn.SparkShuffleManager

org.apache.spark.shuffle.RssShuffleManager

spark.shuffle.manager org.apache.spark.shuffle.RssShuffleManager
spark.rss.coordinator.quorum <coordinatorIp1>:19999,<coordinatorIp2>:19999

https://uniffle.apache.org/docs/intro

org.apache.spark.shuffle.Ors2ShuffleManager

spark.dynamicAllocation.enabled                       false
spark.shuffle.service.enabled                         false
spark.executor.extraClassPath                         rss-xxx-client.jar
spark.shuffle.manager                                 org.apache.spark.shuffle.Ors2ShuffleManager
spark.shuffle.rss.serviceManager.type                 master
spark.shuffle.rss.serviceRegistry.zookeeper.servers  10.10.10.1:2181
spark.shuffle.rss.dataCenter                         dc1
spark.shuffle.rss.cluster                            cluster1

https://github.com/cubefs/shuttle

gluten velox

gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar