调优手册
wds.linkis.rpc.receiver.asyn.consumer.thread.max=400
wds.linkis.rpc.receiver.asyn.queue.size.max=5000
wds.linkis.rpc.sender.asyn.consumer.thread.max=100
wds.linkis.rpc.sender.asyn.queue.size.max=2000
private val groupNameToGroups = new JMap[String, Group]
override def getOrCreateGroup(groupName: String): Group = {
if (!groupNameToGroups.containsKey(groupName)) synchronized {
val initCapacity = 100
val maxCapacity = 100
// 其它代码...
在linkis-engineconn-plugins/engineconn-plugins里的Spark的部分配置,可以调整该配置改变提交到Yarn上的任务运行时环境,由于篇幅有限,如更多有关Hive,Yarn配置需要用户参考源码和参数文档:
"spark.driver.memory" = 2 //单位为G
"wds.linkis.driver.cores" = 1
"spark.executor.memory" = 4 //单位为G
"spark.executor.cores" = 2
"spark.executor.instances" = 3