Spark提交任务

1、Spark提交任务到Yarn

1.1、DwKuduApp

spark-submit --class com.io.etl.dwkudu.DwKuduApp \
--files /etl/etl-dwkudu/conf/doris.property,/etl/etl-dwkudu/conf/redis.property,/etl/etl-dwkudu/conf/log4j.property \
--master yarn --deploy-mode cluster \
--driver-memory 2g \
--executor-memory 7g \
--conf "spark.executor.cores=2" \
--conf "spark.yarn.maxAppAttempts=0" \
--conf "spark.task.maxFailures=1" \
--conf "spark.dynamicAllocation.enabled=false" \
--conf "spark.executor.instances=8" \
--conf "spark.yarn.executor.memoryOverhead=1024m" \
--conf "spark.executor.extraJavaOptions=-Dlog4j.configration=log4j_dwkudu.properties -XX:+UseG1GC" \
--conf "spark.streaming.kafka.maxRatePerPartition=$datacount" \
/etl/etl-dw-kudu/etl-dw-kudu.jar $period > /etl-log/etl-dw-kudu/etl-dw-kudu.log

1.1、GateAppetl

spark-submit --class com.io.etl.gate.GateAppetl \
--files /etl/gate/conf/doris.property,/etl/gate/conf/redis.property,/etl/gate/conf/log4j.property \
--master yarn --deploy-mode cluster \
--driver-memory 600m \
--executor-memory 1g \
--conf "spark.executor.cores=1" \
--conf "spark.yarn.maxAppAttempts=0" \
--conf "spark.yarn.task.maxFailures=1" \
--conf "spark.dynamicAllocation.enabled=false" \
--conf "spark.executor.instances=8" \
--conf "spark.yarn.executor.memoryOverhead=1524m" \
--conf "spark.executor.extraJavaOptions=-Dlog4j.configuration=log4j_gate.properties -XX:+UseG1GC -verbose:gc -XX:PrintGCTimeStamps" \
--conf "spark.streaming.kafka.maxRatePerPartition=1000" \
/etl/etl-gate/etl-gate.jar 10 > /etl-log/etl-dw-kudu/etl-gate.log

1.1、PayKuDuIdApp

spark-submit --class com.io.etl.PayKuDuIdApp \
--files /etl/gate/conf/doris.property,/etl/gate/conf/redis.property,/etl/gate/conf/log4j.property \
--master yarn --deploy-mode cluster \
--driver-memory 2048m \
--executor-memory 4g \
--conf "spark.executor.cores=1" \
--conf "spark.yarn.maxAppAttempts=0" \
--conf "spark.yarn.task.maxFailures=1" \
--conf "spark.dynamicAllocation.enabled=false" \
--conf "spark.executor.instances=4" \
--conf "spark.yarn.executor.memoryOverhead=2048m" \
--conf "spark.driver.extraJavaOptions=-Dfileencoding=utf-8" \
--conf "spark.executor.extraJavaOptions=-Dlog4j.configuration=log4j_id.properties -Dfile.encoding=utf-8 -XX:+UseG1GC -verbose:gc -XX:PrintGCDetails +XX:+PrintGCTimeStamps" \
--conf "spark.streaming.kafka.maxRatePerPartition=12000" \
/etl/etl-id-kudu/etl-id-kudu.jar 2 3000 > /etl-log/etl-id-kudu/etl-id-kudu.log