#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Default system properties included when running spark-submit.
# This is useful for setting default environmental settings.

# Example:

#spark.submit.deployMode          client 
#spark.driver.extraJavaOptions    -javaagent:"/data/kyuubi/spark-3.3.0-bin-hadoop2/jars/aspectjweaver-1.9.9.1.jar"

spark.submit.deployMode          cluster
spark.master                     yarn
spark.driver.extraJavaOptions    -javaagent:aspectjweaver-1.9.9.1.jar -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=13200
#spark.yarn.dist.jars         hdfs://newns:8020/user/aloudata/kyuubi/aspectjweaver-1.9.9.1.jar
spark.yarn.dist.jars file:///data/kyuubi/aspectjweaver-1.9.9.1.jar

spark.yarn.submit.waitAppCompletion false

spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog
spark.sql.catalog.spark_catalog.type=hive
spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
spark.sql.iceberg.vectorization.enabled=false

spark.driver.cores=1
spark.driver.memory=6g
spark.executor.cores=4
spark.executor.memory=10g
spark.shuffle.service.enabled=true
spark.dynamicAllocation.minExecutors=1
spark.dynamicAllocation.maxExecutors=10
spark.kryoserializer.buffer.max=2000m
spark.files.useFetchCache=true
spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes=256m
spark.sql.adaptive.autoBroadcastJoinThreshold=30m
spark.sql.adaptive.coalescePartitions.initialPartitionNum=8192
spark.sql.adaptive.skewJoin.skewedPartitionFactor=5
spark.sql.adaptive.forceApply=false
spark.memory.storageFraction=0.4
spark.sql.adaptive.enabled=true
spark.sql.adaptive.coalescePartitions.minPartitionNum=1
spark.sql.legacy.allowNegativeScaleOfDecimal=true
spark.sql.adaptive.advisoryPartitionSizeInBytes=256m
spark.sql.autoBroadcastJoinThreshold=-1
spark.sql.adaptive.coalescePartitions.enabled=true
spark.blacklist.enabled=true
spark.sql.legacy.charVarcharAsString=true
spark.sql.adaptive.logLevel=info
spark.sql.adaptive.skewJoin.enabled=true
spark.dynamicAllocation.enabled=true
spark.sql.adaptive.fetchShuffleBlocksInBatch=true
spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin=0.2
spark.dynamicAllocation.initialExecutors=1
spark.sql.sources.parallelPartitionDiscovery.parallelism=60
spark.shuffle.service.port=7337
spark.memory.fraction=0.6
spark.sql.adaptive.localShuffleReader.enabled=true
spark.dynamicAllocation.executorIdleTimeout=180s
spark.dynamicAllocation.cachedExecutorIdleTimeout=30min
spark.sql.legacy.timeParserPolicy=LEGACY
spark.serializer=org.apache.spark.serializer.KryoSerializer
spark.files.overwrite=true
spark.dynamicAllocation.executorAllocationRatio=0.5
spark.ui.killEnabled=false
spark.sql.sources.partitionOverwriteMode=dynamic
#spark.sql.crossJoin.enabled=false

# spark.eventLog.enabled           true
# spark.eventLog.dir               hdfs://namenode:8021/directory
# spark.serializer                 org.apache.spark.serializer.KryoSerializer
# spark.driver.memory              5g
# spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"