from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("sg-executor-memory-check").getOrCreate()
spark.sparkContext.setLogLevel("ERROR")

conf = spark.sparkContext.getConf()

for key in (
    "spark.executor.memory",
    "spark.executor.memoryOverhead",
    "spark.executor.pyspark.memory",
):
    print(f"{key} = {conf.get(key, '<not set>')}")

print(f"defaultParallelism = {spark.sparkContext.defaultParallelism}")
print(f"row_count = {spark.range(0, 1000, 1, 2).count()}")

spark.stop()