from pathlib import Path

from pyspark.sql import SparkSession


def load_partition(values):
    from sg_metrics import describe_value
    return [describe_value(value) for value in values]


spark = SparkSession.builder.appName("sg-py-dependency-package").getOrCreate()
spark.sparkContext.setLogLevel("ERROR")

result = spark.sparkContext.parallelize([1, 2, 3], 2).mapPartitions(load_partition).collect()
pyfiles = [Path(uri).name for uri in spark.sparkContext.listFiles if uri.endswith("sg_metrics.zip")]

print("dependency_import=" + ",".join(result))
print("pyfile_added=" + ",".join(pyfiles))

spark.stop()