priority_orders.write.format("avro").mode("overwrite").save(output_path) result = spark.read.format("avro").load(output_path) actual_rows = [tuple(row) for row in result.orderBy("order_id").collect()] expected_rows = [ ("ORD-1001", "APAC", 3), ("ORD-1003", "APAC", 7), ] assert actual_rows == expected_rows, actual_rows data_files = sorted(Path(output_path).glob("part-*.avro")) assert data_files, "No Avro data files were written" print("Output Avro rows:") result.orderBy("order_id").show(truncate=False) print(f"Avro data files: {len(data_files)}") spark.stop()