fix pre commit

Author: Imbruced

python/tests/utils/test_sedona_db_vectorized_udf.py

@@ -16,6 +16,7 @@
 # under the License.
 
 import numpy as np
+import pyspark
 import pytest
 
 from sedona.spark.sql.functions import sedona_db_vectorized_udf
@@ -32,6 +33,10 @@
 class TestSedonaDBArrowFunction(TestBase):
 
     @pytest.mark.vectorized
+    @pytest.mark.skipif(
+        pyspark.__version__ >= "4.0.0",
+        reason="Skip for pyspark > 4.0",
+    )
     def test_vectorized_udf(self):
         @sedona_db_vectorized_udf(
             return_type=GeometryType(), input_types=[ByteType(), IntegerType()]
@@ -57,6 +62,10 @@ def my_own_function(geom, distance):
         df.select(ST_X(my_own_function(df.wkt, lit(100)).alias("geom"))).show()
 
     @pytest.mark.vectorized
+    @pytest.mark.skipif(
+        pyspark.__version__ >= "4.0.0",
+        reason="Skip for pyspark > 4.0",
+    )
     def test_geometry_to_double(self):
         @sedona_db_vectorized_udf(return_type=DoubleType(), input_types=[ByteType()])
         def geometry_to_non_geometry_udf(geom):
@@ -82,6 +91,10 @@ def geometry_to_non_geometry_udf(geom):
         assert values_list == [1.0, 2.0, 3.0]
 
     @pytest.mark.vectorized
+    @pytest.mark.skipif(
+        pyspark.__version__ >= "4.0.0",
+        reason="Skip for pyspark > 4.0",
+    )
     def test_geometry_to_int(self):
         @sedona_db_vectorized_udf(return_type=IntegerType(), input_types=[ByteType()])
         def geometry_to_int(geom):
@@ -106,6 +119,10 @@ def geometry_to_int(geom):
         assert values_list == [0, 0, 0]
 
     @pytest.mark.vectorized
+    @pytest.mark.skipif(
+        pyspark.__version__ >= "4.0.0",
+        reason="Skip for pyspark > 4.0",
+    )
     def test_geometry_crs_preservation(self):
         @sedona_db_vectorized_udf(return_type=GeometryType(), input_types=[ByteType()])
         def return_same_geometry(geom):

spark/spark-3.5/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala

@@ -26,7 +26,6 @@ import org.scalatest.{BeforeAndAfterAll, FunSpec}
 import org.testcontainers.containers.MinIOContainer
 
 import java.io.FileInputStream
-
 import java.util.concurrent.ThreadLocalRandom
 
 trait TestBaseScala extends FunSpec with BeforeAndAfterAll {

spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/StrategySuite.scala

@@ -36,6 +36,10 @@ class StrategySuite extends TestBaseScala with Matchers {
   import spark.implicits._
 
   it("sedona geospatial UDF - geopandas") {
+    if (!sparkSession.version.startsWith("3.5")) {
+      cancel("spark 3.5 is supported only.")
+    }
+
     val df = Seq(
       (1, "value", wktReader.read("POINT(21 52)")),
       (2, "value1", wktReader.read("POINT(20 50)")),
@@ -61,6 +65,10 @@ class StrategySuite extends TestBaseScala with Matchers {
   }
 
   it("sedona geospatial UDF - sedona db") {
+    if (!sparkSession.version.startsWith("3.5")) {
+      cancel("spark 3.5 is supported only.")
+    }
+
     val df = Seq(
       (1, "value", wktReader.read("POINT(21 52)")),
       (2, "value1", wktReader.read("POINT(20 50)")),

spark/spark-3.5/src/test/scala/org/apache/spark/sql/udf/TestScalarPandasUDF.scala

@@ -209,20 +209,6 @@ object ScalarUDF {
     pythonEvalType = UDF.PythonEvalType.SQL_SCALAR_SEDONA_UDF,
     udfDeterministic = true)
 
-  val nonGeometryVectorizedUDF: UserDefinedPythonFunction = UserDefinedPythonFunction(
-    name = "vectorized_udf",
-    func = SimplePythonFunction(
-      command = vectorizedFunction,
-      envVars = workerEnv.clone().asInstanceOf[java.util.Map[String, String]],
-      pythonIncludes = List.empty[String].asJava,
-      pythonExec = pythonExec,
-      pythonVer = pythonVer,
-      broadcastVars = List.empty[Broadcast[PythonBroadcast]].asJava,
-      accumulator = null),
-    dataType = FloatType,
-    pythonEvalType = PythonEvalType.SQL_SCALAR_PANDAS_UDF,
-    udfDeterministic = false)
-
   val sedonaDBGeometryToGeometryFunction: UserDefinedPythonFunction = UserDefinedPythonFunction(
     name = "geospatial_udf",
     func = SimplePythonFunction(