Spark 3.5

.github/workflows/maven.yml

@@ -11,10 +11,10 @@ jobs:
     continue-on-error: true
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Java 17
+      - name: Set up Java 11
         uses: actions/setup-java@v3
         with:
-          java-version: 17
+          java-version: 11
           distribution: adopt
       # Github ubuntu contain Maven, but other testing software use minimal size container
       - name: Check if Maven is Installed
@@ -74,10 +74,10 @@ jobs:
         ]
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Java 17
+      - name: Set up Java 11
         uses: actions/setup-java@v3
         with:
-          java-version: 17
+          java-version: 11
           distribution: adopt
       - name: Restore Cached Maven Packages
         uses: actions/cache@v3

build_antlr_parser.xml

@@ -25,7 +25,7 @@
     <!-- <property name="src.dir" value="${basedir}/src/main/java/sparksoniq"/> prepends full path as comment-->
     <property name="src.dir" value="./src/main/java/org/rumbledb"/>
     <property name="parser.dir" value="${src.dir}/parser/jsoniq"/>
-    <property name="antlr.jar" value="./lib/antlr-4.13.1-complete.jar"/>
+    <property name="antlr.jar" value="./lib/antlr-4.9.3-complete.jar"/>
 
 
     <target name="clean-parser">

build_antlr_parser_xquery.xml

@@ -25,7 +25,7 @@
     <!-- <property name="src.dir" value="${basedir}/src/main/java/sparksoniq"/> prepends full path as comment-->
     <property name="src.dir" value="./src/main/java/org/rumbledb"/>
     <property name="parser.dir" value="${src.dir}/parser/xquery"/>
-    <property name="antlr.jar" value="./lib/antlr-4.13.1-complete.jar"/>
+    <property name="antlr.jar" value="./lib/antlr-4.9.3-complete.jar"/>
 
     <!-- Clean the ANTLR generated files for the parser-->
     <target name="clean-parser">

pom.xml

@@ -41,12 +41,12 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.13.0</version>
+                <version>3.8.1</version>
                 <!-- Holy grail of eclipse configuration:  https://stackoverflow.com/a/49599478/10707488   -->
                 <configuration>
                     <compilerId>eclipse</compilerId>
-                    <source>17</source>
-                    <target>17</target>
+                    <source>11</source>
+                    <target>11</target>
                     <compilerArgs>
                         <arg>-properties</arg>
                         <arg>${project.basedir}/org.eclipse.jdt.core.prefs</arg>
@@ -56,12 +56,12 @@
                     <dependency>
                         <groupId>org.codehaus.plexus</groupId>
                         <artifactId>plexus-compiler-eclipse</artifactId>
-                        <version>2.13.0</version>
+                        <version>2.8.5</version>
                     </dependency>
                     <dependency>
                         <groupId>org.eclipse.jdt</groupId>
                         <artifactId>ecj</artifactId>
-                        <version>3.36.0</version>
+                        <version>3.20.0</version>
                     </dependency>
                 </dependencies>
             </plugin>
@@ -76,7 +76,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
-                <version>3.4.2</version>
+                <version>3.1.2</version>
                 <executions>
                     <execution>
                         <goals>
@@ -117,7 +117,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>3.7.0</version>
+                <version>3.1.1</version>
                 <configuration>
                     <excludePackageNames>sparksoniq.*:sparksoniq</excludePackageNames>
                 </configuration>
@@ -235,7 +235,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
-            <version>3.3.0</version>
+                <version>3.3.1</version>
                 <configuration>
                     <enableOutErrElements>${surefire.enableOutErrElements}</enableOutErrElements>
                     <!-- arguments needed for unit tests to work with Java 17
@@ -247,7 +247,6 @@
                         --add-opens=java.base/java.util=ALL-UNNAMED
                         --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
                         --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
-                        --add-opens java.base/java.net=ALL-UNNAMED
                     </argLine>
                 </configuration>
             </plugin>
@@ -277,25 +276,25 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_2.13</artifactId>
-            <version>4.0.1</version>
+            <version>3.5.7</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_2.13</artifactId>
-            <version>4.0.1</version>
+            <version>3.5.7</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-mllib_2.13</artifactId>
-            <version>4.0.1</version>
+            <version>3.5.7</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-hive_2.13</artifactId>
-            <version>4.0.1</version>
+            <version>3.5.4</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
@@ -307,12 +306,12 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-avro_2.13</artifactId>
-            <version>4.0.1</version>
+            <version>3.5.7</version>
         </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
-            <version>4.13.1</version>
+            <version>4.9.3</version>
         </dependency>
         <dependency>
             <groupId>org.jline</groupId>
@@ -322,7 +321,7 @@
         <dependency>
             <groupId>com.esotericsoftware</groupId>
             <artifactId>kryo</artifactId>
-            <version>5.6.0</version>
+            <version>4.0.2</version> <!-- dont upgrade for now! -->
         </dependency>
         <dependency>
             <groupId>junit</groupId>
@@ -370,17 +369,17 @@
             <artifactId>jackson-dataformat-yaml</artifactId>
             <version>2.17.1</version>
         </dependency>
-        <dependency>
-            <groupId>io.delta</groupId>
-            <artifactId>delta-spark_2.13</artifactId>
-            <version>4.0.0</version>
-        </dependency>
         <dependency>
             <groupId>org.skyscreamer</groupId>
             <artifactId>jsonassert</artifactId>
             <version>1.5.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+          <groupId>io.delta</groupId>
+          <artifactId>delta-spark_2.13</artifactId>
+          <version>3.2.1</version>
+        </dependency>
         <!-- https://mvnrepository.com/artifact/net.sf.py4j/py4j -->
         <dependency>
             <groupId>net.sf.py4j</groupId>

src/main/java/org/rumbledb/api/Rumble.java

@@ -6,7 +6,6 @@
 import org.rumbledb.context.DynamicContext;
 import org.rumbledb.expressions.module.MainModule;
 import org.rumbledb.runtime.RuntimeIterator;
-
 import sparksoniq.spark.SparkSessionManager;
 
 import java.io.IOException;

src/main/java/org/rumbledb/api/SequenceWriter.java

@@ -177,7 +177,7 @@ public SequenceWriter format(String source) {
                     this.sequence,
                     null,
                     source,
-                    (this.dataFrameWriter == null) ? this.mode : this.dataFrameWriter.curmode(),
+                    (this.dataFrameWriter == null) ? this.mode : SaveMode.ErrorIfExists,
                     (this.dataFrameWriter == null) ? this.outputFormatOptions : new HashMap<>(),
                     this.configuration
             );

src/main/java/org/rumbledb/cli/Main.java

@@ -37,15 +37,12 @@ public class Main {
 
     public static void main(String[] args) throws IOException {
         String javaVersion = System.getProperty("java.version");
-        if (
-            !javaVersion.startsWith("17")
-                && !javaVersion.startsWith("21")
-        ) {
-            System.err.println("[Error] RumbleDB requires Java 17 or 21 (17 being the default Spark 4 version).");
+        if (!javaVersion.startsWith("11.") && !javaVersion.startsWith("17.")) {
+            System.err.println("[Error] RumbleDB requires Java 11 or 17.");
             System.err.println("Your Java version: " + System.getProperty("java.version"));
-            System.err.println("You can download Java 17 or 21 from https://adoptium.net/");
+            System.err.println("You can download Java 11 or 17 from https://adoptium.net/");
             System.err.println(
-                "If you do have Java 17 or 21, but the wrong version appears above, then it means you need to set your JAVA_HOME environment variable properly to point to Java 17 or 21."
+                "If you do have Java 11 or 17, but the wrong version appears above, then it means you need to set your JAVA_HOME environment variable properly to point to Java 11 or 17."
             );
             System.exit(43);
         }

src/main/java/org/rumbledb/items/Base64BinaryItem.java

@@ -12,7 +12,7 @@
 import org.rumbledb.runtime.misc.ComparisonIterator;
 import org.rumbledb.types.ItemType;
 
-import java.util.Base64;
+import javax.xml.bind.DatatypeConverter;
 import java.util.Arrays;
 import java.util.regex.Pattern;
 
@@ -41,7 +41,6 @@ public Base64BinaryItem() {
     }
 
     public Base64BinaryItem(String stringValue) {
-        stringValue = stringValue.replaceAll("\\s", "");
         this.value = parseBase64BinaryString(stringValue);
         this.stringValue = StringUtils.chomp(Base64.getEncoder().encodeToString(this.value));
     }
@@ -92,7 +91,7 @@ static byte[] parseBase64BinaryString(String base64BinaryString) throws IllegalA
         if (base64BinaryString == null || !checkInvalidBase64BinaryFormat(base64BinaryString.replaceAll("\\s", ""))) {
             throw new IllegalArgumentException();
         }
-        return Base64.getDecoder().decode(base64BinaryString);
+        return DatatypeConverter.parseBase64Binary(base64BinaryString);
     }
 
     @Override

src/main/java/org/rumbledb/items/structured/JSoundDataFrame.java

@@ -12,7 +12,6 @@
 import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.StructField;
 import org.apache.spark.sql.types.StructType;
-import org.apache.spark.sql.types.VariantType;
 import org.rumbledb.api.Item;
 import org.rumbledb.exceptions.ExceptionMetadata;
 import org.rumbledb.exceptions.OurBadException;
@@ -41,16 +40,6 @@ public JSoundDataFrame(Dataset<Row> dataFrame, ItemType itemType) {
             int i = schema.fieldIndex(SparkSessionManager.atomicJSONiqItemColumnName);
             StructField field = schema.fields()[i];
             DataType type = field.dataType();
-            if (type instanceof VariantType) {
-                if (!this.itemType.equals(BuiltinTypesCatalogue.item)) {
-                    this.dataFrame.printSchema();
-                    throw new OurBadException(
-                            "Inconsistency in internal representation: "
-                                + this.itemType
-                                + " is not the topmost item type."
-                    );
-                }
-            }
             if (type instanceof ArrayType) {
                 if (
                     this.itemType.equals(BuiltinTypesCatalogue.item)
@@ -73,17 +62,6 @@ public JSoundDataFrame(Dataset<Row> dataFrame, ItemType itemType) {
                         "Inconsistency in internal representation: " + this.itemType + " is an object type."
                 );
             }
-            if (type instanceof VariantType) {
-                if (!this.itemType.isSubtypeOf(BuiltinTypesCatalogue.item)) {
-                    this.dataFrame.printSchema();
-                    throw new OurBadException(
-                            "Inconsistency in internal representation: "
-                                + this.itemType
-                                + " is not the topmost item type."
-                    );
-                }
-                return;
-            }
             if (
                 this.itemType.equals(BuiltinTypesCatalogue.item)
                     || this.itemType.equals(BuiltinTypesCatalogue.atomicItem)
@@ -230,27 +208,4 @@ public List<FlworDataFrameColumn> getColumns() {
         }
         return result;
     }
-
-    public static boolean containsVariantType(Dataset<Row> dataFrame) {
-        return containsVariantTypeInSchema(dataFrame.schema());
-    }
-
-    private static boolean containsVariantTypeInSchema(DataType dataType) {
-        if (dataType instanceof VariantType) {
-            return true;
-        }
-        if (dataType instanceof ArrayType) {
-            ArrayType arrayType = (ArrayType) dataType;
-            return containsVariantTypeInSchema(arrayType.elementType());
-        }
-        if (dataType instanceof StructType) {
-            StructType structType = (StructType) dataType;
-            for (StructField field : structType.fields()) {
-                if (containsVariantTypeInSchema(field.dataType())) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
 }

src/main/java/org/rumbledb/runtime/flwor/FlworDataFrameUtils.java

@@ -90,8 +90,6 @@
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 
-import scala.collection.immutable.ArraySeq;
-import scala.collection.Iterator;
 import scala.collection.immutable.ArraySeq;
 import scala.collection.Iterator;
 import sparksoniq.spark.SparkSessionManager;
@@ -832,7 +830,7 @@ public static void deserializeWrappedParameters(
                 continue;
             }
             @SuppressWarnings("unchecked")
-            List<Item> deserializedParam = (List<Item>) deserializeByteArray(bytes, kryo, input);
+            List<Item> deserializedParam = (List<Item>) deserializeByteArray((byte[]) bytes, kryo, input);
             deserializedParams.add(deserializedParam);
         }
     }

src/main/java/org/rumbledb/runtime/flwor/clauses/LetClauseSparkIterator.java

@@ -276,7 +276,6 @@ public FlworDataFrame getDataFrameAsJoin(
                 );
             }
         }
-
         if (rightDependencies.size() == 1 && rightDependencies.contains(Name.CONTEXT_ITEM)) {
             if (!leftDependencies.contains(Name.CONTEXT_ITEM)) {
                 contextItemValueExpression = rightHandSideOfJoinEqualityCriterion;
@@ -294,7 +293,6 @@ public FlworDataFrame getDataFrameAsJoin(
                     getMetadata()
             );
         }
-
         // Now we know we can execute the query as an equi-join.
         // First, we evaluate all input tuples.
         Dataset<Row> inputDF = this.child.getDataFrame(context).getDataFrame();
@@ -328,6 +326,8 @@ public FlworDataFrame getDataFrameAsJoin(
             sequenceDependencies
         ).getDataFrame();
 
+        LogManager.getLogger("LetClauseSparkIterator").info("Rumble detected an equi-join in the left clause.");
+
         // We compute the hashes for both sides of the equality predicate.
         expressionDF = LetClauseSparkIterator.bindLetVariableInDataFrame(
             expressionDF,

src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseArrayMergeAggregateResultsUDF.java

@@ -29,7 +29,6 @@
 import scala.collection.Iterator;
 
 import java.util.ArrayList;
-// import java.util.Iterator;
 import java.util.List;
 
 public class GroupClauseArrayMergeAggregateResultsUDF implements UDF1<ArraySeq<Object>, Object[]> {

src/main/java/org/rumbledb/runtime/navigation/ArrayLookupIterator.java

@@ -231,7 +231,7 @@ public NativeClauseContext generateNativeQuery(NativeClauseContext nativeClauseC
                 )
             );
             newContext.setSchema(((ArrayType) newContext.getSchema()).elementType());
-            newContext.setResultingQuery("get(" + newContext.getResultingQuery() + " ," + (this.lookup - 1) + ")");
+            newContext.setResultingQuery(newContext.getResultingQuery() + "[" + (this.lookup - 1) + "]");
         }
         return newContext;
     }

src/main/java/org/rumbledb/types/ItemTypeFactory.java

@@ -659,8 +659,6 @@ public static ItemType createItemType(DataType dt) {
             return BuiltinTypesCatalogue.dateItem;
         } else if (dt.equals(DataTypes.BinaryType)) {
             return BuiltinTypesCatalogue.hexBinaryItem;
-        } else if (dt.equals(DataTypes.VariantType)) {
-            return BuiltinTypesCatalogue.item;
         } else if (dt instanceof VectorUDT) {
             return createArrayTypeWithSparkDataTypeContent(
                 DataTypes.DoubleType