github-312 Fix build with Spark 2.4 and Maven 3.8

CHANGELOG.md

@@ -26,6 +26,7 @@
 * github-302: Upgrade DeltaLake to 2.2.0
 * github-303: Use multi-stage build for Docker image
 * github-304: Upgrade Cloudera profile to CDP 7.1.8
+* github-312: Fix build with Spark 2.4 and Maven 3.8
 
 This version is fully backwards compatible until and including version 0.27.0.
 

docs/releases.md

@@ -16,7 +16,8 @@ changes over time.
 
 ### Version 0.30.0 - 2023-01-03
 
-* github-278: Parallelize execution of data quality checks
+* github-278: Parallelize execution of data quality checks. This also introduces a new configuration property
+  `flowman.execution.check.parallelism` (default `1`)
 * github-282: Improve implementation for counting records
 * github-288: Support reading local CSV files from fatjar
 * github-290: Simplify specifying project name in fatjar
@@ -33,13 +34,15 @@ changes over time.
 * github-296: Update npm dependencies (vuetify & co)
 * github-297: Parametrize when to execute a specific phase
 * github-299: Move migrationPolicy and migrationStrategy from target into relation
-* github-115: Implement additional build policy in relation target for forcing dirty
+* github-115: Implement additional build policy in relation target for forcing dirty. This also introduces a new
+  configuration property `flowman.default.target.buildPolicy` (default `COMPAT`).
 * github-298: Support fine-grained control when to execute each target of a job
 * github-300: Implement new 'observe' mapping
 * github-301: Upgrade Spark to 3.2.3
 * github-302: Upgrade DeltaLake to 2.2.0
 * github-303: Use multi-stage build for Docker image
 * github-304: Upgrade Cloudera profile to CDP 7.1.8
+* github-312: Fix build with Spark 2.4 and Maven 3.8
 
 
 ### Version 0.29.0 - 2022-11-08

flowman-core/src/main/java/com/dimajix/flowman/templating/VelocityContext.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 Kaya Kupferschmidt
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dimajix.flowman.templating;
+
+
+import java.util.Map;
+
+public class VelocityContext {
+    final org.apache.velocity.VelocityContext context;
+
+    public VelocityContext() {
+        context = new org.apache.velocity.VelocityContext();
+    }
+    public VelocityContext(VelocityContext parent) {
+        context = new org.apache.velocity.VelocityContext(parent.context);
+    }
+    public VelocityContext(Map<String,Object> values, VelocityContext parent) {
+        context = new org.apache.velocity.VelocityContext(values, parent.context);
+    }
+
+    public void put(String name, Object value) {
+        context.put(name, value);
+    }
+}

flowman-core/src/main/java/com/dimajix/flowman/templating/VelocityEngine.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2023 Kaya Kupferschmidt
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dimajix.flowman.templating;
+
+import java.io.StringWriter;
+
+import org.apache.velocity.runtime.RuntimeConstants;
+
+public class VelocityEngine {
+    private final org.apache.velocity.app.VelocityEngine engine;
+
+    public VelocityEngine() {
+        engine = new org.apache.velocity.app.VelocityEngine();
+        engine.setProperty(RuntimeConstants.VM_ARGUMENTS_STRICT, "true");
+        engine.setProperty(RuntimeConstants.RUNTIME_REFERENCES_STRICT, "true");
+        engine.setProperty(RuntimeConstants.RUNTIME_REFERENCES_STRICT_ESCAPE, "true");
+        engine.init();
+    }
+
+    public String evaluate(VelocityContext context, String logTag, String template) {
+        StringWriter output = new StringWriter();
+        engine.evaluate(context.context, output, logTag, template);
+        return output.toString();
+    }
+}

flowman-core/src/main/scala/com/dimajix/flowman/execution/Environment.scala

@@ -21,10 +21,9 @@ import java.io.StringWriter
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
-import org.apache.velocity.VelocityContext
-
 import com.dimajix.flowman.templating.RecursiveValue
 import com.dimajix.flowman.templating.Velocity
+import com.dimajix.flowman.templating.VelocityContext
 
 
 object Environment {
@@ -47,13 +46,11 @@ final class Environment(rawEnvironment:Map[String,Any]) {
     }
 
     private def evaluateNotNull(string:String, additionalValues:Map[String,AnyRef]) : String = {
-        val output = new StringWriter()
         val context = if (additionalValues.nonEmpty)
             new VelocityContext(mutable.Map(additionalValues.toSeq:_*).asJava, templateContext)
         else
             templateContext
-        templateEngine.evaluate(context, output, "context", string)
-        output.getBuffer.toString
+        templateEngine.evaluate(context, "context", string)
     }
 
     /**

flowman-core/src/main/scala/com/dimajix/flowman/fs/FileCollector.scala

@@ -17,16 +17,11 @@
 package com.dimajix.flowman.fs
 
 import java.io.FileNotFoundException
-import java.io.StringWriter
 
-import scala.annotation.tailrec
 import scala.collection.parallel.ParIterable
 import scala.util.control.NonFatal
 
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileStatus
 import org.apache.hadoop.fs.Path
-import org.apache.hadoop.fs.{FileSystem => HadoopFileSystem}
 import org.apache.spark.sql.SparkSession
 import org.apache.velocity.VelocityContext
 import org.slf4j.LoggerFactory
@@ -215,11 +210,9 @@ case class FileCollector(
         pattern.map { filePattern =>
             val partitionValues = defaults ++ partition.toMap
             try {
-                val context = new VelocityContext(templateContext)
+                val context = Velocity.newContext(templateContext)
                 partitionValues.foreach(kv => context.put(kv._1, kv._2))
-                val output = new StringWriter()
-                templateEngine.evaluate(context, output, "FileCollector", filePattern)
-                output.getBuffer.toString
+                templateEngine.evaluate(context, "FileCollector", filePattern)
             }
             catch {
                 case NonFatal(ex) =>

flowman-core/src/main/scala/com/dimajix/flowman/templating/Velocity.scala

@@ -19,13 +19,8 @@ package com.dimajix.flowman.templating
 import scala.collection.mutable
 import scala.util.control.NonFatal
 
-import org.apache.velocity.VelocityContext
-import org.apache.velocity.app.VelocityEngine
-import org.apache.velocity.runtime.RuntimeConstants
 import org.slf4j.LoggerFactory
 
-import com.dimajix.flowman.annotation.TemplateObject
-import com.dimajix.flowman.spi.ClassAnnotationHandler
 import com.dimajix.flowman.spi.ClassAnnotationScanner
 
 
@@ -99,14 +94,7 @@ object Velocity {
       */
     def newEngine() : VelocityEngine = singletonEngine
 
-    private lazy val singletonEngine = {
-        val ve = new VelocityEngine()
-        ve.setProperty(RuntimeConstants.VM_ARGUMENTS_STRICT, "true")
-        ve.setProperty(RuntimeConstants.RUNTIME_REFERENCES_STRICT, "true")
-        ve.setProperty(RuntimeConstants.RUNTIME_REFERENCES_STRICT_ESCAPE, "true")
-        ve.init()
-        ve
-    }
+    private lazy val singletonEngine = new VelocityEngine()
 }
 
 

flowman-core/src/main/scala/com/dimajix/flowman/templating/wrapper.scala

@@ -18,7 +18,6 @@ package com.dimajix.flowman.templating
 
 import java.io.FileInputStream
 import java.io.IOException
-import java.io.StringWriter
 import java.net.URLDecoder
 import java.net.URLEncoder
 import java.nio.charset.Charset
@@ -31,8 +30,6 @@ import java.time.format.DateTimeFormatter
 import java.time.temporal.Temporal
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.fs.Path
-import org.apache.velocity.VelocityContext
-import org.apache.velocity.app.VelocityEngine
 import org.slf4j.LoggerFactory
 import com.dimajix.flowman.fs.{File, FileUtils}
 import com.dimajix.flowman.templating.FileWrapper.logger
@@ -81,9 +78,7 @@ case class FileWrapper(file:File) {
 
 case class RecursiveValue(engine:VelocityEngine, context:VelocityContext, value:String) {
     override def toString: String = {
-        val output = new StringWriter()
-        engine.evaluate(context, output, "RecursiveValue", value)
-        output.toString
+        engine.evaluate(context, "RecursiveValue", value)
     }
 }
 

flowman-core/src/test/scala/com/dimajix/flowman/templating/TemplatingTest.scala

@@ -45,9 +45,7 @@ class TemplatingTest extends AnyFlatSpec with Matchers with LocalTempDir {
     private val context = Velocity.newContext()
 
     private def evaluate(text:String) : String = {
-        val output = new StringWriter()
-        engine.evaluate(context, output, "test", text)
-        output.toString
+        engine.evaluate(context, "test", text)
     }
 
     "Integers" should "be parseable" in {
@@ -93,9 +91,8 @@ class TemplatingTest extends AnyFlatSpec with Matchers with LocalTempDir {
     they should "support arithmetics" in {
         context.put("a", 2.0)
         context.put("b", 3.0)
-        val output = new StringWriter()
-        engine.evaluate(context, output, "test", "#set($r=$a+$b)$r")
-        output.toString should be ("5.0")
+        val output = engine.evaluate(context, "test", "#set($r=$a+$b)$r")
+        output should be ("5.0")
     }
 
     "Booleans" should "be parseable" in {
@@ -130,9 +127,8 @@ class TemplatingTest extends AnyFlatSpec with Matchers with LocalTempDir {
     }
 
     they should "be convertible to LocalDateTime" in {
-        val output = new StringWriter()
-        engine.evaluate(context, output, "test", "$Timestamp.parse('2017-10-10T10:11:23').toLocalDateTime()")
-        output.toString should be ("2017-10-10T10:11:23")
+        val output = engine.evaluate(context, "test", "$Timestamp.parse('2017-10-10T10:11:23').toLocalDateTime()")
+        output should be ("2017-10-10T10:11:23")
     }
 
     they should "be convertible to epochs" in {
@@ -144,9 +140,8 @@ class TemplatingTest extends AnyFlatSpec with Matchers with LocalTempDir {
 
     they should "support complex operations" in {
         context.put("ts", UtcTimestamp.of(2017, Month.JUNE, 19, 0, 0))
-        val output = new StringWriter()
-        engine.evaluate(context, output, "test", """data/$ts.format("yyyy/MM/dd")/${ts.toEpochSeconds()}.i-*.log""")
-        output.toString should be ("data/2017/06/19/1497830400.i-*.log")
+        val output = engine.evaluate(context, "test", """data/$ts.format("yyyy/MM/dd")/${ts.toEpochSeconds()}.i-*.log""")
+        output should be ("data/2017/06/19/1497830400.i-*.log")
     }
 
     "LocalDate" should "be parseable" in {

flowman-core/src/test/scala/com/dimajix/flowman/templating/VelocityTest.scala

@@ -28,9 +28,7 @@ class VelocityTest extends AnyFlatSpec with Matchers {
     private val context = Velocity.newContext()
 
     private def evaluate(text:String) : String = {
-        val output = new StringWriter()
-        engine.evaluate(context, output, "test", text)
-        output.toString
+        engine.evaluate(context, "test", text)
     }
 
     "The Velocity Engine" should "throw an exception on unknown references" in {

pom.xml

@@ -950,7 +950,7 @@
                     <inherited>true</inherited>
                     <groupId>net.alchim31.maven</groupId>
                     <artifactId>scala-maven-plugin</artifactId>
-                    <version>4.7.1</version>
+                    <version>4.8.0</version>
                     <configuration>
                         <scalaVersion>${scala.version}</scalaVersion>
                         <scalaCompatVersion>${scala.api_version}</scalaCompatVersion>
@@ -1041,7 +1041,7 @@
                     <inherited>true</inherited>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-shade-plugin</artifactId>
-                    <version>3.4.0</version>
+                    <version>3.4.1</version>
                     <configuration>
                         <shadedArtifactAttached>false</shadedArtifactAttached>
                         <keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope>