diff --git a/.sbtopts b/.sbtopts
index d0d25ed4f..ca8c83416 100644
--- a/.sbtopts
+++ b/.sbtopts
@@ -1,3 +1 @@
--Djava.awt.headless=true
--J-Xmx2g
-J-XX:MaxMetaspaceSize=1g
diff --git a/bench/src/main/scala/astraea/spark/rasterframes/bench/HistogramEncodeBench.scala b/bench/src/main/scala/astraea/spark/rasterframes/bench/HistogramEncodeBench.scala
deleted file mode 100644
index bfe378570..000000000
--- a/bench/src/main/scala/astraea/spark/rasterframes/bench/HistogramEncodeBench.scala
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2017 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package astraea.spark.rasterframes.bench
-
-import java.util.concurrent.TimeUnit
-
-import geotrellis.raster.histogram.{Histogram, StreamingHistogram}
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.gt.types.HistogramUDT
-import org.openjdk.jmh.annotations._
-
-@BenchmarkMode(Array(Mode.AverageTime))
-@State(Scope.Benchmark)
-@OutputTimeUnit(TimeUnit.MICROSECONDS)
-/**
- * @author sfitch
- * @since 9/29/17
- */
-class HistogramEncodeBench extends SparkEnv {
-
- val encoder: ExpressionEncoder[Histogram[Double]] = ExpressionEncoder()
- val boundEncoder = encoder.resolveAndBind()
-
- @Param(Array("float64"))
- var cellTypeName: String = _
-
- @Param(Array("128"))
- var tileSize: Int = _
-
- @Param(Array("400"))
- var numTiles: Int = _
-
- @transient
- var histogram: Histogram[Double] = _
-
- @Setup(Level.Trial)
- def setupData(): Unit = {
- // Creates what should hopefully be a representative structure
- val tiles = Seq.fill(numTiles)(randomTile(tileSize, tileSize, cellTypeName))
- histogram = tiles.foldLeft(StreamingHistogram(): Histogram[Double])(
- (hist, tile) ⇒ hist.merge(StreamingHistogram.fromTile(tile))
- )
- }
- @Benchmark
- def serialize(): Any = {
- HistogramUDT.serialize(histogram)
- }
-
- @Benchmark
- def encode(): InternalRow = {
- boundEncoder.toRow(histogram)
- }
-
-// @Benchmark
-// def roundTrip(): Tile = {
-// }
-}
-
diff --git a/build.sbt b/build.sbt
index b7d645e3f..15571e4fd 100644
--- a/build.sbt
+++ b/build.sbt
@@ -3,20 +3,33 @@ addCommandAlias("console", "datasource/console")
lazy val root = project
.in(file("."))
- .withId("RF")
- .aggregate(core, datasource)
- .settings(publishArtifact := false)
+ .withId("RasterFrames")
+ .aggregate(core, datasource, pyrasterframes, experimental)
+ .settings(publish / skip := true)
.settings(releaseSettings)
lazy val core = project
+ .disablePlugins(SparkPackagePlugin)
+
+lazy val pyrasterframes = project
+ .dependsOn(core, datasource)
+ .settings(assemblySettings)
lazy val datasource = project
.dependsOn(core % "test->test;compile->compile")
+ .disablePlugins(SparkPackagePlugin)
+
+lazy val experimental = project
+ .dependsOn(core % "test->test;compile->compile")
+ .dependsOn(datasource % "test->test;compile->compile")
+ .disablePlugins(SparkPackagePlugin)
lazy val docs = project
.dependsOn(core, datasource)
+ .disablePlugins(SparkPackagePlugin)
lazy val bench = project
.dependsOn(core)
-
+ .disablePlugins(SparkPackagePlugin)
+ .settings(publish / skip := true)
diff --git a/core/build.sbt b/core/build.sbt
index 94bc554e9..04d6a18df 100644
--- a/core/build.sbt
+++ b/core/build.sbt
@@ -4,8 +4,8 @@ moduleName := "rasterframes"
libraryDependencies ++= Seq(
"com.chuusai" %% "shapeless" % "2.3.2",
- "org.locationtech.geomesa" %% "geomesa-z3" % "1.3.5",
- "org.locationtech.geomesa" %% "geomesa-spark-jts" % "2.0.0-astraea.1" exclude("jgridshift", "jgridshift"),
+ "org.locationtech.geomesa" %% "geomesa-z3" % rfGeoMesaVersion.value,
+ "org.locationtech.geomesa" %% "geomesa-spark-jts" % rfGeoMesaVersion.value exclude("jgridshift", "jgridshift"),
spark("core").value % Provided,
spark("mllib").value % Provided,
spark("sql").value % Provided,
@@ -19,7 +19,7 @@ libraryDependencies ++= Seq(
)
buildInfoKeys ++= Seq[BuildInfoKey](
- name, version, scalaVersion, sbtVersion, rfGeotrellisVersion, rfSparkVersion
+ name, version, scalaVersion, sbtVersion, rfGeoTrellisVersion, rfGeoMesaVersion, rfSparkVersion
)
buildInfoPackage := "astraea.spark.rasterframes"
diff --git a/core/src/main/scala/astraea/spark/rasterframes/PairRDDConverter.scala b/core/src/main/scala/astraea/spark/rasterframes/PairRDDConverter.scala
new file mode 100644
index 000000000..68263d5c9
--- /dev/null
+++ b/core/src/main/scala/astraea/spark/rasterframes/PairRDDConverter.scala
@@ -0,0 +1,144 @@
+package astraea.spark.rasterframes
+
+import astraea.spark.rasterframes.util._
+import geotrellis.raster.{MultibandTile, Tile, TileFeature}
+import geotrellis.spark.{SpaceTimeKey, SpatialKey, TemporalKey}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql._
+import org.apache.spark.sql.gt.types.TileUDT
+import org.apache.spark.sql.types._
+
+import scala.annotation.implicitNotFound
+
+/**
+ * Typeclass for converting a Pair RDD into a dataframe.
+ *
+ * @since 4/8/18
+ */
+@implicitNotFound("An RDD converter is required create a RasterFrame. " +
+ "Please provide an implementation of PairRDDConverter[${K}, ${V}].")
+trait PairRDDConverter[K, V] extends Serializable {
+ val schema: StructType
+ def toDataFrame(rdd: RDD[(K, V)])(implicit spark: SparkSession): DataFrame
+}
+
+object PairRDDConverter {
+ /** Enrichment over a pair RDD for converting it to a DataFrame given a converter. */
+ implicit class RDDCanBeDataFrame[K, V](rdd: RDD[(K, V)])(implicit spark: SparkSession, converter: PairRDDConverter[K, V]) {
+ def toDataFrame: DataFrame = converter.toDataFrame(rdd)
+ }
+
+ // Hack around Spark bug when singletons are used in schemas
+ private val serializableTileUDT = new TileUDT()
+
+ /** Fetch converter from implicit scope. */
+ def apply[K, V](implicit sp: PairRDDConverter[K, V]) = sp
+
+ /** Enables conversion of `RDD[(SpatialKey, Tile)]` to DataFrame. */
+ implicit val spatialTileConverter = new PairRDDConverter[SpatialKey, Tile] {
+ val schema: StructType = {
+ StructType(Seq(
+ StructField(SPATIAL_KEY_COLUMN.columnName, spatialKeyEncoder.schema, nullable = false),
+ StructField(TILE_COLUMN.columnName, serializableTileUDT, nullable = false)
+ ))
+ }
+
+ def toDataFrame(rdd: RDD[(SpatialKey, Tile)])(implicit spark: SparkSession): DataFrame = {
+ import spark.implicits._
+ rdd.toDF(schema.fields.map(_.name): _*)
+ }
+ }
+
+ /** Enables conversion of `RDD[(SpaceTimeKey, Tile)]` to DataFrame. */
+ implicit val spaceTimeTileConverter = new PairRDDConverter[SpaceTimeKey, Tile] {
+ val schema: StructType = {
+ val base = spatialTileConverter.schema
+ val addedFields = Seq(StructField(TEMPORAL_KEY_COLUMN.columnName, temporalKeyEncoder.schema, nullable = false))
+ StructType(base.fields.patch(1, addedFields, 0))
+ }
+
+ def toDataFrame(rdd: RDD[(SpaceTimeKey, Tile)])(implicit spark: SparkSession): DataFrame = {
+ import spark.implicits._
+ rdd.map{ case (k, v) ⇒ (k.spatialKey, k.temporalKey, v)}.toDF(schema.fields.map(_.name): _*)
+ }
+ }
+
+ /** Enables conversion of `RDD[(SpatialKey, TileFeature[Tile, D])]` to DataFrame. */
+ implicit def spatialTileFeatureConverter[D: Encoder] = new PairRDDConverter[SpatialKey, TileFeature[Tile, D]] {
+ implicit val featureEncoder = implicitly[Encoder[D]]
+ implicit val rowEncoder = Encoders.tuple(spatialKeyEncoder, singlebandTileEncoder, featureEncoder)
+
+ val schema: StructType = {
+ val base = spatialTileConverter.schema
+ StructType(base.fields :+ StructField(TILE_FEATURE_DATA_COLUMN.columnName, featureEncoder.schema, nullable = true))
+ }
+
+ def toDataFrame(rdd: RDD[(SpatialKey, TileFeature[Tile, D])])(implicit spark: SparkSession): DataFrame = {
+ import spark.implicits._
+ rdd.map{ case (k, v) ⇒ (k, v.tile, v.data)}.toDF(schema.fields.map(_.name): _*)
+ }
+ }
+
+ /** Enables conversion of `RDD[(SpaceTimeKey, TileFeature[Tile, D])]` to DataFrame. */
+ implicit def spaceTimeTileFeatureConverter[D: Encoder] = new PairRDDConverter[SpaceTimeKey, TileFeature[Tile, D]] {
+ implicit val featureEncoder = implicitly[Encoder[D]]
+ implicit val rowEncoder = Encoders.tuple(spatialKeyEncoder, temporalKeyEncoder, singlebandTileEncoder, featureEncoder)
+
+ val schema: StructType = {
+ val base = spaceTimeTileConverter.schema
+ StructType(base.fields :+ StructField(TILE_FEATURE_DATA_COLUMN.columnName, featureEncoder.schema, nullable = true))
+ }
+
+ def toDataFrame(rdd: RDD[(SpaceTimeKey, TileFeature[Tile, D])])(implicit spark: SparkSession): DataFrame = {
+ import spark.implicits._
+ val tupRDD = rdd.map { case (k, v) ⇒ (k.spatialKey, k.temporalKey, v.tile, v.data) }
+
+ rddToDatasetHolder(tupRDD)
+ tupRDD.toDF(schema.fields.map(_.name): _*)
+ }
+ }
+
+ /** Enables conversion of `RDD[(SpatialKey, MultibandTile)]` to DataFrame. */
+ def forSpatialMultiband(bands: Int) = new PairRDDConverter[SpatialKey, MultibandTile] {
+ val schema: StructType = {
+ val base = spatialTileConverter.schema
+
+ val basename = TILE_COLUMN.columnName
+
+ val tiles = for(i ← 1 to bands) yield {
+ StructField(s"${basename}_$i" , serializableTileUDT, nullable = false)
+ }
+
+ StructType(base.fields.patch(1, tiles, 1))
+ }
+
+ def toDataFrame(rdd: RDD[(SpatialKey, MultibandTile)])(implicit spark: SparkSession): DataFrame = {
+ spark.createDataFrame(
+ rdd.map { case (k, v) ⇒ Row(Row(k.col, k.row) +: v.bands: _*) },
+ schema
+ )
+ }
+ }
+
+ /** Enables conversion of `RDD[(SpaceTimeKey, MultibandTile)]` to DataFrame. */
+ def forSpaceTimeMultiband(bands: Int) = new PairRDDConverter[SpaceTimeKey, MultibandTile] {
+ val schema: StructType = {
+ val base = spaceTimeTileConverter.schema
+
+ val basename = TILE_COLUMN.columnName
+
+ val tiles = for(i ← 1 to bands) yield {
+ StructField(s"${basename}_$i" , serializableTileUDT, nullable = false)
+ }
+
+ StructType(base.fields.patch(2, tiles, 1))
+ }
+
+ def toDataFrame(rdd: RDD[(SpaceTimeKey, MultibandTile)])(implicit spark: SparkSession): DataFrame = {
+ spark.createDataFrame(
+ rdd.map { case (k, v) ⇒ Row(Seq(Row(k.spatialKey.col, k.spatialKey.row), Row(k.temporalKey)) ++ v.bands: _*) },
+ schema
+ )
+ }
+ }
+}
diff --git a/core/src/main/scala/astraea/spark/rasterframes/RasterFunctions.scala b/core/src/main/scala/astraea/spark/rasterframes/RasterFunctions.scala
index e1359943f..4a0e2e054 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/RasterFunctions.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/RasterFunctions.scala
@@ -22,9 +22,9 @@ package astraea.spark.rasterframes
import astraea.spark.rasterframes.encoders.SparkDefaultEncoders
import astraea.spark.rasterframes.expressions.ExplodeTileExpression
import astraea.spark.rasterframes.functions.{CellCountAggregateFunction, CellMeanAggregateFunction}
+import astraea.spark.rasterframes.stats.{CellHistogram, CellStatistics}
import astraea.spark.rasterframes.{functions ⇒ F}
import com.vividsolutions.jts.geom.Envelope
-import geotrellis.raster.histogram.Histogram
import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp
import geotrellis.raster.{CellType, Tile}
import org.apache.spark.annotation.Experimental
@@ -84,21 +84,25 @@ trait RasterFunctions {
def cellType(col: Column): TypedColumn[Any, String] =
expressions.CellTypeExpression(col.expr).asColumn.as[String]
+ /** Change the Tile's cell type */
+ def convertCellType(col: Column, cellType: CellType): TypedColumn[Any, Tile] =
+ udf[Tile, Tile](F.convertCellType(cellType)).apply(col).as[Tile]
+
/** Assign a `NoData` value to the Tiles. */
def withNoData(col: Column, nodata: Double) = withAlias("withNoData", col)(
udf[Tile, Tile](F.withNoData(nodata)).apply(col)
).as[Tile]
/** Compute the full column aggregate floating point histogram. */
- def aggHistogram(col: Column): TypedColumn[Any, Histogram[Double]] =
+ def aggHistogram(col: Column): TypedColumn[Any, CellHistogram] =
withAlias("histogram", col)(
F.aggHistogram(col)
- ).as[Histogram[Double]]
+ ).as[CellHistogram]
/** Compute the full column aggregate floating point statistics. */
- def aggStats(col: Column): TypedColumn[Any, Statistics] = withAlias("aggStats", col)(
+ def aggStats(col: Column): TypedColumn[Any, CellStatistics] = withAlias("aggStats", col)(
F.aggStats(col)
- ).as[Statistics]
+ ).as[CellStatistics]
/** Computes the column aggregate mean. */
def aggMean(col: Column) = CellMeanAggregateFunction(col.expr)
@@ -140,16 +144,16 @@ trait RasterFunctions {
).as[Double]
/** Compute TileHistogram of Tile values. */
- def tileHistogram(col: Column): TypedColumn[Any, Histogram[Double]] =
+ def tileHistogram(col: Column): TypedColumn[Any, CellHistogram] =
withAlias("tileHistogram", col)(
- udf[Histogram[Double], Tile](F.tileHistogram).apply(col)
- ).as[Histogram[Double]]
+ udf[CellHistogram, Tile](F.tileHistogram).apply(col)
+ ).as[CellHistogram]
/** Compute statistics of Tile values. */
- def tileStats(col: Column): TypedColumn[Any, Statistics] =
+ def tileStats(col: Column): TypedColumn[Any, CellStatistics] =
withAlias("tileStats", col)(
- udf[Statistics, Tile](F.tileStats).apply(col)
- ).as[Statistics]
+ udf[CellStatistics, Tile](F.tileStats).apply(col)
+ ).as[CellStatistics]
/** Counts the number of non-NoData cells per Tile. */
def dataCells(tile: Column): TypedColumn[Any, Long] =
diff --git a/core/src/main/scala/astraea/spark/rasterframes/encoders/StandardEncoders.scala b/core/src/main/scala/astraea/spark/rasterframes/encoders/StandardEncoders.scala
index 02a3b1649..95edddd0f 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/encoders/StandardEncoders.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/encoders/StandardEncoders.scala
@@ -19,14 +19,13 @@
package astraea.spark.rasterframes.encoders
-import astraea.spark.rasterframes.Statistics
-import geotrellis.raster.histogram.Histogram
-import geotrellis.raster.{MultibandTile, Tile}
+import astraea.spark.rasterframes.stats.{CellHistogram, CellStatistics}
+import geotrellis.raster.Tile
import geotrellis.spark.tiling.LayoutDefinition
import geotrellis.spark.{KeyBounds, SpaceTimeKey, SpatialKey, TemporalKey, TileLayerMetadata}
import geotrellis.vector.Extent
-import org.apache.spark.sql.Encoder
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.{Encoder, Encoders}
import scala.reflect.runtime.universe._
@@ -34,21 +33,21 @@ import scala.reflect.runtime.universe._
* Implicit encoder definitions for RasterFrame types.
*/
trait StandardEncoders {
+ implicit val spatialKeyEncoder = ExpressionEncoder[SpatialKey]
+ implicit val temporalKeyEncoder = ExpressionEncoder[TemporalKey]
+ implicit val spaceTimeKeyEncoder = ExpressionEncoder[SpaceTimeKey]
+ implicit val statsEncoder = ExpressionEncoder[CellStatistics]
+ implicit val histEncoder = ExpressionEncoder[CellHistogram]
+ implicit val layoutDefinitionEncoder = ExpressionEncoder[LayoutDefinition]
+ implicit val stkBoundsEncoder = ExpressionEncoder[KeyBounds[SpaceTimeKey]]
+ implicit val extentEncoder = ExpressionEncoder[Extent]
+
implicit def singlebandTileEncoder = ExpressionEncoder[Tile]()
- implicit def multibandTileEncoder = ExpressionEncoder[MultibandTile]()
+ implicit def tileLayerMetadataEncoder[K: TypeTag]: Encoder[TileLayerMetadata[K]] = TileLayerMetadataEncoder[K]()
implicit val crsEncoder = CRSEncoder()
- implicit val extentEncoder = ExpressionEncoder[Extent]()
implicit val projectedExtentEncoder = ProjectedExtentEncoder()
implicit val temporalProjectedExtentEncoder = TemporalProjectedExtentEncoder()
- implicit def histogramDoubleEncoder = ExpressionEncoder[Histogram[Double]]()
- implicit val statsEncoder = ExpressionEncoder[Statistics]()
- implicit def tileLayerMetadataEncoder[K: TypeTag]: Encoder[TileLayerMetadata[K]] = TileLayerMetadataEncoder[K]()
- implicit val layoutDefinitionEncoder = ExpressionEncoder[LayoutDefinition]()
- implicit val stkBoundsEncoder = ExpressionEncoder[KeyBounds[SpaceTimeKey]]()
implicit val cellTypeEncoder = CellTypeEncoder()
- implicit val spatialKeyEncoder = ExpressionEncoder[SpatialKey]()
- implicit val temporalKeyEncoder = ExpressionEncoder[TemporalKey]()
- implicit val spaceTimeKeyEncoder = ExpressionEncoder[SpaceTimeKey]()
implicit val uriEncoder = URIEncoder()
implicit val envelopeEncoder = EnvelopeEncoder()
}
diff --git a/core/src/main/scala/astraea/spark/rasterframes/expressions/DimensionsExpression.scala b/core/src/main/scala/astraea/spark/rasterframes/expressions/DimensionsExpression.scala
index 100e337e1..720e60759 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/expressions/DimensionsExpression.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/expressions/DimensionsExpression.scala
@@ -32,6 +32,8 @@ import org.apache.spark.sql.types.{ShortType, StructField, StructType}
case class DimensionsExpression(child: Expression) extends UnaryExpression with RequiresTile {
override def toString: String = s"dimension($child)"
+ override def nodeName: String = "dimension"
+
def dataType = StructType(Seq(
StructField("cols", ShortType),
StructField("rows", ShortType)
diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/ContextRDDMethods.scala b/core/src/main/scala/astraea/spark/rasterframes/extensions/ContextRDDMethods.scala
index 18d244814..ef0c901f8 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/extensions/ContextRDDMethods.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/extensions/ContextRDDMethods.scala
@@ -16,38 +16,32 @@
package astraea.spark.rasterframes.extensions
-import astraea.spark.rasterframes.RasterFrame
-import astraea.spark.rasterframes.extensions.Implicits._
+import astraea.spark.rasterframes.PairRDDConverter._
import astraea.spark.rasterframes.StandardColumns._
-import geotrellis.raster.{Tile, TileFeature}
+import astraea.spark.rasterframes.extensions.Implicits._
+import astraea.spark.rasterframes.util._
+import astraea.spark.rasterframes.{PairRDDConverter, RasterFrame}
+import geotrellis.raster.{CellGrid, Tile}
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.util.MethodExtensions
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
-import spray.json.JsonFormat
-import astraea.spark.rasterframes.util._
-import scala.reflect.ClassTag
-import scala.reflect.runtime.universe._
-
/**
- * Extension method on `ContextRDD`-shaped [[Tile]] RDDs with appropriate context bounds to create a RasterFrame.
+ * Extension method on `ContextRDD`-shaped RDDs with appropriate context bounds to create a RasterFrame.
* @since 7/18/17
*/
-abstract class SpatialContextRDDMethods(implicit spark: SparkSession)
- extends MethodExtensions[RDD[(SpatialKey, Tile)] with Metadata[TileLayerMetadata[SpatialKey]]] {
-
- def toRF: RasterFrame = toRF(TILE_COLUMN.columnName)
+abstract class SpatialContextRDDMethods[T <: CellGrid](implicit spark: SparkSession)
+ extends MethodExtensions[RDD[(SpatialKey, T)] with Metadata[TileLayerMetadata[SpatialKey]]] {
+ import PairRDDConverter._
- def toRF(tileColumnName: String): RasterFrame = {
- import spark.implicits._
+ def toRF(implicit converter: PairRDDConverter[SpatialKey, T]): RasterFrame = toRF(TILE_COLUMN.columnName)
- val rdd = self: RDD[(SpatialKey, Tile)]
- val df = rdd
- .toDF(SPATIAL_KEY_COLUMN.columnName, tileColumnName)
-
- df.setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata)
+ def toRF(tileColumnName: String)(implicit converter: PairRDDConverter[SpatialKey, T]): RasterFrame = {
+ val df = self.toDataFrame.setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata)
+ val defName = TILE_COLUMN.columnName
+ df.mapWhen(_ ⇒ tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName))
.certify
}
}
@@ -56,63 +50,18 @@ abstract class SpatialContextRDDMethods(implicit spark: SparkSession)
* Extension method on `ContextRDD`-shaped [[Tile]] RDDs keyed with [[SpaceTimeKey]], with appropriate context bounds to create a RasterFrame.
* @since 9/11/17
*/
-abstract class SpatioTemporalContextRDDMethods(implicit spark: SparkSession)
- extends MethodExtensions[RDD[(SpaceTimeKey, Tile)] with Metadata[TileLayerMetadata[SpaceTimeKey]]] {
-
- def toRF: RasterFrame = {
- import spark.implicits._
-
- val rdd = self: RDD[(SpaceTimeKey, Tile)]
- val df = rdd
- .map { case (k, v) ⇒ (k.spatialKey, k.temporalKey, v)}
- .toDF(SPATIAL_KEY_COLUMN.columnName, TEMPORAL_KEY_COLUMN.columnName, TILE_COLUMN.columnName)
-
- df
- .setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata)
- .setTemporalColumnRole(TEMPORAL_KEY_COLUMN)
- .certify
- }
-}
-
-/**
- * Extension method on `ContextRDD`-shaped [[TileFeature]] RDDs with appropriate context bounds to create a RasterFrame.
- * @since 7/18/17
- */
-abstract class TFContextRDDMethods[K: SpatialComponent: JsonFormat: ClassTag: TypeTag, D: TypeTag](implicit spark: SparkSession)
- extends MethodExtensions[RDD[(K, TileFeature[Tile, D])] with Metadata[TileLayerMetadata[K]]] {
-
- def toRF: RasterFrame = {
- import spark.implicits._
- val rdd = self: RDD[(K, TileFeature[Tile, D])]
-
- val df = rdd
- .map { case (k, v) ⇒ (k, v.tile, v.data) }
- .toDF(SPATIAL_KEY_COLUMN.columnName, TILE_COLUMN.columnName, TILE_FEATURE_DATA_COLUMN.columnName)
-
- df
- .setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata)
- .certify
- }
-}
-
-/**
- * Extension method on `ContextRDD`-shaped [[TileFeature]] RDDs with appropriate context bounds to create a RasterFrame.
- * @since 7/18/17
- */
-abstract class TFSTContextRDDMethods[D: TypeTag](implicit spark: SparkSession)
- extends MethodExtensions[RDD[(SpaceTimeKey, TileFeature[Tile, D])] with Metadata[TileLayerMetadata[SpaceTimeKey]]] {
-
- def toRF: RasterFrame = {
- import spark.implicits._
- val rdd = self: RDD[(SpaceTimeKey, TileFeature[Tile, D])]
+abstract class SpatioTemporalContextRDDMethods[T <: CellGrid](
+ implicit spark: SparkSession)
+ extends MethodExtensions[RDD[(SpaceTimeKey, T)] with Metadata[TileLayerMetadata[SpaceTimeKey]]] {
- val df = rdd
- .map { case (k, v) ⇒ (k.spatialKey, k.temporalKey, v.tile, v.data)}
- .toDF(SPATIAL_KEY_COLUMN.columnName, TEMPORAL_KEY_COLUMN.columnName, TILE_COLUMN.columnName, TILE_FEATURE_DATA_COLUMN.columnName)
+ def toRF(implicit converter: PairRDDConverter[SpaceTimeKey, T]): RasterFrame = toRF(TILE_COLUMN.columnName)
- df
+ def toRF(tileColumnName: String)(implicit converter: PairRDDConverter[SpaceTimeKey, T]): RasterFrame = {
+ val df = self.toDataFrame
.setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata)
.setTemporalColumnRole(TEMPORAL_KEY_COLUMN)
+ val defName = TILE_COLUMN.columnName
+ df.mapWhen(_ ⇒ tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName))
.certify
}
}
diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/Implicits.scala b/core/src/main/scala/astraea/spark/rasterframes/extensions/Implicits.scala
index 36e819d9c..b2aa42faf 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/extensions/Implicits.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/extensions/Implicits.scala
@@ -19,13 +19,14 @@
package astraea.spark.rasterframes.extensions
-import astraea.spark.rasterframes.RasterFrame
-import geotrellis.raster.{ProjectedRaster, Tile, TileFeature}
+import astraea.spark.rasterframes.{PairRDDConverter, RasterFrame}
+import astraea.spark.rasterframes.util.{WithMergeMethods, WithPrototypeMethods}
+import geotrellis.raster._
import geotrellis.spark.{Metadata, SpaceTimeKey, SpatialComponent, SpatialKey, TileLayerMetadata}
import geotrellis.util.MethodExtensions
import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
-import org.apache.spark.sql.types.{MetadataBuilder, Metadata => SMetadata}
+import org.apache.spark.sql.types.{MetadataBuilder, Metadata ⇒ SMetadata}
import spray.json.JsonFormat
import scala.reflect.ClassTag
@@ -41,32 +42,31 @@ trait Implicits {
implicit class WithSQLContextMethods(val self: SQLContext) extends SQLContextMethods
- implicit class WithProjectedRasterMethods(val self: ProjectedRaster[Tile])
- extends ProjectedRasterMethods
+ implicit class WithProjectedRasterMethods[T <: CellGrid: WithMergeMethods: WithPrototypeMethods: TypeTag](
+ val self: ProjectedRaster[T]) extends ProjectedRasterMethods[T]
implicit class WithDataFrameMethods[D <: DataFrame](val self: D) extends DataFrameMethods[D]
implicit class WithRasterFrameMethods(val self: RasterFrame) extends RasterFrameMethods
- implicit class WithSpatialContextRDDMethods(
- val self: RDD[(SpatialKey, Tile)] with Metadata[TileLayerMetadata[SpatialKey]]
- )(implicit spark: SparkSession) extends SpatialContextRDDMethods
-
- implicit class WithSpatioTemporalContextRDDMethods(
- val self: RDD[(SpaceTimeKey, Tile)] with Metadata[TileLayerMetadata[SpaceTimeKey]]
- )(implicit spark: SparkSession) extends SpatioTemporalContextRDDMethods
-
- implicit class WithTFContextRDDMethods[K: SpatialComponent: JsonFormat: ClassTag: TypeTag,
- D: TypeTag](
- val self: RDD[(K, TileFeature[Tile, D])] with Metadata[TileLayerMetadata[K]]
- )(implicit spark: SparkSession)
- extends TFContextRDDMethods[K, D]
-
- implicit class WithTFSTContextRDDMethods[D: TypeTag](
- val self: RDD[(SpaceTimeKey, TileFeature[Tile, D])] with Metadata[
- TileLayerMetadata[SpaceTimeKey]]
- )(implicit spark: SparkSession)
- extends TFSTContextRDDMethods[D]
+ implicit class WithSpatialContextRDDMethods[T <: CellGrid](
+ val self: RDD[(SpatialKey, T)] with Metadata[TileLayerMetadata[SpatialKey]]
+ )(implicit spark: SparkSession) extends SpatialContextRDDMethods[T]
+
+ implicit class WithSpatioTemporalContextRDDMethods[T <: CellGrid](
+ val self: RDD[(SpaceTimeKey, T)] with Metadata[TileLayerMetadata[SpaceTimeKey]]
+ )(implicit spark: SparkSession) extends SpatioTemporalContextRDDMethods[T]
+
+// implicit class WithTFContextRDDMethods[K: SpatialComponent: JsonFormat: TypeTag, D: TypeTag](
+// val self: RDD[(K, TileFeature[Tile, D])] with Metadata[TileLayerMetadata[K]]
+// )(implicit spark: SparkSession)
+// extends TFContextRDDMethods[K, D]
+//
+// implicit class WithTFSTContextRDDMethods[D: TypeTag](
+// val self: RDD[(SpaceTimeKey, TileFeature[Tile, D])] with Metadata[
+// TileLayerMetadata[SpaceTimeKey]]
+// )(implicit spark: SparkSession)
+// extends TFSTContextRDDMethods[D]
private[astraea] implicit class WithMetadataMethods[R: JsonFormat](val self: R)
extends MetadataMethods[R]
diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/ProjectedRasterMethods.scala b/core/src/main/scala/astraea/spark/rasterframes/extensions/ProjectedRasterMethods.scala
index 23a825d8b..7b14d03fe 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/extensions/ProjectedRasterMethods.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/extensions/ProjectedRasterMethods.scala
@@ -2,23 +2,26 @@ package astraea.spark.rasterframes.extensions
import java.time.ZonedDateTime
-import astraea.spark.rasterframes.{RasterFrame, StandardColumns}
-import geotrellis.raster.{ProjectedRaster, Tile, TileLayout}
+import astraea.spark.rasterframes.util._
+import astraea.spark.rasterframes.{PairRDDConverter, RasterFrame, StandardColumns}
+import geotrellis.raster.{CellGrid, ProjectedRaster}
import geotrellis.spark._
import geotrellis.spark.tiling._
-import geotrellis.spark.io._
import geotrellis.util.MethodExtensions
+import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
-import astraea.spark.rasterframes.util._
+
+import scala.reflect.runtime.universe._
/**
* Extension methods on [[ProjectedRaster]] for creating [[RasterFrame]]s.
*
* @since 8/10/17
*/
-trait ProjectedRasterMethods extends MethodExtensions[ProjectedRaster[Tile]] with StandardColumns {
- import Implicits.WithSpatialContextRDDMethods
- import Implicits.WithSpatioTemporalContextRDDMethods
+abstract class ProjectedRasterMethods[T <: CellGrid: WithMergeMethods: WithPrototypeMethods: TypeTag]
+ extends MethodExtensions[ProjectedRaster[T]] with StandardColumns {
+ import Implicits.{WithSpatialContextRDDMethods, WithSpatioTemporalContextRDDMethods}
+ type XTileLayerRDD[K] = RDD[(K, T)] with Metadata[TileLayerMetadata[K]]
/**
* Convert the wrapped [[ProjectedRaster]] into a [[RasterFrame]] with a
@@ -26,8 +29,7 @@ trait ProjectedRasterMethods extends MethodExtensions[ProjectedRaster[Tile]] wit
*
* @param spark [[SparkSession]] in which to create [[RasterFrame]]
*/
- def toRF(implicit spark: SparkSession): RasterFrame = toRF(TILE_COLUMN.columnName)
-
+ def toRF(implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrame = toRF(TILE_COLUMN.columnName)
/**
* Convert the wrapped [[ProjectedRaster]] into a [[RasterFrame]] with a
@@ -35,7 +37,8 @@ trait ProjectedRasterMethods extends MethodExtensions[ProjectedRaster[Tile]] wit
*
* @param spark [[SparkSession]] in which to create [[RasterFrame]]
*/
- def toRF(tileColName: String)(implicit spark: SparkSession): RasterFrame = {
+ def toRF(tileColName: String)
+ (implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrame = {
val (cols, rows) = self.raster.dimensions
toRF(cols, rows, tileColName)
}
@@ -45,11 +48,26 @@ trait ProjectedRasterMethods extends MethodExtensions[ProjectedRaster[Tile]] wit
* given dimensions as the target per-row tile size.
*
* @param tileCols Max number of horizontal cells per tile
- * @param tileRows Max number of vertical cells per tile.
+ * @param tileRows Max number of vertical cells per tile
+ * @param spark [[SparkSession]] in which to create [[RasterFrame]]
+ */
+ def toRF(tileCols: Int, tileRows: Int)
+ (implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrame =
+ toRF(tileCols, tileRows, TILE_COLUMN.columnName)
+
+ /**
+ * Convert the [[ProjectedRaster]] into a [[RasterFrame]] using the
+ * given dimensions as the target per-row tile size.
+ *
+ * @param tileCols Max number of horizontal cells per tile
+ * @param tileRows Max number of vertical cells per tile
+ * @param tileColName Name to give the created tile column
* @param spark [[SparkSession]] in which to create [[RasterFrame]]
*/
- def toRF(tileCols: Int, tileRows: Int, tileColName: String = TILE_COLUMN.columnName)(implicit spark: SparkSession): RasterFrame =
+ def toRF(tileCols: Int, tileRows: Int, tileColName: String)
+ (implicit spark: SparkSession, schema: PairRDDConverter[SpatialKey, T]): RasterFrame = {
toTileLayerRDD(tileCols, tileRows).toRF(tileColName)
+ }
/**
* Convert the [[ProjectedRaster]] into a [[RasterFrame]] using the
@@ -60,7 +78,8 @@ trait ProjectedRasterMethods extends MethodExtensions[ProjectedRaster[Tile]] wit
* @param timestamp Temporal key value to assign to tiles.
* @param spark [[SparkSession]] in which to create [[RasterFrame]]
*/
- def toRF(tileCols: Int, tileRows: Int, timestamp: ZonedDateTime)(implicit spark: SparkSession): RasterFrame =
+ def toRF(tileCols: Int, tileRows: Int, timestamp: ZonedDateTime)
+ (implicit spark: SparkSession, schema: PairRDDConverter[SpaceTimeKey, T]): RasterFrame =
toTileLayerRDD(tileCols, tileRows, timestamp).toRF
/**
@@ -72,13 +91,15 @@ trait ProjectedRasterMethods extends MethodExtensions[ProjectedRaster[Tile]] wit
* @param spark [[SparkSession]] in which to create RDD
*/
def toTileLayerRDD(tileCols: Int,
- tileRows: Int)(implicit spark: SparkSession): TileLayerRDD[SpatialKey] = {
+ tileRows: Int)(implicit spark: SparkSession): XTileLayerRDD[SpatialKey] = {
val layout = LayoutDefinition(self.rasterExtent, tileCols, tileRows)
val kb = KeyBounds(SpatialKey(0, 0), SpatialKey(layout.layoutCols - 1, layout.layoutRows - 1))
val tlm = TileLayerMetadata(self.tile.cellType, layout, self.extent, self.crs, kb)
val rdd = spark.sparkContext.makeRDD(Seq((self.projectedExtent, self.tile)))
+ implicit val tct = typeTag[T].asClassTag
+
val tiled = rdd.tileToLayout(tlm)
ContextRDD(tiled, tlm)
@@ -93,13 +114,15 @@ trait ProjectedRasterMethods extends MethodExtensions[ProjectedRaster[Tile]] wit
* @param timestamp Temporal key value to assign to tiles.
* @param spark [[SparkSession]] in which to create RDD
*/
- def toTileLayerRDD(tileCols: Int, tileRows: Int, timestamp: ZonedDateTime)(implicit spark: SparkSession): TileLayerRDD[SpaceTimeKey] = {
+ def toTileLayerRDD(tileCols: Int, tileRows: Int, timestamp: ZonedDateTime)(implicit spark: SparkSession): XTileLayerRDD[SpaceTimeKey] = {
val layout = LayoutDefinition(self.rasterExtent, tileCols, tileRows)
val kb = KeyBounds(SpaceTimeKey(0, 0, timestamp), SpaceTimeKey(layout.layoutCols - 1, layout.layoutRows - 1, timestamp))
val tlm = TileLayerMetadata(self.tile.cellType, layout, self.extent, self.crs, kb)
val rdd = spark.sparkContext.makeRDD(Seq((TemporalProjectedExtent(self.projectedExtent, timestamp), self.tile)))
+ implicit val tct = typeTag[T].asClassTag
+
val tiled = rdd.tileToLayout(tlm)
ContextRDD(tiled, tlm)
diff --git a/core/src/main/scala/astraea/spark/rasterframes/extensions/RasterFrameMethods.scala b/core/src/main/scala/astraea/spark/rasterframes/extensions/RasterFrameMethods.scala
index 64fcacf78..e2d2746f5 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/extensions/RasterFrameMethods.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/extensions/RasterFrameMethods.scala
@@ -21,15 +21,14 @@ import java.time.ZonedDateTime
import astraea.spark.rasterframes.util._
import astraea.spark.rasterframes.{MetadataKeys, RasterFrame}
import geotrellis.proj4.CRS
-import geotrellis.raster.resample.{Bilinear, ResampleMethod}
-import geotrellis.raster.{CellGrid, MultibandTile, ProjectedRaster, Tile, TileLayout}
+import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod}
+import geotrellis.raster.{MultibandTile, ProjectedRaster, Tile, TileLayout}
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.tiling.{LayoutDefinition, Tiler}
import geotrellis.util.{LazyLogging, MethodExtensions}
import geotrellis.vector.ProjectedExtent
import org.apache.spark.annotation.Experimental
-import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types.{Metadata, TimestampType}
@@ -302,7 +301,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame]
def toRaster(tileCol: Column,
rasterCols: Int,
rasterRows: Int,
- resampler: ResampleMethod = Bilinear): ProjectedRaster[Tile] = {
+ resampler: ResampleMethod = NearestNeighbor): ProjectedRaster[Tile] = {
val clipped = clipLayerExtent
@@ -338,7 +337,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame]
tileCols: Seq[Column],
rasterCols: Int,
rasterRows: Int,
- resampler: ResampleMethod = Bilinear): ProjectedRaster[MultibandTile] = {
+ resampler: ResampleMethod = NearestNeighbor): ProjectedRaster[MultibandTile] = {
val clipped = clipLayerExtent
diff --git a/core/src/main/scala/astraea/spark/rasterframes/functions/CellMeanAggregateFunction.scala b/core/src/main/scala/astraea/spark/rasterframes/functions/CellMeanAggregateFunction.scala
index fbca53995..5489ee167 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/functions/CellMeanAggregateFunction.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/functions/CellMeanAggregateFunction.scala
@@ -59,7 +59,7 @@ case class CellMeanAggregateFunction(child: Expression) extends DeclarativeAggre
count.left + count.right
)
- val evaluateExpression = sum / Cast(count, DoubleType)
+ val evaluateExpression = sum / new Cast(count, DoubleType)
def inputTypes = Seq(TileUDT)
diff --git a/core/src/main/scala/astraea/spark/rasterframes/functions/CellStatsAggregateFunction.scala b/core/src/main/scala/astraea/spark/rasterframes/functions/CellStatsAggregateFunction.scala
index f21e6146f..7ed353682 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/functions/CellStatsAggregateFunction.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/functions/CellStatsAggregateFunction.scala
@@ -108,16 +108,6 @@ case class CellStatsAggregateFunction() extends UserDefinedAggregateFunction {
}
object CellStatsAggregateFunction {
- case class Statistics(dataCells: Long, min: Double, max: Double, mean: Double, variance: Double)
- object Statistics {
- // Convert GeoTrellis stats object into our simplified one.
- def apply(stats: geotrellis.raster.summary.Statistics[Double]) =
- new Statistics(stats.dataCells, stats.zmin, stats.zmax, stats.mean, stats.stddev * stats.stddev)
-
- def apply(stats: geotrellis.raster.summary.Statistics[Int])(implicit d: DummyImplicit) =
- new Statistics(stats.dataCells, stats.zmin.toDouble, stats.zmax.toDouble, stats.mean, stats.stddev * stats.stddev)
- }
-
/** Column index values. */
private object C {
val COUNT = 0
diff --git a/core/src/main/scala/astraea/spark/rasterframes/functions/HistogramAggregateFunction.scala b/core/src/main/scala/astraea/spark/rasterframes/functions/HistogramAggregateFunction.scala
index a768be958..f85b2ae65 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/functions/HistogramAggregateFunction.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/functions/HistogramAggregateFunction.scala
@@ -16,12 +16,15 @@
package astraea.spark.rasterframes.functions
+import astraea.spark.rasterframes.encoders.StandardEncoders
+import astraea.spark.rasterframes.stats.CellHistogram
import geotrellis.raster.Tile
import geotrellis.raster.histogram.{Histogram, StreamingHistogram}
+import geotrellis.spark.util.KryoSerializer
import org.apache.spark.sql.Row
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
-import org.apache.spark.sql.gt.types.{HistogramUDT, TileUDT}
-import org.apache.spark.sql.types.{DataType, StructField, StructType}
+import org.apache.spark.sql.gt.types.TileUDT
+import org.apache.spark.sql.types._
/**
* Histogram aggregation function for a full column of tiles.
@@ -31,26 +34,43 @@ import org.apache.spark.sql.types.{DataType, StructField, StructType}
case class HistogramAggregateFunction() extends UserDefinedAggregateFunction {
override def inputSchema: StructType = StructType(StructField("value", TileUDT) :: Nil)
- override def bufferSchema: StructType = StructType(StructField("buffer", HistogramUDT) :: Nil)
+ override def bufferSchema: StructType = StructType(StructField("buffer", BinaryType) :: Nil)
- override def dataType: DataType = new HistogramUDT()
+ override def dataType: DataType = StandardEncoders.histEncoder.schema
override def deterministic: Boolean = true
+ @inline
+ private def marshall(hist: Histogram[Double]): Array[Byte] =
+ KryoSerializer.serialize(hist)
+
+ @inline
+ private def unmarshall(blob: Array[Byte]): Histogram[Double] =
+ KryoSerializer.deserialize(blob)
+
override def initialize(buffer: MutableAggregationBuffer): Unit =
- buffer(0) = StreamingHistogram()
+ buffer(0) = marshall(StreamingHistogram())
override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
- val hist = buffer.getAs[Histogram[Double]](0)
+ val hist = unmarshall(buffer.getAs[Array[Byte]](0))
val tile = input.getAs[Tile](0)
- buffer(0) = safeEval((h: Histogram[Double], t: Tile) ⇒ h.merge(StreamingHistogram.fromTile(t)))(hist, tile)
+ val updatedHist = safeEval((h: Histogram[Double], t: Tile) ⇒ h.merge(StreamingHistogram.fromTile(t)))(hist, tile)
+ buffer(0) = marshall(updatedHist)
}
override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
- val hist1 = buffer1.getAs[Histogram[Double]](0)
- val hist2 = buffer2.getAs[Histogram[Double]](0)
- buffer1(0) = safeEval((h1: Histogram[Double], h2: Histogram[Double]) ⇒ h1 merge h2)(hist1, hist2)
+ val hist1 = unmarshall(buffer1.getAs[Array[Byte]](0))
+ val hist2 = unmarshall(buffer2.getAs[Array[Byte]](0))
+ val updatedHist = safeEval((h1: Histogram[Double], h2: Histogram[Double]) ⇒ h1 merge h2)(hist1, hist2)
+ buffer1(0) = marshall(updatedHist)
}
- override def evaluate(buffer: Row): Any = buffer.getAs[Histogram[Double]](0)
+ override def evaluate(buffer: Row): Any = {
+ val hist = unmarshall(buffer.getAs[Array[Byte]](0))
+ CellHistogram(hist)
+ }
+}
+
+object HistogramAggregateFunction {
+ case class RFTileHistogram()
}
diff --git a/core/src/main/scala/astraea/spark/rasterframes/functions/package.scala b/core/src/main/scala/astraea/spark/rasterframes/functions/package.scala
index a5a4f97de..d5eb514d4 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/functions/package.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/functions/package.scala
@@ -15,7 +15,7 @@
*/
package astraea.spark.rasterframes
-import geotrellis.raster.histogram.Histogram
+import astraea.spark.rasterframes.stats.{CellHistogram, CellStatistics}
import geotrellis.raster.mapalgebra.local._
import geotrellis.raster._
import geotrellis.raster.render.ascii.AsciiArtEncoder
@@ -30,7 +30,6 @@ import scala.reflect.runtime.universe._
* @since 9/7/17
*/
package object functions {
- import astraea.spark.rasterframes.functions.CellStatsAggregateFunction.Statistics
@inline
private[rasterframes] def safeBinaryOp[T <: AnyRef, R >: T](op: (T, T) ⇒ R): ((T, T) ⇒ R) =
@@ -144,16 +143,19 @@ package object functions {
/** Computes the column aggregate statistics */
private[rasterframes] val aggStats = CellStatsAggregateFunction()
+ /** Change the tile's cell type. */
+ private[rasterframes] def convertCellType(cellType: CellType) = safeEval[Tile, Tile](_.convert(cellType))
+
/** Set the tile's no-data value. */
private[rasterframes] def withNoData(nodata: Double) = safeEval[Tile, Tile](_.withNoData(Some(nodata)))
/** Single tile histogram. */
- private[rasterframes] val tileHistogram = safeEval[Tile, Histogram[Double]](_.histogramDouble)
+ private[rasterframes] val tileHistogram = safeEval[Tile, CellHistogram](t ⇒ CellHistogram(t.histogramDouble))
/** Single tile statistics. Convenience for `tileHistogram.statistics`. */
- private[rasterframes] val tileStats = safeEval[Tile, Statistics]((t: Tile) ⇒
- if (t.cellType.isFloatingPoint) t.statisticsDouble.map(Statistics.apply).orNull
- else t.statistics.map(Statistics.apply).orNull
+ private[rasterframes] val tileStats = safeEval[Tile, CellStatistics]((t: Tile) ⇒
+ if (t.cellType.isFloatingPoint) t.statisticsDouble.map(CellStatistics.apply).orNull
+ else t.statistics.map(CellStatistics.apply).orNull
)
/** Add up all the cell values. */
diff --git a/core/src/main/scala/astraea/spark/rasterframes/jts/Implicits.scala b/core/src/main/scala/astraea/spark/rasterframes/jts/Implicits.scala
index d58996e9d..3362dc0d0 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/jts/Implicits.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/jts/Implicits.scala
@@ -29,12 +29,13 @@ import geotrellis.vector.{Point ⇒ gtPoint}
import org.apache.spark.sql.TypedColumn
import org.apache.spark.sql.functions._
import org.apache.spark.sql.rf.CanBeColumn
+import org.locationtech.geomesa.spark.jts.DataFrameFunctions.SpatialConstructors
/**
* Extension methods on typed columns allowing for DSL-like queries over JTS types.
* @since 1/10/18
*/
-trait Implicits extends SpatialFunctions {
+trait Implicits extends SpatialConstructors {
import astraea.spark.rasterframes.encoders.SparkDefaultEncoders._
implicit class ExtentColumnMethods[T <: Geometry](val self: TypedColumn[Any, T])
diff --git a/core/src/main/scala/astraea/spark/rasterframes/jts/SpatialFunctions.scala b/core/src/main/scala/astraea/spark/rasterframes/jts/SpatialFunctions.scala
deleted file mode 100644
index c4fe7357a..000000000
--- a/core/src/main/scala/astraea/spark/rasterframes/jts/SpatialFunctions.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-package astraea.spark.rasterframes.jts
-
-import com.vividsolutions.jts.geom._
-import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.jts._
-import org.apache.spark.sql.{Column, Encoder, TypedColumn}
-
-/**
- * Spatial type support functions.
- *
- * @since 2/19/18
- */
-trait SpatialFunctions {
- import astraea.spark.rasterframes.encoders.SparkDefaultEncoders._
-
- /** Constructs a geometric literal from a value and JTS UDT */
- private def udtlit[T >: Null <: Geometry: Encoder, U <: AbstractGeometryUDT[T]](t: T, u: U): TypedColumn[Any, T] =
- new Column(Literal.create(u.serialize(t), u)).as[T]
-
- /** Create a generic geometry literal, encoded as a GeometryUDT. */
- def geomLit(g: Geometry): TypedColumn[Any, Geometry] = udtlit(g, GeometryUDT)
- /** Create a point literal, encoded as a PointUDT. */
- def pointLit(g: Point): TypedColumn[Any, Point] = udtlit(g, PointUDT)
- /** Create a line literal, encoded as a LineUDT. */
- def lineLit(g: LineString): TypedColumn[Any, LineString] = udtlit(g, LineStringUDT)
- /** Create a polygon literal, encoded as a PolygonUDT. */
- def polygonLit(g: Polygon): TypedColumn[Any, Polygon] = udtlit(g, PolygonUDT)
- /** Create a multi-point literal, encoded as a MultiPointUDT. */
- def mPointLit(g: MultiPoint): TypedColumn[Any, MultiPoint] = udtlit(g, MultiPointUDT)
- /** Create a multi-line literal, encoded as a MultiPointUDT. */
- def mLineLit(g: MultiLineString): TypedColumn[Any, MultiLineString] = udtlit(g, MultiLineStringUDT)
- /** Create a multi-polygon literal, encoded as a MultiPolygonUDT. */
- def mPolygonLit(g: MultiPolygon): TypedColumn[Any, MultiPolygon] = udtlit(g, MultiPolygonUDT)
- /** create a geometry collection literal, encoded as a GeometryCollectionUDT. */
- def geomCollLit(g: GeometryCollection): TypedColumn[Any, GeometryCollection] = udtlit(g, GeometryCollectionUDT)
-}
diff --git a/core/src/main/scala/astraea/spark/rasterframes/package.scala b/core/src/main/scala/astraea/spark/rasterframes/package.scala
index 51e7f1f2b..332f8dec1 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/package.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/package.scala
@@ -17,15 +17,18 @@
package astraea.spark
import astraea.spark.rasterframes.encoders.StandardEncoders
-import geotrellis.raster.{Tile, TileFeature}
-import geotrellis.spark.{Bounds, ContextRDD, Metadata, TileLayerMetadata}
-import geotrellis.util.GetComponent
+import geotrellis.raster.{MultibandTile, Tile, TileFeature}
+import geotrellis.spark.{ContextRDD, Metadata, SpaceTimeKey, SpatialKey, TileLayerMetadata}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
import org.locationtech.geomesa.spark.jts.DataFrameFunctions
import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders
import shapeless.tag.@@
+import scala.annotation.implicitNotFound
+import scala.language.higherKinds
+import scala.reflect.runtime.universe._
+
/**
* Module providing support for RasterFrames.
* `import astraea.spark.rasterframes._`., and then call `rfInit(SQLContext)`.
@@ -39,7 +42,6 @@ package object rasterframes extends StandardColumns
with StandardEncoders
with SpatialEncoders
with DataFrameFunctions.Library {
- type Statistics = astraea.spark.rasterframes.functions.CellStatsAggregateFunction.Statistics
/**
* Initialization injection point. Must be called before any RasterFrame
@@ -65,14 +67,6 @@ package object rasterframes extends StandardColumns
/** Tagged type for allowing compiler to help keep track of what has RasterFrame assurances applied to it. */
trait RasterFrameTag
- /**
- * Type lambda alias for components that have bounds with parameterized key.
- * @tparam K bounds key type
- */
- type BoundsComponentOf[K] = {
- type get[M] = GetComponent[M, Bounds[K]]
- }
-
type TileFeatureLayerRDD[K, D] =
RDD[(K, TileFeature[Tile, D])] with Metadata[TileLayerMetadata[K]]
@@ -82,6 +76,7 @@ package object rasterframes extends StandardColumns
new ContextRDD(rdd, metadata)
}
+ /** Provides evidence that a given primitive has an associated CellType. */
trait HasCellType[T] extends Serializable
object HasCellType {
implicit val intHasCellType = new HasCellType[Int] {}
@@ -90,4 +85,20 @@ package object rasterframes extends StandardColumns
implicit val shortHasCellType = new HasCellType[Short] {}
implicit val floatHasCellType = new HasCellType[Float] {}
}
+
+ /** Evidence type class for communicating that only standard key types are supported with the more general GeoTrellis type parameters. */
+ trait StandardLayerKey[T] extends Serializable {
+ val selfType: TypeTag[T]
+ def isType[R: TypeTag]: Boolean = typeOf[R] =:= selfType.tpe
+ }
+ object StandardLayerKey {
+ def apply[T: StandardLayerKey]: StandardLayerKey[T] = implicitly
+ implicit val spatialKeySupport = new StandardLayerKey[SpatialKey] {
+ override val selfType: TypeTag[SpatialKey] = implicitly
+ }
+ implicit val spatioTemporalKeySupport = new StandardLayerKey[SpaceTimeKey] {
+ override val selfType: TypeTag[SpaceTimeKey] = implicitly
+ }
+ }
+
}
diff --git a/core/src/main/scala/astraea/spark/rasterframes/stats/CellHistogram.scala b/core/src/main/scala/astraea/spark/rasterframes/stats/CellHistogram.scala
new file mode 100644
index 000000000..71cb5cc2b
--- /dev/null
+++ b/core/src/main/scala/astraea/spark/rasterframes/stats/CellHistogram.scala
@@ -0,0 +1,45 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2018 Astraea. Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ */
+
+package astraea.spark.rasterframes.stats
+import astraea.spark.rasterframes.stats.CellHistogram.Bin
+import geotrellis.raster.histogram.{Histogram ⇒ GTHistogram}
+
+/**
+ * Container for histogram computed
+ *
+ * @since 4/3/18
+ */
+case class CellHistogram(stats: CellStatistics, bins: Seq[Bin]) {
+ def mean = stats.mean
+ def totalCount = stats.dataCells
+}
+
+object CellHistogram {
+ case class Bin(value: Double, count: Long)
+ def apply(hist: GTHistogram[Int]): CellHistogram = {
+ val stats = CellStatistics(hist.statistics().get)
+ CellHistogram(stats, hist.binCounts().map(p ⇒ Bin(p._1.toDouble, p._2)))
+ }
+ def apply(hist: GTHistogram[Double])(implicit ev: DummyImplicit): CellHistogram = {
+ val stats = CellStatistics(hist.statistics().get)
+ CellHistogram(stats, hist.binCounts().map(p ⇒ Bin(p._1, p._2)))
+ }
+}
diff --git a/core/src/main/scala/astraea/spark/rasterframes/stats/CellStatistics.scala b/core/src/main/scala/astraea/spark/rasterframes/stats/CellStatistics.scala
new file mode 100644
index 000000000..567a69a78
--- /dev/null
+++ b/core/src/main/scala/astraea/spark/rasterframes/stats/CellStatistics.scala
@@ -0,0 +1,38 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2018 Astraea. Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ */
+
+package astraea.spark.rasterframes.stats
+
+/**
+ * Container for computed statistics over cells.
+ *
+ * @since 4/3/18
+ */
+case class CellStatistics(dataCells: Long, min: Double, max: Double, mean: Double, variance: Double) {
+ def stddev: Double = math.sqrt(variance)
+}
+object CellStatistics {
+ // Convert GeoTrellis stats object into our simplified one.
+ def apply(stats: geotrellis.raster.summary.Statistics[Double]) =
+ new CellStatistics(stats.dataCells, stats.zmin, stats.zmax, stats.mean, stats.stddev * stats.stddev)
+
+ def apply(stats: geotrellis.raster.summary.Statistics[Int])(implicit d: DummyImplicit) =
+ new CellStatistics(stats.dataCells, stats.zmin.toDouble, stats.zmax.toDouble, stats.mean, stats.stddev * stats.stddev)
+}
\ No newline at end of file
diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/SubdivideSupport.scala b/core/src/main/scala/astraea/spark/rasterframes/util/SubdivideSupport.scala
new file mode 100644
index 000000000..c94b425d4
--- /dev/null
+++ b/core/src/main/scala/astraea/spark/rasterframes/util/SubdivideSupport.scala
@@ -0,0 +1,118 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2018 Astraea. Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ */
+
+package astraea.spark.rasterframes.util
+
+import geotrellis.raster.crop.Crop
+import geotrellis.raster.{CellGrid, TileLayout}
+import geotrellis.spark.{Bounds, KeyBounds, SpatialComponent, SpatialKey, TileLayerMetadata}
+import geotrellis.util._
+
+/**
+ *
+ *
+ * @since 4/5/18
+ */
+trait SubdivideSupport {
+ implicit class TileLayoutHasSubdivide(self: TileLayout) {
+ def subdivide(divs: Int): TileLayout = {
+ def shrink(num: Int) = {
+ require(num % divs == 0, s"Subdivision of '$divs' does not evenly divide into dimension '$num'")
+ num / divs
+ }
+ def grow(num: Int) = num * divs
+
+ divs match {
+ case 0 ⇒ self
+ case i if i < 0 ⇒ throw new IllegalArgumentException(s"divs=$divs must be positive")
+ case _ ⇒
+ TileLayout(
+ layoutCols = grow(self.layoutCols),
+ layoutRows = grow(self.layoutRows),
+ tileCols = shrink(self.tileCols),
+ tileRows = shrink(self.tileRows)
+ )
+ }
+ }
+ }
+
+ implicit class BoundsHasSubdivide[K: SpatialComponent](self: Bounds[K]) {
+ def subdivide(divs: Int): Bounds[K] = {
+ self.map(kb ⇒ {
+ val currGrid = kb.toGridBounds()
+ // NB: As with GT regrid, we keep the spatial key origin (0, 0) at the same map coordinate
+ val newGrid = currGrid.copy(
+ colMin = currGrid.colMin * divs,
+ rowMin = currGrid.rowMin * divs,
+ colMax = currGrid.colMin * divs + (currGrid.width - 1) * divs + 1,
+ rowMax = currGrid.rowMin * divs + (currGrid.height - 1) * divs + 1
+ )
+ kb.setSpatialBounds(KeyBounds(newGrid))
+ })
+ }
+ }
+
+ /**
+ * Note: this enrichment makes the assumption that the new keys will be used in the
+ * layout regime defined by `TileLayoutHasSubdivide`
+ */
+ implicit class SpatialKeyHasSubdivide[K: SpatialComponent](self: K) {
+ def subdivide(divs: Int): Seq[K] = {
+ val base = self.getComponent[SpatialKey]
+ val shifted = SpatialKey(base.col * divs, base.row * divs)
+
+ for{
+ i ← 0 until divs
+ j ← 0 until divs
+ } yield {
+ val newKey = SpatialKey(shifted.col + j, shifted.row + i)
+ self.setComponent(newKey)
+ }
+ }
+ }
+
+ implicit class TileLayerMetadataHasSubdivide[K: SpatialComponent](tlm: TileLayerMetadata[K]) {
+ def subdivide(divs: Int): TileLayerMetadata[K] = {
+ val tileLayout = tlm.layout.tileLayout.subdivide(divs)
+ val layout = tlm.layout.copy(tileLayout = tileLayout)
+ val bounds = tlm.bounds.subdivide(divs)
+ tlm.copy(layout = layout, bounds = bounds)
+ }
+ }
+
+ implicit class TileHasSubdivide[T <: CellGrid: WithCropMethods](self: T) {
+ def subdivide(divs: Int): Seq[T] = {
+ val (cols, rows) = self.dimensions
+ val (newCols, newRows) = (cols/divs, rows/divs)
+ for {
+ i ← 0 until divs
+ j ← 0 until divs
+ } yield {
+ val startCol = j * newCols
+ val startRow = i * newRows
+ val endCol = startCol + newCols - 1
+ val endRow = startRow + newRows - 1
+ self.crop(startCol, startRow, endCol, endRow, Crop.Options(force = true))
+ }
+ }
+ }
+}
+
+object SubdivideSupport extends SubdivideSupport
diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/debug/package.scala b/core/src/main/scala/astraea/spark/rasterframes/util/debug/package.scala
new file mode 100644
index 000000000..4a0df0bdb
--- /dev/null
+++ b/core/src/main/scala/astraea/spark/rasterframes/util/debug/package.scala
@@ -0,0 +1,48 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2018 Astraea. Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ */
+
+package astraea.spark.rasterframes.util
+
+import java.net.URI
+
+import astraea.spark.rasterframes._
+import geotrellis.proj4.WebMercator
+import geotrellis.raster.MultibandTile
+import geotrellis.raster.io.geotiff.tags.codes.ColorSpace
+import geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff}
+import geotrellis.spark.{ContextRDD, MultibandTileLayerRDD, SpatialKey}
+import geotrellis.spark.io.slippy.HadoopSlippyTileWriter
+import geotrellis.spark.tiling.ZoomedLayoutScheme
+import org.apache.spark.annotation.Experimental
+
+/**
+ * Additional debugging routines. No guarantees these are or will remain stable.
+ *
+ * @since 4/6/18
+ */
+package object debug {
+ implicit class RasterFrameWithDebug(val self: RasterFrame) {
+
+ /** Renders the whole schema with metadata as a JSON string. */
+ def describeFullSchema: String = {
+ self.schema.prettyJson
+ }
+ }
+}
diff --git a/core/src/main/scala/astraea/spark/rasterframes/util/package.scala b/core/src/main/scala/astraea/spark/rasterframes/util/package.scala
index bde570644..f80c122ca 100644
--- a/core/src/main/scala/astraea/spark/rasterframes/util/package.scala
+++ b/core/src/main/scala/astraea/spark/rasterframes/util/package.scala
@@ -19,8 +19,15 @@
package astraea.spark.rasterframes
+import geotrellis.raster.CellGrid
+import geotrellis.raster.crop.TileCropMethods
import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp
-import geotrellis.util.LazyLogging
+import geotrellis.raster.mask.TileMaskMethods
+import geotrellis.raster.merge.TileMergeMethods
+import geotrellis.raster.prototype.TilePrototypeMethods
+import geotrellis.spark.Bounds
+import geotrellis.spark.tiling.TilerKeyMethods
+import geotrellis.util.{GetComponent, LazyLogging}
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
@@ -29,7 +36,6 @@ import org.apache.spark.sql.rf._
import org.apache.spark.sql.{Column, DataFrame, SQLContext}
import shapeless.Lub
-
/**
* Internal utilities.
*
@@ -37,6 +43,29 @@ import shapeless.Lub
*/
package object util extends LazyLogging {
+ import reflect.ClassTag
+ import reflect.runtime.universe._
+
+ implicit class TypeTagCanBeClassTag[T](val t: TypeTag[T]) extends AnyVal {
+ def asClassTag: ClassTag[T] = ClassTag[T](t.mirror.runtimeClass(t.tpe))
+ }
+
+ /**
+ * Type lambda alias for components that have bounds with parameterized key.
+ * @tparam K bounds key type
+ */
+ type BoundsComponentOf[K] = {
+ type Get[M] = GetComponent[M, Bounds[K]]
+ }
+
+ // Type lambda aliases
+ type WithMergeMethods[V] = (V ⇒ TileMergeMethods[V])
+ type WithPrototypeMethods[V <: CellGrid] = (V ⇒ TilePrototypeMethods[V])
+ type WithCropMethods[V <: CellGrid] = (V ⇒ TileCropMethods[V])
+ type WithMaskMethods[V] = (V ⇒ TileMaskMethods[V])
+
+ type KeyMethodsProvider[K1, K2] = K1 ⇒ TilerKeyMethods[K1, K2]
+
/** Internal method for slapping the RasterFrame seal of approval on a DataFrame. */
private[rasterframes] def certifyRasterframe(df: DataFrame): RasterFrame =
shapeless.tag[RasterFrameTag][DataFrame](df)
@@ -55,9 +84,7 @@ package object util extends LazyLogging {
op.getClass.getSimpleName.replace("$", "").toLowerCase
- // $COVERAGE-OFF$
implicit class WithWiden[A, B](thing: Either[A, B]) {
-
/** Returns the value as a LUB of the Left & Right items. */
def widen[Out](implicit ev: Lub[A, B, Out]): Out =
thing.fold(identity, identity).asInstanceOf[Out]
@@ -106,5 +133,4 @@ package object util extends LazyLogging {
logger.error("Extended rule resolution not available in this version of Spark")
analyzer(sqlContext).extendedResolutionRules
}
- // $COVERAGE-ON$
}
diff --git a/core/src/main/scala/org/apache/spark/sql/gt/package.scala b/core/src/main/scala/org/apache/spark/sql/gt/package.scala
index 5f856aaea..9e469552b 100644
--- a/core/src/main/scala/org/apache/spark/sql/gt/package.scala
+++ b/core/src/main/scala/org/apache/spark/sql/gt/package.scala
@@ -16,7 +16,7 @@
package org.apache.spark.sql
-import org.apache.spark.sql.gt.types.{HistogramUDT, TileUDT}
+import org.apache.spark.sql.gt.types.TileUDT
/**
* Module of GeoTrellis UDTs for Spark/Catalyst.
@@ -28,6 +28,5 @@ package object gt {
// Referencing the companion objects here is intended to have it's constructor called,
// which is where the registration actually happens.
TileUDT
- HistogramUDT
}
}
diff --git a/core/src/main/scala/org/apache/spark/sql/gt/types/HistogramUDT.scala b/core/src/main/scala/org/apache/spark/sql/gt/types/HistogramUDT.scala
deleted file mode 100644
index feca59e6a..000000000
--- a/core/src/main/scala/org/apache/spark/sql/gt/types/HistogramUDT.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2017 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.gt.types
-
-import geotrellis.raster.histogram.Histogram
-import org.apache.spark.sql.rf.KryoBackedUDT
-import org.apache.spark.sql.types._
-
-/**
- * Wraps up GT Histogram type.
- *
- * @since 4/18/17
- */
-class HistogramUDT extends UserDefinedType[Histogram[Double]] with KryoBackedUDT[Histogram[Double]] {
-
- override val typeName = "gt_histogram"
-
- override val targetClassTag = scala.reflect.classTag[Histogram[Double]]
-
- private[sql] override def acceptsType(dataType: DataType) = dataType match {
- case o: HistogramUDT ⇒ o.typeName == this.typeName
- case _ ⇒ super.acceptsType(dataType)
- }
-}
-
-object HistogramUDT extends HistogramUDT {
- UDTRegistration.register(classOf[Histogram[Double]].getName, classOf[HistogramUDT].getName)
-}
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/package.scala b/core/src/main/scala/org/apache/spark/sql/rf/package.scala
index 532c723c5..9fbe0d651 100644
--- a/core/src/main/scala/org/apache/spark/sql/rf/package.scala
+++ b/core/src/main/scala/org/apache/spark/sql/rf/package.scala
@@ -16,11 +16,13 @@
package org.apache.spark.sql
+import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry, MultiAlias}
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.catalyst.expressions.{CreateArray, Expression, Inline}
import org.apache.spark.sql.types.{StructType, UDTRegistration, UserDefinedType}
+
import scala.reflect.runtime.universe._
/**
@@ -45,7 +47,15 @@ package object rf {
sqlContext.sessionState.analyzer
}
- // $COVERAGE-OFF$
+ /** Lookup the registered Catalyst UDT for the given Scala type. */
+ def udtOf[T >: Null: TypeTag]: UserDefinedType[T] =
+ UDTRegistration.getUDTFor(typeTag[T].tpe.toString).map(_.newInstance().asInstanceOf[UserDefinedType[T]])
+ .getOrElse(throw new IllegalArgumentException(typeTag[T].tpe + " doesn't have a corresponding UDT"))
+
+ /** Creates a Catalyst expression for flattening the fields in a struct into columns. */
+ def projectStructExpression(dataType: StructType, input: Expression) =
+ MultiAlias(Inline(CreateArray(Seq(input))), dataType.fields.map(_.name))
+
implicit class WithDecoder[T](enc: ExpressionEncoder[T]) {
def decode(row: InternalRow): T =
enc.resolveAndBind(enc.schema.toAttributes).fromRow(row)
@@ -63,13 +73,4 @@ package object rf {
}
}
- /** Lookup the registered Catalyst UDT for the given Scala type. */
- def udtOf[T >: Null: TypeTag]: UserDefinedType[T] =
- UDTRegistration.getUDTFor(typeTag[T].tpe.toString).map(_.newInstance().asInstanceOf[UserDefinedType[T]])
- .getOrElse(throw new IllegalArgumentException(typeTag[T].tpe + " doesn't have a corresponding UDT"))
-
- /** Creates a Catalyst expression for flattening the fields in a struct into columns. */
- def projectStructExpression(dataType: StructType, input: Expression) =
- MultiAlias(Inline(CreateArray(Seq(input))), dataType.fields.map(_.name))
- // $COVERAGE-ON$
}
diff --git a/core/src/test/resources/NAIP-VA-b1.tiff b/core/src/test/resources/NAIP-VA-b1.tiff
index 65140ed09..f1e4338a2 100644
Binary files a/core/src/test/resources/NAIP-VA-b1.tiff and b/core/src/test/resources/NAIP-VA-b1.tiff differ
diff --git a/core/src/test/resources/NAIP-VA-b2.tiff b/core/src/test/resources/NAIP-VA-b2.tiff
index 3e46fd32f..664d21f18 100644
Binary files a/core/src/test/resources/NAIP-VA-b2.tiff and b/core/src/test/resources/NAIP-VA-b2.tiff differ
diff --git a/core/src/test/resources/NAIP-VA-b3.tiff b/core/src/test/resources/NAIP-VA-b3.tiff
index ad107c38c..fc049a40d 100644
Binary files a/core/src/test/resources/NAIP-VA-b3.tiff and b/core/src/test/resources/NAIP-VA-b3.tiff differ
diff --git a/core/src/test/resources/NAIP-VA-b4.tiff b/core/src/test/resources/NAIP-VA-b4.tiff
index d9d9eaeb0..84e6c6432 100644
Binary files a/core/src/test/resources/NAIP-VA-b4.tiff and b/core/src/test/resources/NAIP-VA-b4.tiff differ
diff --git a/core/src/test/scala/Scratch.sc b/core/src/test/scala/Scratch.sc
index 13c8bfada..53bbd1457 100644
--- a/core/src/test/scala/Scratch.sc
+++ b/core/src/test/scala/Scratch.sc
@@ -1,16 +1,3 @@
-import com.vividsolutions.jts.geom._
-import org.geotools.geometry.jts.JTSFactoryFinder
+val s = Seq(1,2,3,4,5)
-val f = JTSFactoryFinder.getGeometryFactory
-
-val pt = f.createPoint(new Coordinate(1, 2))
-
-val poly = f.createPolygon(Array(new Coordinate(0, 0), new Coordinate(0, 3), new Coordinate(3, 3), new Coordinate(3, 0), new Coordinate(0, 0)))
-
-
-poly.intersects(pt)
-pt.intersects(poly)
-pt.intersects(pt)
-
-
-Double.NaN.toInt
+s.patch(1, Seq(9), 1)
\ No newline at end of file
diff --git a/core/src/test/scala/astraea/spark/rasterframes/ExtensionMethodSpec.scala b/core/src/test/scala/astraea/spark/rasterframes/ExtensionMethodSpec.scala
index 1834f1fa2..aabfb9498 100644
--- a/core/src/test/scala/astraea/spark/rasterframes/ExtensionMethodSpec.scala
+++ b/core/src/test/scala/astraea/spark/rasterframes/ExtensionMethodSpec.scala
@@ -19,6 +19,12 @@
*/
package astraea.spark.rasterframes
+import astraea.spark.rasterframes.util.SubdivideSupport._
+import geotrellis.proj4.LatLng
+import geotrellis.raster.{ByteCellType, GridBounds, TileLayout}
+import geotrellis.spark.{KeyBounds, SpatialKey, TileLayerMetadata}
+import geotrellis.spark.tiling.LayoutDefinition
+import geotrellis.spark.tiling.CRSWorldExtent
/**
* Tests miscellaneous extension methods.
@@ -47,4 +53,52 @@ class ExtensionMethodSpec extends TestEnvironment with TestData {
"val Some(col) = rf.spatialKeyColumn" shouldNot compile
}
}
+ describe("Miscellaneous extensions") {
+ it("should split TileLayout") {
+ val tl1 = TileLayout(2, 3, 10, 10)
+ assert(tl1.subdivide(0) === tl1)
+ assert(tl1.subdivide(1) === tl1)
+ assert(tl1.subdivide(2) === TileLayout(4, 6, 5, 5))
+ assertThrows[IllegalArgumentException](tl1.subdivide(-1))
+
+ }
+ it("should split KeyBounds[SpatialKey]") {
+ val grid = GridBounds(0, 0, 9, 9)
+ val kb = KeyBounds(grid)
+ val kb2 = kb.subdivide(2)
+ assert(kb2.get.toGridBounds() === GridBounds(0, 0, 19, 19))
+
+ val grid2 = GridBounds(2, 2, 9, 9)
+ val kb3 = KeyBounds(grid2)
+ val kb4 = kb3.subdivide(2)
+ assert(kb4.get.toGridBounds() === GridBounds(4, 4, 19, 19))
+ }
+
+ it("should split key") {
+ val s1 = SpatialKey(0, 0).subdivide(2)
+ assert(s1 === Seq(SpatialKey(0,0), SpatialKey(1,0), SpatialKey(0,1), SpatialKey(1,1)))
+
+ val s2 = SpatialKey(2, 3).subdivide(3)
+ assert(s2 === Seq(SpatialKey(6,9), SpatialKey(7,9), SpatialKey(8,9), SpatialKey(6,10), SpatialKey(7,10), SpatialKey(8,10), SpatialKey(6,11), SpatialKey(7,11), SpatialKey(8,11)))
+ }
+
+ it("should split TileLayerMetadata[SpatialKey]") {
+ val tileSize = 12
+ val dataGridSize = 2
+ val grid = GridBounds(2, 4, 10, 11)
+ val layout = LayoutDefinition(LatLng.worldExtent, TileLayout(dataGridSize, dataGridSize, tileSize, tileSize))
+ val tlm = TileLayerMetadata(ByteCellType, layout, LatLng.worldExtent, LatLng, KeyBounds(grid))
+
+ val divided = tlm.subdivide(2)
+
+ assert(divided.tileLayout.tileDimensions === (tileSize/2, tileSize/2))
+
+ println(tlm)
+
+ println(tlm.subdivide(2))
+ println(tlm.tileLayout.tileDimensions)
+ println(tlm.subdivide(2).tileLayout.tileDimensions)
+ }
+
+ }
}
diff --git a/core/src/test/scala/astraea/spark/rasterframes/GTSQLSpec.scala b/core/src/test/scala/astraea/spark/rasterframes/GTSQLSpec.scala
index 07f0d251f..3ceb0ac34 100644
--- a/core/src/test/scala/astraea/spark/rasterframes/GTSQLSpec.scala
+++ b/core/src/test/scala/astraea/spark/rasterframes/GTSQLSpec.scala
@@ -132,7 +132,7 @@ class GTSQLSpec extends TestEnvironment with TestData {
val datasets = Seq(
{
- val tiles = Array.fill[Tile](30)(randomTile(5, 5, "float32"))
+ val tiles = Array.fill[Tile](30)(randomTile(5, 5, FloatCellType))
tiles(1) = null
tiles(11) = null
tiles(29) = null
diff --git a/core/src/test/scala/astraea/spark/rasterframes/RasterFrameSpec.scala b/core/src/test/scala/astraea/spark/rasterframes/RasterFrameSpec.scala
index 18fd4b761..653af750e 100644
--- a/core/src/test/scala/astraea/spark/rasterframes/RasterFrameSpec.scala
+++ b/core/src/test/scala/astraea/spark/rasterframes/RasterFrameSpec.scala
@@ -5,16 +5,15 @@ package astraea.spark.rasterframes
import java.sql.Timestamp
import java.time.ZonedDateTime
+import astraea.spark.rasterframes.util._
import geotrellis.proj4.LatLng
-import geotrellis.raster.render.{ColorMap, ColorRamp, ColorRamps}
-import geotrellis.raster.{IntCellType, ProjectedRaster, Tile, TileFeature, TileLayout}
+import geotrellis.raster.render.{ColorMap, ColorRamp}
+import geotrellis.raster.{ProjectedRaster, Tile, TileFeature, TileLayout, UByteCellType}
import geotrellis.spark._
-import geotrellis.spark.io._
import geotrellis.spark.tiling._
import geotrellis.vector.{Extent, ProjectedExtent}
-import org.apache.spark.sql.{SQLContext, SparkSession}
import org.apache.spark.sql.functions._
-import astraea.spark.rasterframes.util._
+import org.apache.spark.sql.{SQLContext, SparkSession}
import scala.util.control.NonFatal
@@ -75,6 +74,10 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys
)
assert(rf.count() === 4)
+
+ val cols = tileLayerRDD.toRF("foo").columns
+ assert(!cols.contains("tile"))
+ assert(cols.contains("foo"))
}
it("should implicitly convert from spatiotemporal layer type") {
@@ -94,10 +97,13 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys
println(rf.schema.prettyJson)
throw ex
}
+ val cols = tileLayerRDD.toRF("foo").columns
+ assert(!cols.contains("tile"))
+ assert(cols.contains("foo"))
}
it("should implicitly convert layer of TileFeature") {
- val tile = TileFeature(randomTile(20, 20, "uint8"), (1, "b", 3.0))
+ val tile = TileFeature(randomTile(20, 20, UByteCellType), (1, "b", 3.0))
val tileLayout = TileLayout(1, 1, 20, 20)
@@ -116,7 +122,7 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys
}
it("should implicitly convert spatiotemporal layer of TileFeature") {
- val tile = TileFeature(randomTile(20, 20, "uint8"), (1, "b", 3.0))
+ val tile = TileFeature(randomTile(20, 20, UByteCellType), (1, "b", 3.0))
val tileLayout = TileLayout(1, 1, 20, 20)
@@ -154,7 +160,6 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys
assert(wt.columns.contains(TEMPORAL_KEY_COLUMN.columnName))
val joined = wt.spatialJoin(wt, "outer")
- joined.printSchema
// Should be both left and right column names.
assert(joined.columns.count(_.contains(TEMPORAL_KEY_COLUMN.columnName)) === 2)
@@ -168,14 +173,13 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys
val right = left.withColumnRenamed(left.tileColumns.head.columnName, "rightTile")
.asRF
- val joined = left.spatialJoin(right, "inner")
- joined.printSchema
+ val joined = left.spatialJoin(right)
+ // since right is a copy of left, should not drop any rows with inner join
+ assert(joined.count === left.count)
// Should use left's key column names
assert(joined.spatialKeyColumn.columnName === left.spatialKeyColumn.columnName)
assert(joined.temporalKeyColumn.map(_.columnName) === left.temporalKeyColumn.map(_.columnName))
- // since right is a copy of left, should not drop any rows with inner join
- assert(joined.count === left.count)
}
@@ -206,22 +210,22 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys
assert(bounds._2 === SpaceTimeKey(3, 1, now))
}
- it("should clip TileLayerMetadata extent") {
- val tiled = sampleTileLayerRDD
-
- val rf = tiled.toRF
-
- val worldish = Extent(-179, -89, 179, 89)
- val areaish = Extent(-90, 30, -81, 39)
-
- val orig = tiled.metadata.extent
- assert(orig.contains(worldish))
- assert(orig.contains(areaish))
-
- val clipped = rf.clipLayerExtent.tileLayerMetadata.widen.extent
- assert(!clipped.contains(worldish))
- assert(clipped.contains(areaish))
- }
+// it("should clip TileLayerMetadata extent") {
+// val tiled = sampleTileLayerRDD
+//
+// val rf = tiled.reproject(LatLng, tiled.metadata.layout)._2.toRF
+//
+// val worldish = Extent(-179, -89, 179, 89)
+// val areaish = Extent(-90, 30, -81, 40)
+//
+// val orig = rf.tileLayerMetadata.widen.extent
+// assert(worldish.contains(orig))
+// assert(areaish.contains(orig))
+//
+// val clipped = rf.clipLayerExtent.tileLayerMetadata.widen.extent
+// assert(!clipped.contains(worldish))
+// assert(clipped.contains(areaish))
+// }
def basicallySame(expected: Extent, computed: Extent): Unit = {
val components = Seq(
@@ -306,7 +310,6 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys
val green = TestData.naipSample(2).projectedRaster.toRF.withRFColumnRenamed("tile", "green")
val blue = TestData.naipSample(3).projectedRaster.toRF.withRFColumnRenamed("tile", "blue")
val joined = blue.spatialJoin(green).spatialJoin(red)
- joined.printSchema
noException shouldBe thrownBy {
val raster = joined.toMultibandRaster(Seq($"red", $"green", $"blue"), 256, 256)
diff --git a/core/src/test/scala/astraea/spark/rasterframes/TestData.scala b/core/src/test/scala/astraea/spark/rasterframes/TestData.scala
index ade97090b..175691a10 100644
--- a/core/src/test/scala/astraea/spark/rasterframes/TestData.scala
+++ b/core/src/test/scala/astraea/spark/rasterframes/TestData.scala
@@ -108,12 +108,8 @@ trait TestData {
}
def sampleTileLayerRDD(implicit spark: SparkSession): TileLayerRDD[SpatialKey] = {
- val raster = sampleGeoTiff.projectedRaster.reproject(LatLng)
- val layout = LayoutDefinition(LatLng.worldExtent, TileLayout(36, 18, 128, 128))
- val kb = KeyBounds(SpatialKey(0, 0), SpatialKey(layout.layoutCols, layout.layoutRows))
- val tlm = TileLayerMetadata(raster.tile.cellType, layout, layout.extent, LatLng, kb)
- val rdd = spark.sparkContext.makeRDD(Seq((raster.projectedExtent, raster.tile)))
- ContextRDD(rdd.tileToLayout(tlm), tlm)
+ val rf = sampleGeoTiff.projectedRaster.toRF(128, 128)
+ rf.toTileLayerRDD(rf.tileColumns.head).left.get
}
object JTS {
@@ -136,48 +132,59 @@ object TestData extends TestData {
val rnd = new scala.util.Random(42)
/** Construct a tile of given size and cell type populated with random values. */
- def randomTile(cols: Int, rows: Int, cellTypeName: String): Tile = {
- val cellType = CellType.fromName(cellTypeName)
- val tile = ArrayTile.alloc(cellType, cols, rows)
-
- def possibleND(c: Int) =
- c == NODATA || c == byteNODATA || c == ubyteNODATA || c == shortNODATA || c == ushortNODATA
-
+ def randomTile(cols: Int, rows: Int, cellType: CellType): Tile = {
// Initialize tile with some initial random values
- var result = tile.dualMap(_ ⇒ rnd.nextInt())(_ ⇒ rnd.nextGaussian())
-
- // Due to cell width narrowing and custom NoData values, we can end up randomly creating
- // NoData values. While perhaps inefficient, the safest way to ensure a tile with no-NoData values
- // with the current CellType API (GT 1.1), while still generating random data is to
- // iteratively pass through all the cells and replace NoData values as we find them.
- do {
- result = result.dualMap(
- z ⇒ if (isNoData(z)) rnd.nextInt() else z
- ) (
- z ⇒ if (isNoData(z)) rnd.nextGaussian() else z
- )
- } while (F.noDataCells(result) != 0L)
+ val base: Tile = cellType match {
+ case _: FloatCells ⇒
+ val data = Array.fill(cols * rows)(rnd.nextGaussian().toFloat)
+ ArrayTile(data, cols, rows)
+ case _: DoubleCells ⇒
+ val data = Array.fill(cols * rows)(rnd.nextGaussian().toFloat)
+ ArrayTile(data, cols, rows)
+ case _ ⇒
+ val words = cellType.bits / 8
+ val bytes = Array.ofDim[Byte](cols * rows * words)
+ rnd.nextBytes(bytes)
+ ArrayTile.fromBytes(bytes, cellType, cols, rows)
+ }
- assert(F.noDataCells(result) == 0L,
- s"Should not have any NoData cells for $cellTypeName:\n${result.asciiDraw()}")
- result
+ cellType match {
+ case _: NoNoData ⇒ base
+ case _ ⇒
+ // Due to cell width narrowing and custom NoData values, we can end up randomly creating
+ // NoData values. While perhaps inefficient, the safest way to ensure a tile with no-NoData values
+ // with the current CellType API (GT 1.1), while still generating random data is to
+ // iteratively pass through all the cells and replace NoData values as we find them.
+ var result = base
+ do {
+ result = result.dualMap(
+ z ⇒ if (isNoData(z)) rnd.nextInt(1 << cellType.bits) else z
+ ) (
+ z ⇒ if (isNoData(z)) rnd.nextGaussian() else z
+ )
+ } while (F.noDataCells(result) != 0L)
+
+ assert(F.noDataCells(result) == 0L,
+ s"Should not have any NoData cells for $cellType:\n${result.asciiDraw()}")
+ result
+ }
}
/** Create a series of random tiles. */
val makeTiles: (Int) ⇒ Array[Tile] = (count) ⇒
- Array.fill(count)(randomTile(4, 4, "int8raw"))
+ Array.fill(count)(randomTile(4, 4, UByteCellType))
def randomSpatialTileLayerRDD(
rasterCols: Int, rasterRows: Int,
layoutCols: Int, layoutRows: Int)(implicit sc: SparkContext): TileLayerRDD[SpatialKey] = {
- val tile = randomTile(rasterCols, rasterRows, "uint8")
+ val tile = randomTile(rasterCols, rasterRows, UByteCellType)
TileLayerRDDBuilders.createTileLayerRDD(tile, layoutCols, layoutRows, LatLng)._2
}
def randomSpatioTemporalTileLayerRDD(
rasterCols: Int, rasterRows: Int,
layoutCols: Int, layoutRows: Int)(implicit sc: SparkContext): TileLayerRDD[SpaceTimeKey] = {
- val tile = randomTile(rasterCols, rasterRows, "uint8")
+ val tile = randomTile(rasterCols, rasterRows, UByteCellType)
val tileLayout = TileLayout(layoutCols, layoutRows, rasterCols/layoutCols, rasterRows/layoutRows)
TileLayerRDDBuilders.createSpaceTimeTileLayerRDD(Seq((tile, ZonedDateTime.now())), tileLayout, tile.cellType)
}
diff --git a/core/src/test/scala/astraea/spark/rasterframes/TileStatsSpec.scala b/core/src/test/scala/astraea/spark/rasterframes/TileStatsSpec.scala
index 70cc581ee..ad3c7038c 100644
--- a/core/src/test/scala/astraea/spark/rasterframes/TileStatsSpec.scala
+++ b/core/src/test/scala/astraea/spark/rasterframes/TileStatsSpec.scala
@@ -20,8 +20,8 @@
package astraea.spark.rasterframes
import astraea.spark.rasterframes.TestData.randomTile
+import astraea.spark.rasterframes.stats.CellHistogram
import geotrellis.raster._
-import geotrellis.raster.histogram.Histogram
import geotrellis.raster.mapalgebra.local.{Max, Min}
import org.apache.spark.sql.functions._
@@ -84,7 +84,7 @@ class TileStatsSpec extends TestEnvironment with TestData {
}
it("should count data and no-data cells") {
- val ds = (Seq.fill[Tile](10)(injectND(10)(randomTile(10, 10, "uint8"))) :+ null).toDF("tile")
+ val ds = (Seq.fill[Tile](10)(injectND(10)(randomTile(10, 10, UByteConstantNoDataCellType))) :+ null).toDF("tile")
val expectedNoData = 10 * 10
val expectedData = 10 * 10 * 10 - expectedNoData
@@ -103,8 +103,7 @@ class TileStatsSpec extends TestEnvironment with TestData {
}
it("should compute tile statistics") {
- val ds = (Seq.fill[Tile](3)(randomTile(5, 5, "float32")) :+ null).toDS()
- ds.printSchema()
+ val ds = (Seq.fill[Tile](3)(randomTile(5, 5, FloatConstantNoDataCellType)) :+ null).toDS()
val means1 = ds.select(tileStats($"value")).map(s ⇒ Option(s).map(_.mean).getOrElse(0.0)).collect
val means2 = ds.select(tileMean($"value")).collect.map(m ⇒ if (m.isNaN) 0.0 else m)
// Compute the mean manually, knowing we're not dealing with no-data values.
@@ -115,49 +114,51 @@ class TileStatsSpec extends TestEnvironment with TestData {
}
it("should compute per-tile histogram") {
- val ds = (Seq.fill[Tile](3)(randomTile(5, 5, "float32")) :+ null).toDF("tiles")
+ val ds = Seq.fill[Tile](3)(randomTile(5, 5, FloatCellType)).toDF("tiles")
ds.createOrReplaceTempView("tmp")
- val r1 = ds.select(tileHistogram($"tiles").as[Histogram[Double]])
- assert(r1.first.totalCount() === 5 * 5)
+ val r1 = ds.select(tileHistogram($"tiles"))
+ assert(r1.first.totalCount === 5 * 5)
write(r1)
- val r2 = sql("select rf_tileHistogram(tiles) from tmp")
+ val r2 = sql("select hist.* from (select rf_tileHistogram(tiles) as hist from tmp)").as[CellHistogram]
write(r2)
-
- assert(r1.first.mean === r2.as[Histogram[Double]].first.mean)
+ assert(r1.first.mean === r2.first.mean)
}
it("should compute aggregate histogram") {
- val ds = Seq.fill[Tile](10)(randomTile(5, 5, "float32")).toDF("tiles")
+ val ds = Seq.fill[Tile](10)(randomTile(5, 5, FloatConstantNoDataCellType)).toDF("tiles")
ds.createOrReplaceTempView("tmp")
- val agg = ds.select(aggHistogram($"tiles")).as[Histogram[Double]]
+ val agg = ds.select(aggHistogram($"tiles")).as[CellHistogram]
val hist = agg.collect()
assert(hist.length === 1)
- val stats = agg.map(_.statistics().get).as("stats")
+ val stats = agg.map(_.stats).as("stats")
//stats.select("stats.*").show(false)
assert(stats.first().stddev === 1.0 +- 0.3) // <-- playing with statistical fire :)
- val hist2 = sql("select rf_aggHistogram(tiles) as hist from tmp").as[Histogram[Double]]
+ val hist2 = sql("select hist.* from (select rf_aggHistogram(tiles) as hist from tmp)").as[CellHistogram]
- assert(hist2.first.totalCount() === 250)
+ assert(hist2.first.totalCount === 250)
}
it("should compute aggregate mean") {
- val ds = (Seq.fill[Tile](10)(randomTile(5, 5, "float32")) :+ null).toDF("tiles")
+ val ds = (Seq.fill[Tile](10)(randomTile(5, 5, FloatCellType)) :+ null).toDF("tiles")
val agg = ds.select(aggMean($"tiles"))
val stats = ds.select(aggStats($"tiles") as "stats").select($"stats.mean".as[Double])
assert(agg.first() === stats.first())
}
it("should compute aggregate statistics") {
- val ds = Seq.fill[Tile](10)(randomTile(5, 5, "float32")).toDF("tiles")
+ val ds = Seq.fill[Tile](10)(randomTile(5, 5, FloatConstantNoDataCellType)).toDF("tiles")
val exploded = ds.select(explodeTiles($"tiles"))
val (mean, vrnc) = exploded.agg(avg($"tiles"), var_pop($"tiles")).as[(Double, Double)].first
val stats = ds.select(aggStats($"tiles") as "stats")///.as[(Long, Double, Double, Double, Double)]
- ds.select(aggStats($"tiles")).show(false)
+
+ noException shouldBe thrownBy {
+ ds.select(aggStats($"tiles")).collect()
+ }
val agg = stats.select($"stats.variance".as[Double])
@@ -174,7 +175,7 @@ class TileStatsSpec extends TestEnvironment with TestData {
it("should compute aggregate local stats") {
val ave = (nums: Array[Double]) ⇒ nums.sum / nums.length
- val ds = (Seq.fill[Tile](30)(randomTile(5, 5, "float32"))
+ val ds = (Seq.fill[Tile](30)(randomTile(5, 5, FloatConstantNoDataCellType))
.map(injectND(2)) :+ null).toDF("tiles")
ds.createOrReplaceTempView("tmp")
@@ -238,7 +239,7 @@ class TileStatsSpec extends TestEnvironment with TestData {
val tsize = 5
val count = 20
val nds = 2
- val tiles = (Seq.fill[Tile](count)(randomTile(tsize, tsize, "uint8ud255"))
+ val tiles = (Seq.fill[Tile](count)(randomTile(tsize, tsize, UByteUserDefinedNoDataCellType(255.toByte)))
.map(injectND(nds)) :+ null).toDF("tiles")
val counts = tiles.select(noDataCells($"tiles")).collect().dropRight(1)
diff --git a/core/src/test/scala/astraea/spark/rasterframes/TileUDTSpec.scala b/core/src/test/scala/astraea/spark/rasterframes/TileUDTSpec.scala
index 39ede1700..0001c4509 100644
--- a/core/src/test/scala/astraea/spark/rasterframes/TileUDTSpec.scala
+++ b/core/src/test/scala/astraea/spark/rasterframes/TileUDTSpec.scala
@@ -21,7 +21,7 @@
package astraea.spark.rasterframes
-import geotrellis.raster.Tile
+import geotrellis.raster.{CellType, Tile}
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.gt.types.TileUDT
import org.apache.spark.sql.rf.InternalRowTile
@@ -46,7 +46,7 @@ class TileUDTSpec extends TestEnvironment with TestData with Inspectors with Int
def forEveryConfig(test: (Tile) ⇒ Unit): Unit = {
forEvery(tileSizes.combinations(2).toSeq) { case Seq(cols, rows) ⇒
forEvery(ct) { c ⇒
- val tile = randomTile(cols, rows, c)
+ val tile = randomTile(cols, rows, CellType.fromName(c))
test(tile)
}
}
diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/DefaultSource.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/DefaultSource.scala
index 31b75f4bc..2aeed5335 100644
--- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/DefaultSource.scala
+++ b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/DefaultSource.scala
@@ -19,25 +19,63 @@
package astraea.spark.rasterframes.datasource.geotiff
-import java.net.URI
-
-import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.sql.sources.{DataSourceRegister, RelationProvider}
+import _root_.geotrellis.raster.io.geotiff.GeoTiff
import astraea.spark.rasterframes._
+import astraea.spark.rasterframes.datasource._
+import astraea.spark.rasterframes.util._
+import com.typesafe.scalalogging.LazyLogging
+import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, DataSourceRegister, RelationProvider}
+import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode}
/**
- *
+ * Spark SQL data source over GeoTIFF files.
* @since 1/14/18
*/
-@Experimental
-class DefaultSource extends DataSourceRegister with RelationProvider {
- def shortName() = "geotiff"
+class DefaultSource extends DataSourceRegister with RelationProvider with CreatableRelationProvider with LazyLogging {
+ def shortName() = DefaultSource.SHORT_NAME
+
+ def path(parameters: Map[String, String]) =
+ uriParam(DefaultSource.PATH_PARAM, parameters)
def createRelation(sqlContext: SQLContext, parameters: Map[String, String]) = {
- require(parameters.contains("path"), "'path' parameter required.")
- val uri: URI = URI.create(parameters("path"))
+ val pathO = path(parameters)
+ require(pathO.isDefined, "Valid URI 'path' parameter required.")
+ sqlContext.withRasterFrames
+ GeoTiffRelation(sqlContext, pathO.get)
+ }
+
+ override def createRelation(sqlContext: SQLContext, mode: SaveMode, parameters: Map[String, String], data: DataFrame): BaseRelation = {
+ val pathO = path(parameters)
+ require(pathO.isDefined, "Valid URI 'path' parameter required.")
+ require(pathO.get.getScheme == "file" || pathO.get.getScheme == null, "Currently only 'file://' destinations are supported")
sqlContext.withRasterFrames
- GeoTiffRelation(sqlContext, uri)
+
+
+ require(data.isRF, "GeoTIFF can only be constructed from a RasterFrame")
+ val rf = data.certify
+
+ val tl = rf.tileLayerMetadata.widen.layout.tileLayout
+
+ val cols = numParam(DefaultSource.IMAGE_WIDTH_PARAM, parameters).getOrElse(tl.totalCols)
+ val rows = numParam(DefaultSource.IMAGE_HEIGHT_PARAM, parameters).getOrElse(tl.totalRows)
+
+ require(cols <= Int.MaxValue && rows <= Int.MaxValue, s"Can't construct a GeoTIFF of size $cols x $rows. (Too big!)")
+
+ // Should we really play traffic cop here?
+ if(cols.toDouble * rows * 64.0 > Runtime.getRuntime.totalMemory() * 0.5)
+ logger.warn(s"You've asked for the construction of a very large image ($cols x $rows), destined for ${pathO.get}. Out of memory error likely.")
+
+ println()
+ val raster = rf.toMultibandRaster(rf.tileColumns, cols.toInt, rows.toInt)
+
+ GeoTiff(raster).write(pathO.get.getPath)
+ GeoTiffRelation(sqlContext, pathO.get)
}
}
+
+object DefaultSource {
+ final val SHORT_NAME = "geotiff"
+ final val PATH_PARAM = "path"
+ final val IMAGE_WIDTH_PARAM = "imageWidth"
+ final val IMAGE_HEIGHT_PARAM = "imageWidth"
+}
diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffRelation.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffRelation.scala
index 1d14b7ec8..9f54cccbc 100644
--- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffRelation.scala
+++ b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffRelation.scala
@@ -22,13 +22,14 @@ package astraea.spark.rasterframes.datasource.geotiff
import java.net.URI
import astraea.spark.rasterframes._
+import astraea.spark.rasterframes.util._
+import geotrellis.raster.TileLayout
import geotrellis.raster.io.geotiff.reader.GeoTiffReader
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.io.hadoop._
import geotrellis.spark.tiling.LayoutDefinition
import geotrellis.util._
-import geotrellis.vector.Extent
import org.apache.hadoop.fs.Path
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
@@ -36,9 +37,6 @@ import org.apache.spark.sql.gt.types.TileUDT
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Row, SQLContext}
-import astraea.spark.rasterframes.util._
-import geotrellis.raster.TileLayout
-import geotrellis.raster.io.geotiff.MultibandGeoTiff
/**
* Spark SQL data source over a single GeoTiff file. Works best with CoG compliant ones.
diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/package.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/package.scala
index 8e46f4641..c47a6d742 100644
--- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/package.scala
+++ b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotiff/package.scala
@@ -22,25 +22,34 @@ package astraea.spark.rasterframes.datasource
import java.net.URI
import astraea.spark.rasterframes._
-import org.apache.spark.sql.DataFrameReader
+import org.apache.spark.sql.{DataFrameReader, DataFrameWriter}
import shapeless.tag
import shapeless.tag.@@
/**
+ * Extension methods enabled by this module.
*
* @since 1/16/18
*/
package object geotiff {
-
/** Tagged type construction for enabling type-safe extension methods for loading
* a RasterFrame in expected form. */
type GeoTiffRasterFrameReader = DataFrameReader @@ GeoTiffRasterFrameReaderTag
trait GeoTiffRasterFrameReaderTag
+ /** Tagged type construction for enabling type-safe extension methods for writing
+ * a RasterFrame to a GeoTIFF. */
+ type GeoTiffRasterFrameWriter[T] = DataFrameWriter[T] @@ GeoTiffRasterFrameWriterTag
+ trait GeoTiffRasterFrameWriterTag
/** Adds `geotiff` format specifier to `DataFrameReader`. */
implicit class DataFrameReaderHasGeoTiffFormat(val reader: DataFrameReader) {
def geotiff: GeoTiffRasterFrameReader =
- tag[GeoTiffRasterFrameReaderTag][DataFrameReader](reader.format("geotiff"))
+ tag[GeoTiffRasterFrameReaderTag][DataFrameReader](reader.format(DefaultSource.SHORT_NAME))
+ }
+
+ implicit class DataFrameWriterHasGeoTiffFormat[T](val writer: DataFrameWriter[T]) {
+ def geotiff: GeoTiffRasterFrameWriter[T] =
+ tag[GeoTiffRasterFrameWriterTag][DataFrameWriter[T]](writer.format(DefaultSource.SHORT_NAME))
}
/** Adds `loadRF` to appropriately tagged `DataFrameReader` */
diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/DefaultSource.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/DefaultSource.scala
index 92137ecb3..47d589621 100644
--- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/DefaultSource.scala
+++ b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/DefaultSource.scala
@@ -23,6 +23,7 @@ import java.net.URI
import astraea.spark.rasterframes.rules.registerOptimization
import astraea.spark.rasterframes._
+import astraea.spark.rasterframes.datasource.geotrellis.DefaultSource._
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.io.index.ZCurveKeyIndexMethod
@@ -38,7 +39,7 @@ import scala.util.Try
*/
@Experimental
class DefaultSource extends DataSourceRegister with RelationProvider with CreatableRelationProvider {
- def shortName(): String = "geotrellis"
+ def shortName(): String = DefaultSource.SHORT_NAME
/**
* Create a GeoTrellis data source.
@@ -48,30 +49,33 @@ class DefaultSource extends DataSourceRegister with RelationProvider with Creata
* `layer`-layer name (e.g. "LC08_L1GT");
* `zoom`-positive integer zoom level (e.g. "8");
* `numPartitions`-(optional) integer specifying initial number of partitions;
+ * `tileSubdivisions`-(optional) positive integer defining how many division horizontally and vertically should be applied to a tile;
* `failOnUnrecognizedFilter`-(optional) if true, predicate push-down filters not translated into GeoTrellis query syntax are fatal.
*/
def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = {
require(parameters.contains("path"), "'path' parameter required.")
- require(parameters.contains("layer"), "'layer' parameter for raster layer name required.")
- require(parameters.contains("zoom"), "'zoom' parameter for raster layer zoom level required.")
+ require(parameters.contains(LAYER_PARAM), "'layer' parameter for raster layer name required.")
+ require(parameters.contains(ZOOM_PARAM), "'zoom' parameter for raster layer zoom level required.")
sqlContext.withRasterFrames
registerOptimization(sqlContext, SpatialFilterPushdownRules)
val uri: URI = URI.create(parameters("path"))
- val layerId: LayerId = LayerId(parameters("layer"), parameters("zoom").toInt)
- val numPartitions = parameters.get("numPartitions").map(_.toInt)
+ val layerId: LayerId = LayerId(parameters(LAYER_PARAM), parameters(ZOOM_PARAM).toInt)
+ val numPartitions = parameters.get(NUM_PARTITIONS_PARAM).map(_.toInt)
+ val tileSubdivisions = parameters.get(TILE_SUBDIVISIONS_PARAM).map(_.toInt)
+ tileSubdivisions.foreach(s ⇒ require(s >= 0, TILE_SUBDIVISIONS_PARAM + " must be a postive integer"))
val failOnUnrecognizedFilter = parameters.get("failOnUnrecognizedFilter").exists(_.toBoolean)
- GeoTrellisRelation(sqlContext, uri, layerId, numPartitions, failOnUnrecognizedFilter)
+ GeoTrellisRelation(sqlContext, uri, layerId, numPartitions, failOnUnrecognizedFilter, tileSubdivisions)
}
/** Write relation. */
def createRelation(sqlContext: SQLContext, mode: SaveMode, parameters: Map[String, String], data: DataFrame): BaseRelation = {
- val zoom = parameters.get("zoom").flatMap(p ⇒ Try(p.toInt).toOption)
+ val zoom = parameters.get(ZOOM_PARAM).flatMap(p ⇒ Try(p.toInt).toOption)
val path = parameters.get("path").flatMap(p ⇒ Try(new URI(p)).toOption)
- val layerName = parameters.get("layer")
+ val layerName = parameters.get(LAYER_PARAM)
require(path.isDefined, "Valid URI 'path' parameter required.")
require(layerName.isDefined, "'layer' parameter for raster layer name required.")
@@ -107,3 +111,11 @@ class DefaultSource extends DataSourceRegister with RelationProvider with Creata
createRelation(sqlContext, parameters)
}
}
+
+object DefaultSource {
+ final val SHORT_NAME = "geotrellis"
+ final val TILE_SUBDIVISIONS_PARAM = "tileSubdivisions"
+ final val NUM_PARTITIONS_PARAM = "numPartitions"
+ final val LAYER_PARAM = "layer"
+ final val ZOOM_PARAM = "zoom"
+}
diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala
index 95599f3c1..ec0ae40d4 100644
--- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala
+++ b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisRelation.scala
@@ -29,13 +29,14 @@ import astraea.spark.rasterframes.jts.SpatialFilters.{BetweenTimes, Contains ⇒
import astraea.spark.rasterframes.datasource.geotrellis.GeoTrellisRelation.TileFeatureData
import astraea.spark.rasterframes.util._
import com.vividsolutions.jts.geom
-import geotrellis.raster.{MultibandTile, Tile, TileFeature}
+import geotrellis.raster.{CellGrid, MultibandTile, Tile, TileFeature}
import geotrellis.spark.io._
import geotrellis.spark.io.avro.AvroRecordCodec
import geotrellis.spark.util.KryoWrapper
import geotrellis.spark.{LayerId, Metadata, SpatialKey, TileLayerMetadata, _}
import geotrellis.util.LazyLogging
import geotrellis.vector._
+import geotrellis.util._
import org.apache.avro.Schema
import org.apache.avro.generic.GenericRecord
import org.apache.spark.rdd.RDD
@@ -46,6 +47,9 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.{Row, SQLContext, sources}
import spray.json.DefaultJsonProtocol._
import spray.json.JsValue
+import TileFeatureSupport._
+import geotrellis.spark.tiling.{CutTiles, TilerKeyMethods}
+import SubdivideSupport._
import scala.reflect.ClassTag
import scala.reflect.runtime.universe._
@@ -54,12 +58,13 @@ import scala.reflect.runtime.universe._
* A Spark SQL `Relation` over a standard GeoTrellis layer.
*/
case class GeoTrellisRelation(sqlContext: SQLContext,
- uri: URI,
- layerId: LayerId,
- numPartitions: Option[Int] = None,
- failOnUnrecognizedFilter: Boolean = false,
- filters: Seq[Filter] = Seq.empty)
- extends BaseRelation with PrunedScan with LazyLogging {
+ uri: URI,
+ layerId: LayerId,
+ numPartitions: Option[Int] = None,
+ failOnUnrecognizedFilter: Boolean = false,
+ tileSubdivisions: Option[Int] = None,
+ filters: Seq[Filter] = Seq.empty)
+ extends BaseRelation with PrunedScan with LazyLogging {
implicit val sc = sqlContext.sparkContext
@@ -112,6 +117,15 @@ case class GeoTrellisRelation(sqlContext: SQLContext,
)
}
+ def subdividedTileLayerMetadata: Either[TileLayerMetadata[SpatialKey], TileLayerMetadata[SpaceTimeKey]] = {
+ tileSubdivisions.filter(_ > 1) match {
+ case None ⇒ tileLayerMetadata
+ case Some(divs) ⇒ tileLayerMetadata
+ .right.map(_.subdivide(divs))
+ .left.map(_.subdivide(divs))
+ }
+ }
+
private object Cols {
lazy val SK = SPATIAL_KEY_COLUMN.columnName
lazy val TK = TEMPORAL_KEY_COLUMN.columnName
@@ -121,7 +135,7 @@ case class GeoTrellisRelation(sqlContext: SQLContext,
lazy val EX = BOUNDS_COLUMN.columnName
}
- /** This unfortunate routine is here because the number bands in a multiband layer isn't written
+ /** This unfortunate routine is here because the number bands in a multiband layer isn't written
* in the metadata anywhere. This is potentially an expensive hack, which needs further quantifying of impact.
* Another option is to force the user to specify the number of bands. */
private lazy val peekBandCount = {
@@ -143,8 +157,8 @@ case class GeoTrellisRelation(sqlContext: SQLContext,
override def schema: StructType = {
val skSchema = ExpressionEncoder[SpatialKey]().schema
- val skMetadata = attributes.readMetadata[JsValue](layerId) |>
- (m ⇒ Metadata.fromJson(m.compactPrint)) |>
+ val skMetadata = subdividedTileLayerMetadata.
+ fold(_.asColumnMetadata, _.asColumnMetadata) |>
(Metadata.empty.append.attachContext(_).tagSpatialKey.build)
val keyFields = keyType match {
@@ -244,11 +258,16 @@ case class GeoTrellisRelation(sqlContext: SQLContext,
}
}
- def query[T: AvroRecordCodec: ClassTag](reader: FilteringLayerReader[LayerId], columnIndexes: Seq[Int]) = {
- tileLayerMetadata.fold(
+ private def subdivider[K: SpatialComponent, T <: CellGrid: WithCropMethods](divs: Int) = (p: (K, T)) ⇒ {
+ val newKeys = p._1.subdivide(divs)
+ val newTiles = p._2.subdivide(divs)
+ newKeys.zip(newTiles)
+ }
+
+ private def query[T <: CellGrid: WithCropMethods: WithMergeMethods: AvroRecordCodec: ClassTag](reader: FilteringLayerReader[LayerId], columnIndexes: Seq[Int]) = {
+ subdividedTileLayerMetadata.fold(
// Without temporal key case
(tlm: TileLayerMetadata[SpatialKey]) ⇒ {
- val trans = tlm.mapTransform
val parts = numPartitions.getOrElse(reader.defaultNumPartitions)
@@ -256,11 +275,15 @@ case class GeoTrellisRelation(sqlContext: SQLContext,
reader.query[SpatialKey, T, TileLayerMetadata[SpatialKey]](layerId, parts)
)(applyFilter(_, _))
- val rdd = query.result
+ val rdd = tileSubdivisions.filter(_ > 1) match {
+ case Some(divs) ⇒
+ query.result.flatMap(subdivider[SpatialKey, T](divs))
+ case None ⇒ query.result
+ }
+ val trans = tlm.mapTransform
rdd
.map { case (sk: SpatialKey, tile: T) ⇒
-
val entries = columnIndexes.map {
case 0 ⇒ sk
case 1 ⇒ trans.keyToExtent(sk).jtsGeom
@@ -286,7 +309,11 @@ case class GeoTrellisRelation(sqlContext: SQLContext,
reader.query[SpaceTimeKey, T, TileLayerMetadata[SpaceTimeKey]](layerId, parts)
)(applyFilterTemporal(_, _))
- val rdd = query.result
+ val rdd = tileSubdivisions.filter(_ > 1) match {
+ case Some(divs) ⇒
+ query.result.flatMap(subdivider[SpaceTimeKey, T](divs))
+ case None ⇒ query.result
+ }
rdd
.map { case (stk: SpaceTimeKey, tile: T) ⇒
diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupport.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupport.scala
index a34dbe139..caca1707e 100644
--- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupport.scala
+++ b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupport.scala
@@ -28,16 +28,12 @@ import geotrellis.raster.resample.ResampleMethod
import geotrellis.raster.{CellGrid, CellType, GridBounds, TileFeature}
import geotrellis.util.MethodExtensions
import geotrellis.vector.{Extent, Geometry}
+import astraea.spark.rasterframes.util._
import scala.reflect.ClassTag
trait TileFeatureSupport {
- type WithMergeMethods[V] = (V => TileMergeMethods[V])
- type WithPrototypeMethods[V <: CellGrid] = (V => TilePrototypeMethods[V])
- type WithCropMethods[V <: CellGrid] = (V => TileCropMethods[V])
- type WithMaskMethods[V] = (V => TileMaskMethods[V])
-
implicit class TileFeatureMethodsWrapper[V <: CellGrid: ClassTag: WithMergeMethods: WithPrototypeMethods: WithCropMethods: WithMaskMethods, D: MergeableData](val self: TileFeature[V, D])
extends TileMergeMethods[TileFeature[V, D]]
with TilePrototypeMethods[TileFeature[V,D]]
diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/package.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/package.scala
index 5d3e6fb98..0699c3b08 100644
--- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/package.scala
+++ b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/geotrellis/package.scala
@@ -21,6 +21,7 @@ package astraea.spark.rasterframes.datasource
import java.net.URI
import _root_.geotrellis.spark.LayerId
+import astraea.spark.rasterframes.datasource.geotrellis.DefaultSource._
import astraea.spark.rasterframes.{RasterFrame, _}
import org.apache.spark.sql._
import org.apache.spark.sql.functions.col
@@ -28,7 +29,7 @@ import shapeless.tag
import shapeless.tag.@@
/**
- * Module utilities.
+ * Extension methods for literate and type-safe loading of geotrellis layers.
*
* @since 1/12/18
*/
@@ -55,35 +56,44 @@ package object geotrellis {
reader.format("geotrellis-catalog").load(base.toASCIIString)
def geotrellis: GeoTrellisRasterFrameReader =
- tag[GeoTrellisRasterFrameReaderTag][DataFrameReader](reader.format("geotrellis"))
+ tag[GeoTrellisRasterFrameReaderTag][DataFrameReader](reader.format(SHORT_NAME))
}
implicit class DataFrameWriterHasGeotrellisFormat[T](val writer: DataFrameWriter[T]) {
def geotrellis: GeoTrellisRasterFrameWriter[T] =
- tag[GeoTrellisRasterFrameWriterTag][DataFrameWriter[T]](writer.format("geotrellis"))
+ tag[GeoTrellisRasterFrameWriterTag][DataFrameWriter[T]](writer.format(SHORT_NAME))
}
implicit class GeoTrellisWriterAddLayer[T](val writer: GeoTrellisRasterFrameWriter[T]) {
def asLayer(id: LayerId): DataFrameWriter[T] =
writer
- .option("layer", id.name)
- .option("zoom", id.zoom.toString)
+ .option(LAYER_PARAM, id.name)
+ .option(ZOOM_PARAM, id.zoom.toString)
def asLayer(layer: Layer): DataFrameWriter[T] =
asLayer(layer.id)
.option("path", layer.base.toASCIIString)
}
- /** Extension methods for loading a RasterFrame from a tagged `DataFrameReader`. */
+ /** Extension methods for loading a RasterFrame from a tagged `DataFrameReader`. */
implicit class GeoTrellisReaderWithRF(val reader: GeoTrellisRasterFrameReader) {
+ def withTileSubdivisions(divs: Int): GeoTrellisRasterFrameReader =
+ tag[GeoTrellisRasterFrameReaderTag][DataFrameReader](
+ reader.option(DefaultSource.TILE_SUBDIVISIONS_PARAM, divs)
+ )
+
+ def withNumPartitions(partitions: Int): GeoTrellisRasterFrameReader =
+ tag[GeoTrellisRasterFrameReaderTag][DataFrameReader](
+ reader.option(DefaultSource.NUM_PARTITIONS_PARAM, partitions)
+ )
+
def loadRF(uri: URI, id: LayerId): RasterFrame =
reader
- .option("layer", id.name)
- .option("zoom", id.zoom.toString)
+ .option(LAYER_PARAM, id.name)
+ .option(ZOOM_PARAM, id.zoom.toString)
.load(uri.toASCIIString)
.asRF
def loadRF(layer: Layer): RasterFrame = loadRF(layer.base, layer.id)
}
-
}
diff --git a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/package.scala b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/package.scala
index 4dc603d0e..8f1852e25 100644
--- a/datasource/src/main/scala/astraea/spark/rasterframes/datasource/package.scala
+++ b/datasource/src/main/scala/astraea/spark/rasterframes/datasource/package.scala
@@ -18,9 +18,23 @@
*/
package astraea.spark.rasterframes
+
+import java.net.URI
+
+import scala.util.Try
+
/**
+ * Module utilities
*
* @since 1/13/18
*/
package object datasource {
+
+ private[rasterframes]
+ def numParam(key: String, parameters: Map[String, String]): Option[Long] =
+ parameters.get(key).map(_.toLong)
+
+ private[rasterframes]
+ def uriParam(key: String, parameters: Map[String, String]) =
+ parameters.get(key).flatMap(p ⇒ Try(URI.create(p)).toOption)
}
diff --git a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala b/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala
index 00e8b5d4a..6b59af02b 100644
--- a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala
+++ b/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala
@@ -18,6 +18,8 @@
*/
package astraea.spark.rasterframes.datasource.geotiff
+import java.nio.file.Paths
+
import astraea.spark.rasterframes._
/**
@@ -80,11 +82,24 @@ class GeoTiffDataSourceSpec
}
- it("should write RF to parquet") {
+ it("should write GeoTIFF RF to parquet") {
val rf = spark.read
.geotiff
.loadRF(cogPath)
assert(write(rf))
}
+
+ it("should write GeoTIFF") {
+
+ val rf = spark.read
+ .geotiff
+ .loadRF(cogPath)
+
+ val out = Paths.get(outputLocalPath, "example-geotiff.tiff")
+ //val out = Paths.get("target", "example-geotiff.tiff")
+ noException shouldBe thrownBy {
+ rf.write.geotiff.save(out.toString)
+ }
+ }
}
}
diff --git a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala b/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala
index ea5ab493d..d63e1e398 100644
--- a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala
+++ b/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/GeoTrellisDataSourceSpec.scala
@@ -22,9 +22,14 @@ import java.io.File
import java.time.ZonedDateTime
import astraea.spark.rasterframes._
+import astraea.spark.rasterframes.datasource.geotrellis.DefaultSource._
import astraea.spark.rasterframes.util._
+import astraea.spark.rasterframes.util.debug._
import geotrellis.proj4.LatLng
import geotrellis.raster._
+import geotrellis.raster.io.geotiff.GeoTiff
+import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod}
+import geotrellis.raster.testkit.RasterMatchers
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.io.avro.AvroRecordCodec
@@ -37,33 +42,40 @@ import org.apache.hadoop.fs.FileUtil
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.functions.{udf ⇒ sparkUdf, _}
import org.apache.spark.sql.{DataFrame, Row}
-import org.scalatest.{BeforeAndAfter, Inspectors}
+import org.scalatest.{BeforeAndAfterAll, Inspectors}
import org.apache.avro.generic._
import org.apache.spark.storage.StorageLevel
-import scala.math.{min, max}
+import scala.math.{max, min}
class GeoTrellisDataSourceSpec
- extends TestEnvironment with TestData with BeforeAndAfter with Inspectors
- with IntelliJPresentationCompilerHack {
+ extends TestEnvironment with TestData with BeforeAndAfterAll with Inspectors
+ with RasterMatchers with IntelliJPresentationCompilerHack {
+ val tileSize = 12
lazy val layer = Layer(new File(outputLocalPath).toURI, LayerId("test-layer", 4))
lazy val tfLayer = Layer(new File(outputLocalPath).toURI, LayerId("test-tf-layer", 4))
+ lazy val sampleImageLayer = Layer(new File(outputLocalPath).toURI, LayerId("sample", 0))
val now = ZonedDateTime.now()
val tileCoordRange = 2 to 5
lazy val testRdd = {
+ val ct = IntCellType
+
+ //UByteConstantNoDataCellType
val recs: Seq[(SpaceTimeKey, Tile)] = for {
col ← tileCoordRange
row ← tileCoordRange
- } yield SpaceTimeKey(col, row, now) -> ArrayTile.alloc(DoubleConstantNoDataCellType, 12, 12)
-
+ } yield SpaceTimeKey(col, row, now) -> TestData.randomTile(tileSize, tileSize, ct)
val rdd = sc.parallelize(recs)
- val scheme = ZoomedLayoutScheme(LatLng, tileSize = 12)
+ val scheme = ZoomedLayoutScheme(LatLng, tileSize = tileSize)
val layerLayout = scheme.levelForZoom(4).layout
- val layerBounds = KeyBounds(SpaceTimeKey(2, 2, now.minusMonths(1)), SpaceTimeKey(5, 5, now.plusMonths(1)))
+ val layerBounds = KeyBounds(
+ SpaceTimeKey(tileCoordRange.start, tileCoordRange.start, now.minusMonths(1)),
+ SpaceTimeKey(tileCoordRange.end, tileCoordRange.end, now.plusMonths(1))
+ )
val md = TileLayerMetadata[SpaceTimeKey](
- cellType = DoubleConstantNoDataCellType,
+ cellType = ct,
crs = LatLng,
bounds = layerBounds,
layout = layerLayout,
@@ -71,7 +83,7 @@ class GeoTrellisDataSourceSpec
ContextRDD(rdd, md)
}
- before {
+ override def beforeAll = {
val outputDir = new File(layer.base)
FileUtil.fullyDelete(outputDir)
outputDir.deleteOnExit()
@@ -102,8 +114,12 @@ class GeoTrellisDataSourceSpec
val writer = LayerWriter(tfLayer.base)
val tlfRdd = ContextRDD(tfRdd, testRdd.metadata)
writer.write(tfLayer.id, tlfRdd, ZCurveKeyIndexMethod.byDay())
- }
+ //TestData.sampleTileLayerRDD.toRF.write.geotrellis.asLayer(sampleImageLayer).save()
+ val writer2 = LayerWriter(sampleImageLayer.base)
+ val imgRDD = TestData.sampleTileLayerRDD
+ writer2.write(sampleImageLayer.id, imgRDD, ZCurveKeyIndexMethod)
+ }
describe("DataSource reading") {
def layerReader = spark.read.geotrellis
@@ -150,20 +166,96 @@ class GeoTrellisDataSourceSpec
}
describe("DataSource options") {
- def layerReader = spark.read.geotrellis
-
it("should respect partitions 2") {
val expected = 2
- val df = spark.read.option("numPartitions", expected)
- .geotrellis.loadRF(layer)
+ val df = spark.read.geotrellis
+ .withNumPartitions(expected)
+ .loadRF(layer)
assert(df.rdd.partitions.length === expected)
}
it("should respect partitions 20") {
val expected = 20
- val df = spark.read.option("numPartitions", expected)
- .geotrellis.loadRF(layer)
+ val df = spark.read.geotrellis
+ .withNumPartitions(expected)
+ .loadRF(layer)
assert(df.rdd.partitions.length === expected)
}
+ it("should respect subdivide 2") {
+ val param = 2
+ val df: RasterFrame = spark.read.geotrellis
+ .withTileSubdivisions(param)
+ .loadRF(layer)
+
+ val dims = df.select(tileDimensions(df.tileColumns.head)("cols"), tileDimensions(df.tileColumns.head)("rows")).first()
+ assert(dims.getAs[Int](0) === tileSize / param)
+ assert(dims.getAs[Int](1) === tileSize / param)
+
+ // row count will increase
+ assert(df.count === testRdd.count() * param * param)
+ }
+ it("should respect subdivide with TileFeature"){
+ val param = 2
+ val rf: RasterFrame = spark.read.geotrellis
+ .withTileSubdivisions(param)
+ .loadRF(tfLayer)
+
+ val dims = rf.select(tileDimensions(rf.tileColumns.head)("cols"), tileDimensions(rf.tileColumns.head)("rows"))
+ .first()
+ assert(dims.getAs[Int](0) === tileSize / param)
+ assert(dims.getAs[Int](1) === tileSize / param)
+
+ assert(rf.count() === testRdd.count() * param * param)
+ }
+
+ it("should respect both subdivideTile and numPartitions"){
+ val subParam = 3
+
+ val rf = spark.read
+ .geotrellis
+ .withNumPartitions(7)
+ .withTileSubdivisions(subParam)
+ .loadRF(layer)
+
+ // is it partitioned correctly?
+ assert(rf.rdd.partitions.length === 7)
+
+ // is it subdivided?
+ assert(rf.count === testRdd.count * subParam * subParam)
+ val dims = rf.select(tileDimensions(rf.tileColumns.head)("cols"), tileDimensions(rf.tileColumns.head)("rows"))
+ .first()
+ assert(dims.getAs[Int](0) === tileSize / subParam)
+ assert(dims.getAs[Int](1) === tileSize / subParam)
+ }
+
+ it("should subdivide tiles properly") {
+ val subs = 4
+ val rf = spark.read.geotrellis
+ .withTileSubdivisions(subs)
+ .loadRF(sampleImageLayer)
+
+
+ assert(rf.count === (TestData.sampleTileLayerRDD.count * subs * subs))
+
+ val (width, height) = sampleGeoTiff.tile.dimensions
+
+ val raster = rf.toRaster(rf.tileColumns.head, width, height, NearestNeighbor)
+
+ assertEqual(raster.tile, sampleGeoTiff.tile)
+
+ //GeoTiff(raster).write("target/from-split.tiff")
+ // 774 x 500
+ }
+
+ it("should throw on subdivide 5") {
+ // only throws when an action is taken...
+ assertThrows[IllegalArgumentException](spark.read.geotrellis.withTileSubdivisions(5).loadRF(layer).cache)
+ }
+ it("should throw on subdivide 13") {
+ assertThrows[IllegalArgumentException](spark.read.geotrellis.withTileSubdivisions(13).loadRF(layer).cache)
+ }
+ it("should throw on subdivide -3") {
+ assertThrows[IllegalArgumentException](spark.read.geotrellis.withTileSubdivisions(-3).loadRF(layer).count)
+ }
}
describe("Predicate push-down support") {
@@ -329,5 +421,37 @@ class GeoTrellisDataSourceSpec
//rf.show(false)
assert(rf.collect().length === testRdd.count())
}
+ it("should respect subdivideTile option on TileFeature RasterFrame") {
+ val subParam = 4
+ val rf = spark.read.option(TILE_SUBDIVISIONS_PARAM, subParam).geotrellis.loadRF(tfLayer)
+
+ assert(rf.count === testRdd.count * subParam * subParam)
+
+ val dims = rf.select(tileDimensions(rf.tileColumns.head)("cols"), tileDimensions(rf.tileColumns.head)("rows"))
+ .first()
+ assert(dims.getAs[Int](0) === tileSize / subParam)
+ assert(dims.getAs[Int](1) === tileSize / subParam)
+ }
+ it("should respect both `subdivideTile` and `numPartition` options on TileFeature"){
+ val subParam = 2
+
+ val rf = spark.read
+ .option(TILE_SUBDIVISIONS_PARAM, subParam)
+ .option(NUM_PARTITIONS_PARAM, 10)
+ .geotrellis.loadRF(tfLayer)
+
+ // is it subdivided?
+ assert(rf.count === testRdd.count * subParam * subParam)
+ val dims = rf.select(tileDimensions(rf.tileColumns.head)("cols"), tileDimensions(rf.tileColumns.head)("rows"))
+ .first()
+ assert(dims.getAs[Int](0) === tileSize / subParam)
+ assert(dims.getAs[Int](1) === tileSize / subParam)
+
+ // is it partitioned correctly?
+ assert(rf.rdd.partitions.length === 10)
+ }
+ it("should respect options on spatial-only TileFeature"){
+ assert(true === true)
+ }
}
}
diff --git a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala b/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala
index b621fa915..e7fa73d7c 100644
--- a/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala
+++ b/datasource/src/test/scala/astraea/spark/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala
@@ -21,11 +21,12 @@ package astraea.spark.rasterframes.datasource.geotrellis
import astraea.spark.rasterframes._
import astraea.spark.rasterframes.datasource.geotrellis.TileFeatureSupport._
+import astraea.spark.rasterframes.util.{WithCropMethods, WithMaskMethods, WithMergeMethods, WithPrototypeMethods}
import geotrellis.proj4.LatLng
import geotrellis.raster.crop.Crop
import geotrellis.raster.rasterize.Rasterizer
import geotrellis.raster.resample.Bilinear
-import geotrellis.raster.{CellGrid, GridBounds, IntCellType, ShortConstantNoDataCellType, Tile, TileFeature, TileLayout}
+import geotrellis.raster.{CellGrid, GridBounds, IntCellType, ShortCellType, ShortConstantNoDataCellType, Tile, TileFeature, TileLayout}
import geotrellis.spark.tiling.Implicits._
import geotrellis.spark.tiling._
import geotrellis.vector.{Extent, ProjectedExtent}
@@ -167,7 +168,7 @@ object TileFeatureSupportSpec {
sc.parallelize((1 to n).map(i => {
val (latMin, latMax) = rnd.nextOrderedPair(90)
val (lonMin, lonMax) = rnd.nextOrderedPair(180)
- (ProjectedExtent(Extent(lonMin, latMin, lonMax, latMax),LatLng),TestData.randomTile(20, 20, "int16"))
+ (ProjectedExtent(Extent(lonMin, latMin, lonMax, latMax),LatLng),TestData.randomTile(20, 20, ShortCellType))
}))
}
diff --git a/docs/src/main/tut/getting-started.md b/docs/src/main/tut/getting-started.md
index 216fa54fd..7e8c195f8 100644
--- a/docs/src/main/tut/getting-started.md
+++ b/docs/src/main/tut/getting-started.md
@@ -26,13 +26,21 @@ To use RasterFrames, add the following library dependencies:
@@dependency[sbt,Maven,Gradle] {
group="io.astraea"
- artifact="raster-frames_2.11"
+ artifact="rasterframes_2.11"
version="x.y.z"
}
@@dependency[sbt,Maven,Gradle] {
group="io.astraea"
- artifact="raster-frames-datasource_2.11"
+ artifact="rasterframes-datasource_2.11"
+ version="x.y.z"
+}
+
+Optional:
+
+@@dependency[sbt,Maven,Gradle] {
+ group="io.astraea"
+ artifact="rasterframes-experimental_2.11"
version="x.y.z"
}
diff --git a/docs/src/main/tut/release-notes.md b/docs/src/main/tut/release-notes.md
index 451955129..51e2bc77b 100644
--- a/docs/src/main/tut/release-notes.md
+++ b/docs/src/main/tut/release-notes.md
@@ -2,6 +2,21 @@
## 0.6.x
+### 0.6.2
+
+* Updated to GeoMesa version 2.0.0-rc.1.
+* Added for for writing GeoTIFFs from RasterFrames via with DataFrameWriter.
+* Added `spark.read.geotrellis.withNumPartitions(Int)` for setting the initial number of partitions to use when reading a layer.
+* Added `spark.read.geotrellis.withTileSubdivisions(Int)` for evenly subdividing tiles before they become rows in a RasterFrame.
+* Added `experimental` package for sandboxing new feature ideas.
+* Added `SlippyExport` experimental feature for exporting the contents of a RasterFrame as a [SlippyMap](https://wiki.openstreetmap.org/wiki/Slippy_Map)
+ tile image directory structure and Leaflet-enabled HTML file.
+* _Change_: Default interpoation for `toRaster` and `toMultibandRaster` has been changed from `Bilinear` to `NearestNeighbor`.
+* _Breaking_: Renamed/moved `astraea.spark.rasterframes.functions.CellStatsAggregateFunction.Statistics` to
+`astraea.spark.rasterframes.stats.CellStatistics`.
+* _Breaking_: `HistogramAggregateFunction` now generates a `astraea.spark.rasterframes.stats.CellHistogram`.
+
+
### 0.6.1
* Added support for reading striped GeoTiffs (#64).
diff --git a/experimental/README.md b/experimental/README.md
new file mode 100644
index 000000000..e6da48cdf
--- /dev/null
+++ b/experimental/README.md
@@ -0,0 +1,7 @@
+# RasterFrames Experimental Module
+
+## Notice
+
+Features in this modules are experimental, and as such should not be considered stable or production-ready,
+regardless of project version number. It is a place to try out new ideas, iterate over them, and over time either reject
+them or promote them to core RasterFrame features.
\ No newline at end of file
diff --git a/experimental/build.sbt b/experimental/build.sbt
new file mode 100644
index 000000000..ddb8b7450
--- /dev/null
+++ b/experimental/build.sbt
@@ -0,0 +1,8 @@
+moduleName := "rasterframes-experimental"
+
+libraryDependencies ++= Seq(
+ geotrellis("s3").value,
+ spark("core").value % Provided,
+ spark("mllib").value % Provided,
+ spark("sql").value % Provided
+)
\ No newline at end of file
diff --git a/experimental/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/experimental/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
new file mode 100644
index 000000000..adc5d2c83
--- /dev/null
+++ b/experimental/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -0,0 +1 @@
+astraea.spark.rasterframes.experimental.datasource.geojson.DefaultSource
diff --git a/experimental/src/main/resources/slippy.html b/experimental/src/main/resources/slippy.html
new file mode 100644
index 000000000..321fbc5bb
--- /dev/null
+++ b/experimental/src/main/resources/slippy.html
@@ -0,0 +1,84 @@
+
+
+
+
+
+
+ RasterFrames
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/DefaultSource.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/DefaultSource.scala
new file mode 100644
index 000000000..198d9c3b1
--- /dev/null
+++ b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/DefaultSource.scala
@@ -0,0 +1,52 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2018 Astraea. Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ */
+
+package astraea.spark.rasterframes.experimental.datasource.geojson
+
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.sql.sources.{BaseRelation, DataSourceRegister, RelationProvider}
+import org.locationtech.geomesa.spark.jts._
+import astraea.spark.rasterframes.datasource._
+import org.apache.spark.annotation.Experimental
+
+/**
+ * Basic support for parsing GeoJson into a DataFrame with a geometry/spatial column.
+ * Properties as rendered as a `Map[String,String]`
+ *
+ * @since 5/2/18
+ */
+@Experimental
+class DefaultSource extends DataSourceRegister with RelationProvider {
+ override def shortName(): String = DefaultSource.SHORT_NAME
+
+ override def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = {
+ val path = parameters.getOrElse(DefaultSource.PATH_PARAM,
+ throw new IllegalArgumentException("Valid URI 'path' parameter required.")
+ )
+ sqlContext.withJTS
+ GeoJsonRelation(sqlContext, path)
+ }
+}
+
+object DefaultSource {
+ final val SHORT_NAME = "geojson"
+ final val PATH_PARAM = "path"
+
+}
diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonRelation.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonRelation.scala
new file mode 100644
index 000000000..93b716aae
--- /dev/null
+++ b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonRelation.scala
@@ -0,0 +1,73 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2018 Astraea. Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ */
+
+package astraea.spark.rasterframes.experimental.datasource.geojson
+
+import java.net.URI
+
+import astraea.spark.rasterframes.experimental.datasource.geojson.GeoJsonRelation._
+import com.vividsolutions.jts.io.geojson.GeoJsonReader
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.jts.JTSTypes
+import org.apache.spark.sql.{Row, SQLContext}
+import org.apache.spark.sql.sources.{BaseRelation, TableScan}
+import org.apache.spark.sql.types.{DataTypes, StringType, StructField, StructType}
+import spray.json.DefaultJsonProtocol._
+import spray.json._
+
+/**
+ * Basic support for parsing GeoJson into a DataFrame with a geometry/spatial column.
+ * Properties as rendered as a `Map[String,String]`.
+ *
+ * @since 5/2/18
+ */
+@Experimental
+case class GeoJsonRelation(sqlContext: SQLContext, path: String) extends BaseRelation with TableScan {
+ override def schema: StructType = StructType(Seq(
+ StructField(GEOMETRY_COLUMN, JTSTypes.GeometryTypeInstance, false),
+ StructField(PROPERTIES_COLUMN,DataTypes.createMapType(StringType, StringType, true), true)
+ ))
+
+ override def buildScan(): RDD[Row] = {
+ val sc = sqlContext.sparkContext
+ sc.wholeTextFiles(path).mapPartitions { iter ⇒
+ val reader = new GeoJsonReader()
+ for {
+ (_, text) ← iter
+ dom = text.parseJson.convertTo[GeoJsonDom]
+ feature ← dom.features
+ geom = reader.read(feature.geometry.compactPrint)
+ props = feature.properties.mapValues(_.toString)
+ } yield Row(geom, props)
+ }
+ }
+}
+
+object GeoJsonRelation {
+ final val GEOMETRY_COLUMN = "geometry"
+ final val PROPERTIES_COLUMN = "properties"
+
+ case class GeoJsonDom(features: Seq[GeoJsonFeature])
+ case class GeoJsonFeature(geometry: JsValue, properties: Map[String, JsValue])
+
+ implicit val featureFormat: RootJsonFormat[GeoJsonFeature] = jsonFormat2(GeoJsonFeature)
+ implicit val domFormat: RootJsonFormat[GeoJsonDom] = jsonFormat1(GeoJsonDom)
+}
diff --git a/experimental/src/main/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExport.scala b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExport.scala
new file mode 100644
index 000000000..eba225e74
--- /dev/null
+++ b/experimental/src/main/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExport.scala
@@ -0,0 +1,158 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2018 Astraea. Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ */
+
+package astraea.spark.rasterframes.experimental.slippy
+
+import java.net.URI
+
+import astraea.spark.rasterframes._
+import astraea.spark.rasterframes.util._
+import geotrellis.proj4.{LatLng, WebMercator}
+import geotrellis.raster._
+import geotrellis.raster.io.geotiff._
+import geotrellis.raster.io.geotiff.tags.codes.ColorSpace
+import geotrellis.raster.render.{ColorMap, ColorRamps}
+import geotrellis.raster.resample.Bilinear
+import geotrellis.spark._
+import geotrellis.spark.io.slippy.HadoopSlippyTileWriter
+import geotrellis.spark.pyramid.Pyramid
+import geotrellis.spark.tiling.ZoomedLayoutScheme
+import geotrellis.util.MethodExtensions
+import org.apache.commons.lang3.text.StrSubstitutor
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.spark.annotation.Experimental
+
+import scala.io.Source
+import scala.util.Try
+
+/**
+ * Experimental support for exporting a RasterFrame into Slippy map format.
+ *
+ * @since 4/10/18
+ */
+@Experimental
+trait SlippyExport extends MethodExtensions[RasterFrame]{
+ /**
+ * Export GeoTiff tiles in a slippy map directory structure; for debugging purposes only.
+ * NB: Temporal components are ignored blindly.
+ */
+ @Experimental
+ def exportGeoTiffTiles(dest: URI): Unit = {
+ val spark = self.sparkSession
+ implicit val sc = spark.sparkContext
+
+ val tlm = self.tileLayerMetadata.widen
+ val crs = tlm.crs
+ val mapTransform = tlm.mapTransform
+
+ val writer = new HadoopSlippyTileWriter[MultibandTile](dest.toASCIIString, "tiff")({ (key, tile) =>
+ val extent = mapTransform(key)
+ // If we have exactly 3 columns, we assume RGB color space.
+ val opts = GeoTiffOptions.DEFAULT
+ .mapWhen(_ ⇒ tile.bands.lengthCompare(3) == 0,
+ _.copy(colorSpace = ColorSpace.RGB)
+ )
+ MultibandGeoTiff(tile, extent, crs, opts).toByteArray
+ })
+
+ val tlrdd: MultibandTileLayerRDD[SpatialKey] = self.toMultibandTileLayerRDD(self.tileColumns: _*) match {
+ case Left(spatial) ⇒ spatial
+ case Right(origRDD) ⇒
+ val newMD = origRDD.metadata.map(_.spatialKey)
+ val rdd = origRDD.map { case (k, v) ⇒ (k.spatialKey, v)}
+ ContextRDD(rdd, newMD)
+ }
+
+ writer.write(0, tlrdd)
+ }
+
+ /**
+ * Export tiles as a slippy map. For debugging purposes only.
+ * NB: Temporal components are ignored blindly.
+ * @param dest URI for Hadoop supported storage endpoint (e.g. 'file://', 'hdfs://', etc.).
+ * @param colorMap Optional color map to use for rendering tiles in non-RGB RasterFrames.
+ */
+ @Experimental
+ def exportSlippyMap(dest: URI, colorMap: Option[ColorMap] = None): Unit = {
+ val spark = self.sparkSession
+ implicit val sc = spark.sparkContext
+
+ val tileDirName = "rf-tiles"
+
+
+ val inputRDD: MultibandTileLayerRDD[SpatialKey] = self.toMultibandTileLayerRDD(self.tileColumns: _*) match {
+ case Left(spatial) ⇒ spatial
+ case Right(origRDD) ⇒
+ val newMD = origRDD.metadata.map(_.spatialKey)
+ val rdd = origRDD.map { case (k, v) ⇒ (k.spatialKey, v)}
+ ContextRDD(rdd, newMD)
+ }
+
+ val layoutScheme = ZoomedLayoutScheme(WebMercator, tileSize = 256)
+
+ val (zoom, reprojected) = inputRDD.reproject(WebMercator, layoutScheme, Bilinear)
+ val writer = new HadoopSlippyTileWriter[MultibandTile](dest.toASCIIString + "/" + tileDirName, "png")({ (_, tile) =>
+ val png = if(colorMap.isEmpty && tile.bands.lengthCompare(3) == 0) {
+ // `Try` below is due to https://github.com/locationtech/geotrellis/issues/2621
+ tile.mapBands((_, t) ⇒ Try(t.rescale(0, 255)).getOrElse(t)).renderPng()
+ }
+ else {
+ // Are there other ways to use the other bands?
+ val selected = tile.bands.head
+ colorMap.map(m ⇒ selected.renderPng(m)).getOrElse(selected.renderPng(ColorRamps.greyscale(256)))
+ }
+ png.bytes
+ })
+
+ val center = reprojected.metadata.extent.center.reproject(WebMercator, LatLng)
+
+ SlippyExport.writeHtml(dest, sc.hadoopConfiguration, Map(
+ "maxZoom" -> zoom.toString,
+ "id" -> tileDirName,
+ "viewLat" -> center.y.toString,
+ "viewLon" -> center.x.toString
+ ))
+
+ // Pyramiding up the zoom levels, write our tiles out to the local file system.
+ Pyramid.upLevels(reprojected, layoutScheme, zoom, Bilinear) { (rdd, z) =>
+ writer.write(z, rdd)
+ }
+ }
+}
+
+object SlippyExport {
+ import scala.collection.JavaConverters._
+ implicit class RasterFrameHasSlippy(val self: RasterFrame) extends SlippyExport
+ private def writeHtml(dest: URI, conf: Configuration, subs: Map[String, String]): Unit = {
+ val rawLines = Source.fromInputStream(getClass.getResourceAsStream("/slippy.html")).getLines()
+
+ val subst = new StrSubstitutor(subs.asJava)
+
+ val fs = FileSystem.get(dest, conf)
+ withResource(fs.create(new Path(new Path(dest), "index.html"), true)) { out ⇒
+ for(line ← rawLines) {
+ out.writeBytes(subst.replace(line))
+ out.writeChar('\n')
+ }
+ }
+ }
+}
+
diff --git a/experimental/src/test/resources/example.json b/experimental/src/test/resources/example.json
new file mode 100644
index 000000000..9e71c1ec5
--- /dev/null
+++ b/experimental/src/test/resources/example.json
@@ -0,0 +1,41 @@
+{
+ "type": "FeatureCollection",
+ "features": [
+ { "type": "Feature",
+ "geometry": {
+ "type": "Point",
+ "coordinates": [102.0, 0.5]
+ },
+ "properties": {
+ "prop0": "value0"
+ }
+ },
+ { "type": "Feature",
+ "geometry": {
+ "type": "LineString",
+ "coordinates": [
+ [102.0, 0.0], [103.0, 1.0], [104.0, 0.0], [105.0, 1.0]
+ ]
+ },
+ "properties": {
+ "prop0": "value0",
+ "prop1": 0.0
+ }
+ },
+ { "type": "Feature",
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0],
+ [100.0, 1.0], [100.0, 0.0]
+ ]
+ ]
+ },
+ "properties": {
+ "prop0": "value0",
+ "prop1": { "this": "that" }
+ }
+ }
+ ]
+}
diff --git a/experimental/src/test/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonDataSourceTest.scala b/experimental/src/test/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonDataSourceTest.scala
new file mode 100644
index 000000000..b40f216dd
--- /dev/null
+++ b/experimental/src/test/scala/astraea/spark/rasterframes/experimental/datasource/geojson/GeoJsonDataSourceTest.scala
@@ -0,0 +1,42 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2018 Astraea. Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ */
+
+package astraea.spark.rasterframes.experimental.datasource.geojson
+
+import astraea.spark.rasterframes.TestEnvironment
+import org.apache.spark.sql.types.MapType
+
+/**
+ * Test rig for GeoJsonRelation.
+ *
+ * @since 5/2/18
+ */
+class GeoJsonDataSourceTest extends TestEnvironment {
+
+ describe("GeoJson spark reader") {
+ it("should read geometry") {
+ val examplePath = getClass.getResource("/example.json").toURI.toASCIIString
+ val results = spark.read.format("geojson").load(examplePath)
+ assert(results.columns.length === 2)
+ assert(results.schema.fields(1).dataType.isInstanceOf[MapType])
+ assert(results.count() === 3)
+ }
+ }
+}
diff --git a/experimental/src/test/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExportDriver.scala b/experimental/src/test/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExportDriver.scala
new file mode 100644
index 000000000..6920c0b08
--- /dev/null
+++ b/experimental/src/test/scala/astraea/spark/rasterframes/experimental/slippy/SlippyExportDriver.scala
@@ -0,0 +1,58 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2018 Astraea. Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ *
+ */
+
+package astraea.spark.rasterframes.experimental.slippy
+
+import java.io.File
+
+import astraea.spark.rasterframes._
+import geotrellis.raster._
+import geotrellis.raster.io.geotiff.SinglebandGeoTiff
+import org.apache.spark.sql.SparkSession
+import SlippyExport._
+
+object SlippyExportDriver {
+ def main(args: Array[String]): Unit = {
+
+ implicit val spark = SparkSession
+ .builder()
+ .master("local[*]")
+ .appName("RasterFrames")
+ .getOrCreate()
+ .withRasterFrames
+
+ val bands: Seq[SinglebandGeoTiff] = for (i ← 1 to 3) yield {
+ TestData.readSingleband(s"NAIP-VA-b$i.tiff")
+ //s"L8-B$i-Elkton-VA.tiff")
+ }
+
+ val mtile = MultibandTile(bands.map(_.tile))
+
+ val pr = ProjectedRaster(mtile, bands.head.extent, bands.head.crs)
+
+ implicit val bandCount = PairRDDConverter.forSpatialMultiband(bands.length)
+
+ val rf = pr.toRF(64, 64)
+
+ //rf.exportGeoTiffTiles(new File("target/slippy-tiff").toURI)
+
+ rf.exportSlippyMap(new File("target/slippy-png/").toURI)
+ }
+}
diff --git a/project/ProjectPlugin.scala b/project/ProjectPlugin.scala
index e2d8567aa..c85b35d4a 100644
--- a/project/ProjectPlugin.scala
+++ b/project/ProjectPlugin.scala
@@ -18,7 +18,8 @@ object ProjectPlugin extends AutoPlugin {
import autoImport._
override def projectSettings = Seq(
- organization := "org.locationtech.rasterframes",
+ //organization := "org.locationtech.rasterframes",
+ organization := "io.astraea",
organizationName := "LocationTech RasterFrames",
startYear := Some(2017),
homepage := Some(url("http://rasterframes.io")),
@@ -31,12 +32,12 @@ object ProjectPlugin extends AutoPlugin {
cancelable in Global := true,
resolvers ++= Seq(
"locationtech-releases" at "https://repo.locationtech.org/content/groups/releases",
- "boundless-releases" at "https://repo.boundlessgeo.com/main/",
-// "osgeo-releases" at "http://download.osgeo.org/webdav/geotools/",
- Resolver.bintrayRepo("s22s", "maven")
+ "boundless-releases" at "https://repo.boundlessgeo.com/main/"
),
+
rfSparkVersion in ThisBuild := "2.2.1" ,
- rfGeotrellisVersion in ThisBuild := "1.2.0",
+ rfGeoTrellisVersion in ThisBuild := "1.2.1",
+ rfGeoMesaVersion in ThisBuild := "2.0.0",
publishTo := sonatypePublishTo.value,
publishMavenStyle := true,
@@ -69,10 +70,11 @@ object ProjectPlugin extends AutoPlugin {
object autoImport {
val rfSparkVersion = settingKey[String]("Apache Spark version")
- val rfGeotrellisVersion = settingKey[String]("GeoTrellis version")
+ val rfGeoTrellisVersion = settingKey[String]("GeoTrellis version")
+ val rfGeoMesaVersion = settingKey[String]("GeoMesa version")
def geotrellis(module: String) = Def.setting {
- "org.locationtech.geotrellis" %% s"geotrellis-$module" % rfGeotrellisVersion.value
+ "org.locationtech.geotrellis" %% s"geotrellis-$module" % rfGeoTrellisVersion.value
}
def spark(module: String) = Def.setting {
"org.apache.spark" %% s"spark-$module" % rfSparkVersion.value
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 125ebd419..4e2051238 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -28,6 +28,8 @@ addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.1")
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0")
+addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.7-astraea.1")
+
addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1")
// addSbtPlugin("laughedelic" % "literator" % "0.8.0")
diff --git a/pyrasterframes/.gitignore b/pyrasterframes/.gitignore
new file mode 100644
index 000000000..392b39883
--- /dev/null
+++ b/pyrasterframes/.gitignore
@@ -0,0 +1,7 @@
+*.pyc
+**/dist/
+**/build/
+**/*.egg-info
+.eggs
+.pytest_cache
+
diff --git a/pyrasterframes/build.sbt b/pyrasterframes/build.sbt
new file mode 100644
index 000000000..28e3a1544
--- /dev/null
+++ b/pyrasterframes/build.sbt
@@ -0,0 +1,150 @@
+import scala.sys.process.Process
+import sbt.Keys.`package`
+
+enablePlugins(SparkPackagePlugin, AssemblyPlugin)
+
+//--------------------------------------------------------------------
+// Spark Packages Plugin
+//--------------------------------------------------------------------
+
+spName := "io.astraea/pyrasterframes"
+sparkVersion := rfSparkVersion.value
+sparkComponents ++= Seq("sql", "mllib")
+spAppendScalaVersion := false
+spIncludeMaven := false
+spIgnoreProvided := true
+spShade := true
+spPackage := {
+ val dist = spDist.value
+ val extracted = IO.unzip(dist, (target in spPackage).value, GlobFilter("*.jar"))
+ if(extracted.size != 1) sys.error("Didn't expect to find multiple .jar files in distribution.")
+ extracted.head
+}
+spShortDescription := description.value
+spHomepage := homepage.value.get.toString
+spDescription := """
+ |RasterFrames brings the power of Spark DataFrames to geospatial raster data,
+ |empowered by the map algebra and tile layer operations of GeoTrellis.
+ |
+ |The underlying purpose of RasterFrames is to allow data scientists and software
+ |developers to process and analyze geospatial-temporal raster data with the
+ |same flexibility and ease as any other Spark Catalyst data type. At its core
+ |is a user-defined type (UDT) called TileUDT, which encodes a GeoTrellis Tile
+ |in a form the Spark Catalyst engine can process. Furthermore, we extend the
+ |definition of a DataFrame to encompass some additional invariants, allowing
+ |for geospatial operations within and between RasterFrames to occur, while
+ |still maintaining necessary geo-referencing constructs.
+ """.stripMargin
+
+test in assembly := {}
+
+spPublishLocal := {
+ // This unfortunate override is necessary because
+ // the ivy resolver in pyspark defaults to the cache more
+ // frequently than we'd like.
+ val id = (projectID in spPublishLocal).value
+ val home = ivyPaths.value.ivyHome
+ .getOrElse(io.Path.userHome / ".ivy2")
+ val cacheDir = home / "cache" / id.organization / id.name
+ IO.delete(cacheDir)
+ spPublishLocal.value
+}
+
+//--------------------------------------------------------------------
+// Python Build
+//--------------------------------------------------------------------
+
+lazy val pythonSource = settingKey[File]("Default Python source directory.")
+pythonSource := baseDirectory.value / "python"
+
+val Python = config("Python")
+
+Python / target := target.value / "python-dist"
+
+// Alias
+lazy val pyZip = Python / packageBin
+
+Python / packageBin / artifact := {
+ val java = (Compile / packageBin / artifact).value
+ java.withType("zip").withClassifier(Some("python")).withExtension("zip")
+}
+
+Python / packageBin / artifactPath := {
+ val dir = (Python / target).value
+ val art = (Python / packageBin / artifact).value
+ val ver = version.value
+ dir / s"${art.name}-python-$ver.zip"
+}
+
+//Python / packageBin / crossVersion := CrossVersion.disabled
+
+//artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
+// // Couldn't figure out how to config scope this, so having to handle for whole module
+// artifact.classifier match {
+// case Some("python") ⇒
+// val ver = version.value
+// s"${artifact.name}-python-${ver}.${artifact.extension}"
+// case _ ⇒ artifactName.value(sv, module, artifact)
+// }
+//}
+
+addArtifact(Python / packageBin / artifact, Python / packageBin)
+
+val pysparkCmd = taskKey[Unit]("Builds pyspark package and emits command string for running pyspark with package")
+
+lazy val pyTest = taskKey[Unit]("Run pyrasterframes tests.")
+
+lazy val pyExamples = taskKey[Unit]("Run pyrasterframes examples.")
+
+lazy val pyEgg = taskKey[Unit]("Creates a Python .egg file")
+
+lazy val spJarFile = Def.taskDyn {
+ if (spShade.value) {
+ Def.task((assembly in spPackage).value)
+ } else {
+ Def.task(spPackage.value)
+ }
+}
+
+Python / packageBin := {
+ val jar = (`package` in Compile).value
+ val license = pythonSource.value / "LICENSE.md"
+ val pyDir = pythonSource.value / "pyrasterframes"
+ val files = (IO.listFiles(pyDir, GlobFilter("*.py") | GlobFilter("*.rst")) ++ Seq(jar, license))
+ .map(f => (f, "pyrasterframes/" + f.getName))
+ val zipFile = (Python / packageBin / artifactPath).value
+ IO.zip(files, zipFile)
+ zipFile
+}
+
+pysparkCmd := {
+ val _ = spPublishLocal.value
+ val id = (projectID in spPublishLocal).value
+ val args = "pyspark" :: "--packages" :: s"${id.organization}:${id.name}:${id.revision}" :: Nil
+ streams.value.log.info("PySpark Command:\n" + args.mkString(" "))
+ // --conf spark.jars.ivy=(ivyPaths in pysparkCmd).value....
+}
+
+ivyPaths in pysparkCmd := ivyPaths.value.withIvyHome(target.value / "ivy")
+
+pyTest := {
+ val _ = spPublishLocal.value
+ val s = streams.value
+ val wd = pythonSource.value
+ Process("python setup.py test", wd) ! s.log
+}
+
+Test / test := (Test / test).dependsOn(pyTest).value
+
+pyEgg := {
+ val s = streams.value
+ val wd = pythonSource.value
+ Process("python setup.py bdist_egg", wd) ! s.log
+}
+
+pyExamples := {
+ val _ = spPublishLocal.value
+ val s = streams.value
+ val wd = pythonSource.value
+ Process("python setup.py examples", wd) ! s.log
+}
diff --git a/pyrasterframes/python-bindings.md b/pyrasterframes/python-bindings.md
new file mode 100644
index 000000000..ec2c92169
--- /dev/null
+++ b/pyrasterframes/python-bindings.md
@@ -0,0 +1,60 @@
+# RasterFrames Python Bindings
+
+## Goals
+
+The first goal of this effort is obvious: make the Scala-implemented features of RasterFrames available to Python programmers, via `pyspark` and `spark-submit`. The aim is to do so in way that is both idiomatic Python, yet synchronous with the Scala implementation and mental model.
+
+The second goal is to provide both embedded "python docstring" documentation and extensive Users' Manual examples of the Python-enabled RasterFrames features.
+
+The third goal is package and deploy this capability in a form that is the minimum possible amount of friction for new users to give it a spin. Something like
+
+ pyspark --packages io.astraea:rasterframes:1.2.3
+
+would be fantastic.
+
+The collection of these goals defines the über-goal: inspire and facilitate adoption of RasterFrames for solving real-world problems.
+
+## Implementing Bindings
+
+A proof-of concept implementation has already been started on the `feature/python-api` branch.
+See the instructions [here](https://github.com/s22s/raster-frames/tree/feature/python-api/python/README.rst) for how to run it.
+
+One of the sub-goals of the implementation is to have as little of it as possible! There are two types of bindings: Scala class wrappers, and UDF references. The former requires attention to delivering RasterFrames extension methods on Dataframes in a Pythonic way, [wrapping them](https://github.com/s22s/raster-frames/blob/feature/python-api/python/pyrasterframes/__init__.py) in Python classes, and handling type conversions via the Py4J API, as necessary. The latter simply requires adding a symbolic reference to the UDF namd in [`functions.py`](https://github.com/s22s/raster-frames/blob/feature/python-api/python/pyrasterframes/functions.py).
+
+The primary wrapper classes are [`RFContext`](https://github.com/s22s/raster-frames/blob/47a6f1af4c601ce75baf32b440bc3943d6e92f09/python/pyrasterframes/__init__.py#L12-L22) and [`RasterFrame`](https://github.com/s22s/raster-frames/blob/47a6f1af4c601ce75baf32b440bc3943d6e92f09/python/pyrasterframes/__init__.py#L25-L52). `RFContext` is made available as `spark_session.rf` when `SparkSession.withRasterFrames` is called. It is through this instance that `RasterFrame` instances are created.
+
+The `RasterFrame` wrapper is primarily there to expose extension methods on Scala side RasterFrames, and maintain the `RasterFrame` invariants over `TileLayerMetadata`.
+
+## Documenting Bindings
+
+RasterFames has a baseline [Users' Manual](http://rasterframes.io/index.html), built from Markdown files using a Scala tool called [Paradox](https://developer.lightbend.com/docs/paradox/latest/index.html). This tool has a special feature called ["Groups"](https://developer.lightbend.com/docs/paradox/latest/features/groups.html
+) for showing code snippets in multiple Programming languages. A possible approach to fulfilling this goal is to take the existing Scala examples and reimplement them in Python. Most of these examples live in full source form in an [examples directory](https://github.com/s22s/raster-frames/tree/develop/src/test/scala/examples). Once the example is working, it is manually reformatted into Markdown (in the case of Scala, we have a tool that will evaluate the code and inject the results automatically).
+
+A stretch goal is to also have the examples written in SQL.
+
+From a pure Python perspective, we'd like to have whatever is customary for a mature Python library.
+
+## Deploying Bindings
+
+We eventually want RasterFrames deployed via https://spark-packages.org/, in a form supporting both languages. We will be looking at https://github.com/databricks/sbt-spark-package to assist with the process.
+
+## Tasks
+
+* [x] Implement `asRF` on `Dataframe`
+* [x] Expose RasterFrame extension methods
+* [x] Add mirror for TileUDT (and update `pyUDT` field in Scala)
+* [ ] Properly (pythondoc) document wrapper classes
+* [ ] Enable language groups in Paradox
+* [x] Implement Scala examples in Python, finding missing API bindings
+* [ ] ~~Publish via spark-packages.org~~
+* [x] Figure out how to unit test python code
+
+
+## References
+
+* https://www.py4j.org/
+* http://aseigneurin.github.io/2016/09/01/spark-calling-scala-code-from-pyspark.html
+* https://github.com/harsha2010/magellan/tree/master/python
+* https://github.com/locationtech/geomesa/tree/master/geomesa-spark/geomesa_pyspark/src/main/python/geomesa_pyspark
+* https://github.com/locationtech-labs/geopyspark
+* https://stackoverflow.com/a/36024707/296509
diff --git a/pyrasterframes/python/LICENSE.md b/pyrasterframes/python/LICENSE.md
new file mode 100644
index 000000000..0dd8f3f8c
--- /dev/null
+++ b/pyrasterframes/python/LICENSE.md
@@ -0,0 +1,16 @@
+This software is licensed under the Apache 2 license, quoted below.
+
+Copyright 2017-2018 Astraea. Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License. You may obtain a copy of
+the License at
+
+ [http://www.apache.org/licenses/LICENSE-2.0]
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.
+
diff --git a/pyrasterframes/python/README.rst b/pyrasterframes/python/README.rst
new file mode 100644
index 000000000..e028a63ec
--- /dev/null
+++ b/pyrasterframes/python/README.rst
@@ -0,0 +1,53 @@
+PyRasterFrames
+--------------
+
+PyRasterFrames provides a Python API for RasterFrames!
+
+Getting started
+
+Build the shaded JAR.
+
+ $ sbt assembly
+
+Install the python package (for development / local use)
+
+ $ pip install -e python
+
+Get a Spark REPL
+
+ $ pyspark --jars target/scala-2.11/pyrasterframes-assembly-$VERSION.jar --master local[2]
+
+You can then try some of the commands in `tests/PyRasterFramesTests.py`.
+
+Submit a script
+
+ $ spark-submit --jars target/scala-2.11/pyrasterframes-assembly-$VERSION.jar --master local[2] \
+ python/examples/CreatingRasterFrames.py
+
+Run tests
+
+ $ sbt pyTest
+
+ OR
+
+ $ python setup.py test
+
+Run examples
+
+ $ sbt pyExamples
+
+ OR
+
+ $ python setup.py examples [--e EXAMPLENAME,EXAMPLENAME]
+
+
+To initialize PyRasterFrames:
+
+ >>> from pyrasterframes import *
+ >>> spark = SparkSession.builder \
+ ... .master("local[*]") \
+ ... .appName("Using RasterFrames") \
+ ... .config("spark.some.config.option", "some-value") \
+ ... .getOrCreate() \
+ ... .withRasterFrames()
+
diff --git a/pyrasterframes/python/examples/Clustering.py b/pyrasterframes/python/examples/Clustering.py
new file mode 100644
index 000000000..55ec8e663
--- /dev/null
+++ b/pyrasterframes/python/examples/Clustering.py
@@ -0,0 +1,88 @@
+
+#examples_setup
+from examples import resource_dir
+#examples_setup
+
+#py_cl_imports
+from pyrasterframes import *
+from pyrasterframes.rasterfunctions import *
+from pyspark.sql import *
+from pyspark.ml.feature import VectorAssembler
+from pyspark.ml.clustering import KMeans
+from pyspark.ml import Pipeline
+#py_cl_imports
+
+#py_cl_create_session
+spark = SparkSession.builder. \
+ master("local[*]"). \
+ appName("RasterFrames"). \
+ config("spark.ui.enabled", "false"). \
+ getOrCreate(). \
+ withRasterFrames()
+#py_cl_create_session
+
+# The first step is to load multiple bands of imagery and construct
+# a single RasterFrame from them.
+filenamePattern = "L8-B{}-Elkton-VA.tiff"
+bandNumbers = range(1, 8)
+bandColNames = list(map(lambda n: 'band_{}'.format(n), bandNumbers))
+
+# For each identified band, load the associated image file
+from functools import reduce
+joinedRF = reduce(lambda rf1, rf2: rf1.asRF().spatialJoin(rf2.drop('bounds').drop('metadata')),
+ map(lambda bf: spark.read.geotiff(bf[1]) \
+ .withColumnRenamed('tile', 'band_{}'.format(bf[0])),
+ map(lambda b: (b, resource_dir.joinpath(filenamePattern.format(b)).as_uri()), bandNumbers)))
+
+# We should see a single spatial_key column along with columns of tiles.
+joinedRF.printSchema()
+
+
+# SparkML requires that each observation be in its own row, and those
+# observations be packed into a single `Vector`. The first step is to
+# "explode" the tiles into a single row per cell/pixel
+exploder = TileExploder()
+
+
+# To "vectorize" the the band columns we use the SparkML `VectorAssembler`
+assembler = VectorAssembler() \
+ .setInputCols(bandColNames) \
+ .setOutputCol("features")
+
+# Configure our clustering algorithm
+k = 5
+kmeans = KMeans().setK(k)
+
+# Combine the two stages
+pipeline = Pipeline().setStages([exploder, assembler, kmeans])
+
+
+# Compute clusters
+model = pipeline.fit(joinedRF)
+
+# Run the data through the model to assign cluster IDs to each
+clustered = model.transform(joinedRF)
+clustered.show(8)
+
+
+# If we want to inspect the model statistics, the SparkML API requires us to go
+# through this unfortunate contortion:
+clusterResults = list(filter(lambda x: str(x).startswith('KMeans'), model.stages))[0]
+
+# Compute sum of squared distances of points to their nearest center
+metric = clusterResults.computeCost(clustered)
+print("Within set sum of squared errors: %s" % metric)
+
+tlm = joinedRF.tileLayerMetadata()
+layout = tlm['layoutDefinition']['tileLayout']
+
+retiled = clustered.groupBy('spatial_key').agg(
+ assembleTile('column_index', 'row_index', 'prediction',
+ layout['tileCols'], layout['tileRows'], 'int8')
+)
+
+rf = retiled.asRF('spatial_key', tlm)
+rf.printSchema()
+rf.show()
+
+spark.stop()
\ No newline at end of file
diff --git a/pyrasterframes/python/examples/CreatingRasterFrames.py b/pyrasterframes/python/examples/CreatingRasterFrames.py
new file mode 100644
index 000000000..3009db362
--- /dev/null
+++ b/pyrasterframes/python/examples/CreatingRasterFrames.py
@@ -0,0 +1,37 @@
+
+#examples_setup
+from examples import resource_dir
+#examples_setup
+
+#py_crf_imports
+from pyrasterframes import *
+from pyspark.sql import *
+#py_crf_imports
+
+#py_crf_create_session
+spark = SparkSession.builder. \
+ master("local[*]"). \
+ appName("RasterFrames"). \
+ config("spark.ui.enabled", "false"). \
+ getOrCreate(). \
+ withRasterFrames()
+#py_crf_create_session
+
+#py_crf_more_imports
+# No additional imports needed for Python.
+#py_crf_more_imports
+
+#py_crf_create_rasterframe
+rf = spark.read.geotiff(resource_dir.joinpath('L8-B8-Robinson-IL.tiff').as_uri())
+rf.show(5, False)
+#py_crf_create_rasterframe
+
+#py_crf_metadata
+rf.tileColumns()
+
+rf.spatialKeyColumn()
+
+rf.temporalKeyColumn()
+
+rf.tileLayerMetadata()
+#py_crf_metadata
diff --git a/pyrasterframes/python/examples/MeanValue.py b/pyrasterframes/python/examples/MeanValue.py
new file mode 100644
index 000000000..3bff7ac8b
--- /dev/null
+++ b/pyrasterframes/python/examples/MeanValue.py
@@ -0,0 +1,30 @@
+
+#examples_setup
+from examples import resource_dir
+#examples_setup
+
+
+#py_mv_imports
+from pyrasterframes import *
+from pyrasterframes.rasterfunctions import *
+from pyspark.sql import *
+#py_mv_imports
+
+#py_mv_create_session
+spark = SparkSession.builder. \
+ master("local[*]"). \
+ appName("RasterFrames"). \
+ config("spark.ui.enabled", "false"). \
+ getOrCreate(). \
+ withRasterFrames()
+#py_mv_create_session
+
+#py_mv_create_rasterframe
+rf = spark.read.geotiff(resource_dir.joinpath('L8-B8-Robinson-IL.tiff').as_uri())
+rf.show(5, False)
+#py_mv_create_rasterframe
+
+#py_mv_find_mean
+tileCol = 'tile'
+rf.agg(aggNoDataCells(tileCol), aggDataCells(tileCol), aggMean(tileCol)).show(5, False)
+#py_mv_find_mean
\ No newline at end of file
diff --git a/pyrasterframes/python/examples/NDVI.py b/pyrasterframes/python/examples/NDVI.py
new file mode 100644
index 000000000..1d3a4fda1
--- /dev/null
+++ b/pyrasterframes/python/examples/NDVI.py
@@ -0,0 +1,35 @@
+
+
+#examples_setup
+from examples import resource_dir
+#examples_setup
+
+#py_nvdi_imports
+from pyrasterframes import *
+from pyrasterframes.rasterfunctions import *
+from pyspark.sql import *
+from pyspark.sql.functions import udf
+#py_nvdi_imports
+
+#py_nvdi_create_session
+spark = SparkSession.builder. \
+ master("local[*]"). \
+ appName("RasterFrames"). \
+ config("spark.ui.enabled", "false"). \
+ getOrCreate(). \
+ withRasterFrames()
+#py_nvdi_create_session
+
+#py_ndvi_create_rasterframe
+redBand = spark.read.geotiff(resource_dir.joinpath('L8-B4-Elkton-VA.tiff').as_uri()).withColumnRenamed('tile', 'red_band')
+nirBand = spark.read.geotiff(resource_dir.joinpath('L8-B5-Elkton-VA.tiff').as_uri()).withColumnRenamed('tile', 'nir_band')
+#py_ndvi_create_rasterframe
+
+#py_ndvi_column
+rf = redBand.asRF().spatialJoin(nirBand.asRF()) \
+ .withColumn("ndvi", normalizedDifference('red_band', 'nir_band')).asRF()
+rf.printSchema()
+rf.show(20)
+#py_ndvi_column
+
+spark.stop()
\ No newline at end of file
diff --git a/pyrasterframes/python/examples/__init__.py b/pyrasterframes/python/examples/__init__.py
new file mode 100644
index 000000000..37c799c60
--- /dev/null
+++ b/pyrasterframes/python/examples/__init__.py
@@ -0,0 +1,8 @@
+#examples_setup
+from pathlib import Path
+import os
+jarpath = list(Path('../target').resolve().glob('**/pyrasterframes*.jar'))[0]
+os.environ["SPARK_CLASSPATH"] = jarpath.as_uri()
+# hard-coded relative path for resources
+resource_dir = Path('./static').resolve()
+#examples_setup
diff --git a/pyrasterframes/python/pyrasterframes/README.rst b/pyrasterframes/python/pyrasterframes/README.rst
new file mode 100644
index 000000000..cf9330217
--- /dev/null
+++ b/pyrasterframes/python/pyrasterframes/README.rst
@@ -0,0 +1,15 @@
+PyRasterFrames
+--------------
+
+PyRasterFrames provides a Python API for RasterFrames!
+
+To initialize PyRasterFrames:
+
+ >>> from pyrasterframes import *
+ >>> spark = SparkSession.builder \
+ ... .master("local[*]") \
+ ... .appName("Using RasterFrames") \
+ ... .config("spark.some.config.option", "some-value") \
+ ... .getOrCreate() \
+ ... .withRasterFrames()
+
diff --git a/pyrasterframes/python/pyrasterframes/__init__.py b/pyrasterframes/python/pyrasterframes/__init__.py
new file mode 100644
index 000000000..4a65d3cd6
--- /dev/null
+++ b/pyrasterframes/python/pyrasterframes/__init__.py
@@ -0,0 +1,73 @@
+"""
+Module initialization for PyRasterFrames. This is where much of the cool stuff is
+appended to PySpark classes.
+"""
+
+
+from __future__ import absolute_import
+from pyspark.sql.types import UserDefinedType
+from pyspark import SparkContext
+from pyspark.sql import SparkSession, DataFrame, DataFrameReader
+from pyspark.sql.types import *
+from pyspark.sql.column import _to_java_column
+
+# Import RasterFrame types and functions
+from pyrasterframes.types import *
+from pyrasterframes import rasterfunctions
+
+
+__all__ = ['RasterFrame', 'TileExploder']
+
+
+def _rf_init(spark_session):
+ """ Adds RasterFrames functionality to PySpark session."""
+ if not hasattr(spark_session, "rasterframes"):
+ spark_session.rasterframes = RFContext(spark_session)
+ spark_session.sparkContext._rf_context = spark_session.rasterframes
+
+ return spark_session
+
+
+def _reader(df_reader, format_key, path, **options):
+ """ Loads the file of the given type at the given path."""
+ df = df_reader.format(format_key).load(path, **options)
+ return _convertDF(df)
+
+
+def _convertDF(df, sp_key = None, metadata = None):
+ ctx = SparkContext._active_spark_context._rf_context
+
+ if sp_key is None:
+ return RasterFrame(ctx._jrfctx.asRF(df._jdf), ctx._spark_session)
+ else:
+ import json
+ return RasterFrame(ctx._jrfctx.asRF(
+ df._jdf, _to_java_column(sp_key), json.dumps(metadata)), ctx._spark_session)
+
+
+_prevFJ = UserDefinedType.fromJson
+# this classmethod annotation is needed for 2.7
+@classmethod
+def _fromJson(cls, json_val):
+ if str(json_val['class']).startswith('org.apache.spark.sql.jts'):
+ json_val['pyClass'] = 'pyrasterframes.GeometryUDT'
+
+ return _prevFJ(json_val)
+
+
+# Patch new method on SparkSession to mirror Scala approach
+SparkSession.withRasterFrames = _rf_init
+
+# Add the 'asRF' method to pyspark DataFrame
+DataFrame.asRF = _convertDF
+
+# Add DataSource convenience methods to the DataFrameReader
+# TODO: make sure this supports **options
+DataFrameReader.geotiff = lambda df_reader, path: _reader(df_reader, "geotiff", path)
+DataFrameReader.geotrellis = lambda df_reader, path: _reader(df_reader, "geotrellis", path)
+
+# If you don't have Python support, you will get it anyway
+UserDefinedType.fromJson = _fromJson
+
+
+
diff --git a/pyrasterframes/python/pyrasterframes/context.py b/pyrasterframes/python/pyrasterframes/context.py
new file mode 100644
index 000000000..4e9a8c6f6
--- /dev/null
+++ b/pyrasterframes/python/pyrasterframes/context.py
@@ -0,0 +1,13 @@
+"""
+This module contains access to the jvm SparkContext with RasterFrame support.
+"""
+
+from pyspark import SparkContext
+
+def _checked_context():
+ """ Get the active SparkContext and throw an error if it is not enabled for RasterFrames."""
+ sc = SparkContext._active_spark_context
+ if not hasattr(sc, '_rf_context'):
+ raise AttributeError(
+ "RasterFrames have not been enabled for the active session. Call 'SparkSession.withRasterFrames()'.")
+ return sc._rf_context._jrfctx
\ No newline at end of file
diff --git a/pyrasterframes/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/python/pyrasterframes/rasterfunctions.py
new file mode 100644
index 000000000..194996f12
--- /dev/null
+++ b/pyrasterframes/python/pyrasterframes/rasterfunctions.py
@@ -0,0 +1,137 @@
+"""
+This module creates explicit Python functions that map back to the existing Scala
+implementations. Most functions are standard Column functions, but those with unique
+signatures are handled here as well.
+"""
+
+
+from __future__ import absolute_import
+from pyspark.sql.types import *
+from pyspark.sql.column import Column, _to_java_column
+from pyrasterframes.context import _checked_context
+
+
+def _celltype(cellTypeStr):
+ """ Convert the string cell type to the expected CellType object."""
+ f = getattr(_checked_context(), 'cellType')
+ return f(cellTypeStr)
+
+
+def _create_withNoData():
+ """ Create a function mapping to the Scala implementation."""
+ def _(col, noDataVal):
+ jfcn = getattr(_checked_context(), 'withNoData')
+ return Column(jfcn(_to_java_column(col), noDataVal))
+ _.__name__ = 'withNoData'
+ _.__doc__ = "Assign a `NoData` value to the Tiles in the given Column."
+ _.__module__ = 'pyrasterframes'
+ return _
+
+
+def _create_assembleTile():
+ """ Create a function mapping to the Scala implementation."""
+ def _(colIndex, rowIndex, cellData, numCols, numRows, cellType):
+ ctx = _checked_context()
+ jfcn = getattr(ctx, 'assembleTile')
+ return Column(jfcn(_to_java_column(colIndex), _to_java_column(rowIndex), _to_java_column(cellData), numCols, numRows, _celltype(cellType)))
+ _.__name__ = 'assembleTile'
+ _.__doc__ = "Create a Tile from a column of cell data with location indices"
+ _.__module__ = 'pyrasterframes'
+ return _
+
+
+def _create_arrayToTile():
+ """ Create a function mapping to the Scala implementation."""
+ def _(arrayCol, numCols, numRows):
+ jfcn = getattr(_checked_context(), 'arrayToTile')
+ return Column(jfcn(_to_java_column(arrayCol), numCols, numRows))
+ _.__name__ = 'arrayToTile'
+ _.__doc__ = "Convert array in `arrayCol` into a Tile of dimensions `numCols` and `numRows'"
+ _.__module__ = 'pyrasterframes'
+ return _
+
+
+def _create_normalizedDifference():
+ """ Create a function mapping to the Scala implementation."""
+ def _(tileCol1, tileCol2):
+ jfcn = getattr(_checked_context(), 'normalizedDifference')
+ return Column(jfcn(_to_java_column(tileCol1), _to_java_column(tileCol2)))
+ _.__name__ = 'normalizedDifference'
+ _.__doc__ = "Create a Tile containing the normalized difference between `tileCol1` and `tileCol2`"
+ _.__module__ = 'pyrasterframes'
+ return _
+
+
+def _create_convertCellType():
+ """ Create a function mapping to the Scala implementation."""
+ def _(tileCol, cellType):
+ jfcn = getattr(_checked_context(), 'convertCellType')
+ return Column(jfcn(_to_java_column(tileCol), _celltype(cellType)))
+ _.__name__ = 'convertCellType'
+ _.__doc__ = "Convert the numeric type of the Tiles in `tileCol`"
+ _.__module__ = 'pyrasterframes'
+ return _
+
+
+
+_rf_unique_functions = {
+ 'withNoData': _create_withNoData(),
+ 'assembleTile': _create_assembleTile(),
+ 'arrayToTile': _create_arrayToTile(),
+ 'normalizedDifference': _create_normalizedDifference(),
+ 'convertCellType': _create_convertCellType(),
+}
+
+
+_rf_column_functions = {
+ 'explodeTiles': 'Create a row for each cell in Tile.',
+ 'tileDimensions': 'Query the number of (cols, rows) in a Tile.',
+ 'box2D': 'Extracts the bounding box (envelope) of the geometry.',
+ 'tileToIntArray': 'Flattens Tile into an array of integers.',
+ 'tileToDoubleArray': 'Flattens Tile into an array of doubles.',
+ 'cellType': 'Extract the Tile\'s cell type',
+ #'aggHistogram': 'Compute the full column aggregate floating point histogram',
+ 'aggStats': 'Compute the full column aggregate floating point statistics',
+ 'aggMean': 'Computes the column aggregate mean',
+ 'aggDataCells': 'Computes the number of non-NoData cells in a column',
+ 'aggNoDataCells': 'Computes the number of NoData cells in a column',
+ 'tileMean': 'Compute the Tile-wise mean',
+ 'tileSum': 'Compute the Tile-wise sum',
+ 'tileMin': 'Compute the Tile-wise minimum',
+ 'tileMax': 'Compute the Tile-wise maximum',
+ 'tileStats': 'Compute the Tile-wise floating point statistics',
+ 'renderAscii': 'Render ASCII art of tile',
+ 'noDataCells': 'Count of NODATA cells',
+ 'dataCells': 'Count of cells with valid data',
+ 'localAdd': 'Add two Tiles',
+ 'localSubtract': 'Subtract two Tiles',
+ 'localMultiply': 'Multiply two Tiles',
+ 'localDivide': 'Divide two Tiles',
+}
+
+
+__all__ = list(_rf_column_functions.keys()) + list(_rf_unique_functions.keys())
+
+
+def _create_column_function(name, doc=""):
+ """ Create a mapping to Scala UDF for a column function by name"""
+ def _(*args):
+ jfcn = getattr(_checked_context(), name)
+ jcols = [_to_java_column(arg) for arg in args]
+ return Column(jfcn(*jcols))
+ _.__name__ = name
+ _.__doc__ = doc
+ _.__module__ = 'pyrasterframes'
+ return _
+
+
+def _register_functions():
+ """ Register each function in the scope"""
+ for name, doc in _rf_column_functions.items():
+ globals()[name] = _create_column_function(name, doc)
+
+ for name, func in _rf_unique_functions.items():
+ globals()[name] = func
+
+
+_register_functions()
\ No newline at end of file
diff --git a/pyrasterframes/python/pyrasterframes/types.py b/pyrasterframes/python/pyrasterframes/types.py
new file mode 100644
index 000000000..5593059c9
--- /dev/null
+++ b/pyrasterframes/python/pyrasterframes/types.py
@@ -0,0 +1,143 @@
+"""
+This module contains all types relevant to PyRasterFrames. Classes in this module are
+meant to provide smoother pathways between the jvm and Python, and whenever possible,
+the implementations take advantage of the existing Scala functionality. The RasterFrame
+class here provides the PyRasterFrames entry point.
+"""
+
+from pyspark.sql.types import UserDefinedType
+from pyspark import SparkContext
+from pyspark.sql import SparkSession, DataFrame, Column, Row
+from pyspark.sql.types import *
+from pyspark.ml.wrapper import JavaTransformer
+from pyspark.ml.util import JavaMLReadable, JavaMLWritable
+from pyrasterframes.context import _checked_context
+
+__all__ = ['RFContext', 'RasterFrame', 'TileUDT', 'GeometryUDT', 'TileExploder']
+
+class RFContext(object):
+ """
+ Entrypoint to RasterFrames services
+ """
+ def __init__(self, spark_session):
+ self._spark_session = spark_session
+ self._gateway = spark_session.sparkContext._gateway
+ self._jvm = self._gateway.jvm
+ jsess = self._spark_session._jsparkSession
+ self._jrfctx = self._jvm.astraea.spark.rasterframes.py.PyRFContext(jsess)
+
+
+class RasterFrame(DataFrame):
+ def __init__(self, jdf, spark_session):
+ DataFrame.__init__(self, jdf, spark_session)
+ self._jrfctx = spark_session.rasterframes._jrfctx
+
+ def tileColumns(self):
+ """
+ Fetches columns of type Tile.
+ :return: One or more Column instances associated with Tiles.
+ """
+ cols = self._jrfctx.tileColumns(self._jdf)
+ return [Column(c) for c in cols]
+
+ def spatialKeyColumn(self):
+ """
+ Fetch the tagged spatial key column.
+ :return: Spatial key column
+ """
+ col = self._jrfctx.spatialKeyColumn(self._jdf)
+ return Column(col)
+
+ def temporalKeyColumn(self):
+ """
+ Fetch the temporal key column, if any.
+ :return: Temporal key column, or None.
+ """
+ col = self._jrfctx.temporalKeyColumn(self._jdf)
+ return col and Column(col)
+
+ def tileLayerMetadata(self):
+ """
+ Fetch the tile layer metadata.
+ :return: A dictionary of metadata.
+ """
+ import json
+ return json.loads(str(self._jrfctx.tileLayerMetadata(self._jdf)))
+
+ def spatialJoin(self, other_df):
+ """
+ Spatially join this RasterFrame to the given RasterFrame.
+ :return: Joined RasterFrame.
+ """
+ ctx = SparkContext._active_spark_context._rf_context
+ df = ctx._jrfctx.spatialJoin(self._jdf, other_df._jdf)
+ return RasterFrame(df, ctx._spark_session)
+
+
+class TileUDT(UserDefinedType):
+ """User-defined type (UDT).
+
+ .. note:: WARN: Internal use only.
+ """
+
+ @classmethod
+ def sqlType(self):
+ return StructType([
+ StructField("cellType", StringType(), False),
+ StructField("cols", ShortType(), False),
+ StructField("rows", ShortType(), False),
+ StructField("data", BinaryType(), False)
+ ])
+
+ @classmethod
+ def module(cls):
+ return 'pyrasterframes'
+
+ @classmethod
+ def scalaUDT(cls):
+ return 'org.apache.spark.sql.gt.types.TileUDT'
+
+ def serialize(self, obj):
+ if (obj is None): return None
+ return Row(obj.cellType().name().encode("UTF8"),
+ obj.cols().toShort(),
+ obj.rows().toShort(),
+ obj.toBytes)
+
+ def deserialize(self, datum):
+ return _checked_context().generateTile(datum[0], datum[1], datum[2], datum[3])
+
+
+
+class GeometryUDT(UserDefinedType):
+ """User-defined type (UDT).
+
+ .. note:: WARN: Internal use only.
+ """
+
+ @classmethod
+ def sqlType(self):
+ return StructField("wkb", BinaryType(), False)
+
+ @classmethod
+ def module(cls):
+ return 'pyrasterframes'
+
+ @classmethod
+ def scalaUDT(cls):
+ return 'org.apache.spark.sql.jts.GeometryUDT'
+
+ def serialize(self, obj):
+ if (obj is None): return None
+ return Row(obj.toBytes)
+
+ def deserialize(self, datum):
+ return _checked_context().generateGeometry(datum[0])
+
+
+
+class TileExploder(JavaTransformer, JavaMLReadable, JavaMLWritable):
+
+ def __init__(self):
+ super(TileExploder, self).__init__()
+ self._java_obj = self._new_java_obj("astraea.spark.rasterframes.ml.TileExploder", self.uid)
diff --git a/pyrasterframes/python/setup.cfg b/pyrasterframes/python/setup.cfg
new file mode 100644
index 000000000..d8fb59953
--- /dev/null
+++ b/pyrasterframes/python/setup.cfg
@@ -0,0 +1,8 @@
+
+[aliases]
+test=pytest
+
+[tool:pytest]
+addopts = --verbose
+testpaths = tests
+python_files = *.py
\ No newline at end of file
diff --git a/pyrasterframes/python/setup.py b/pyrasterframes/python/setup.py
new file mode 100644
index 000000000..2c6a6ca27
--- /dev/null
+++ b/pyrasterframes/python/setup.py
@@ -0,0 +1,152 @@
+"""
+The operations in this file are designed for development and testing only.
+"""
+
+
+from setuptools import setup, find_packages
+import distutils.log
+import importlib
+
+
+def _extract_module(mod):
+ module = importlib.import_module(mod)
+
+ if hasattr(module, '__all__'):
+ globals().update({n: getattr(module, n) for n in module.__all__})
+ else:
+ globals().update({k: v for (k, v) in module.__dict__.items() if not k.startswith('_')})
+
+
+
+class ExampleCommand(distutils.cmd.Command):
+ """A custom command to run pyrasterframes examples."""
+
+ description = 'run pyrasterframes examples'
+ user_options = [
+ # The format is (long option, short option, description).
+ ('examples=', 'e', 'examples to run'),
+ ]
+
+ def initialize_options(self):
+ from pathlib import Path
+ """Set default values for options."""
+ # Each user option must be listed here with their default value.
+ self.examples = filter(lambda x: not x.name.startswith('_'),
+ list(Path('./examples').resolve().glob('*.py')))
+
+ def _check_ex_path(self, ex):
+ from pathlib import Path
+ file = Path(ex)
+ if not file.suffix:
+ file = file.with_suffix('.py')
+ file = (Path('./examples') / file).resolve()
+
+ assert file.is_file(), ('Invalid example %s' % file)
+ return file
+
+ def finalize_options(self):
+ """Post-process options."""
+ import re
+ if isinstance(self.examples, str):
+ self.examples = re.split('\W+', self.examples)
+ self.examples = map(lambda x: 'examples.' + x.stem,
+ map(self._check_ex_path, self.examples))
+
+
+ def run(self):
+ """Run the examples."""
+ import traceback
+ for ex in self.examples:
+ print(('-' * 50) + '\nRunning %s' % ex + '\n' + ('-' * 50))
+ try:
+ _extract_module(ex)
+ except Exception:
+ print(('-' * 50) + '\n%s Failed:' % ex + '\n' + ('-' * 50))
+ print(traceback.format_exc())
+
+
+class ZipCommand(distutils.cmd.Command):
+ """A custom command to create a minimal zip distribution."""
+
+ description = 'create a minimal pyrasterframes source zip file'
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ """Create the zip."""
+ import zipfile
+ from pathlib import Path
+ import os
+ zfile = 'pyrasterframes.zip'
+
+ if os.path.isfile(zfile):
+ os.remove(zfile)
+ with zipfile.ZipFile(zfile, 'w') as przip:
+ przip.write('pyrasterframes')
+ # Bring in source files and readme
+ patterns = ['*.py', '*.rst', '*.jar']
+ root = Path('.').resolve()
+ for pattern in patterns:
+ for file in list(root.glob('pyrasterframes/' + pattern)):
+ przip.write(str(file.relative_to(root)))
+ # Put a copy of the license in the zip
+ przip.write('LICENSE.md', 'pyrasterframes/LICENSE.md')
+
+
+
+
+
+with open('README.rst') as f:
+ readme = f.read()
+
+pyspark_ver = 'pyspark>=2.1.0,<2.2'
+
+setup_args = dict(
+ name='pyrasterframes',
+ description='Python bindings for RasterFrames',
+ long_description=readme,
+ version='0.0.1',
+ url='http://rasterframes.io',
+ author='Simeon H.K. Fitch',
+ author_email='fitch@astraea.io',
+ license='Apache 2',
+ setup_requires=['pytest-runner', pyspark_ver, 'pathlib'],
+ install_requires=[
+ # pyspark_ver,
+ # 'pathlib'
+ ],
+ tests_require=[
+ pyspark_ver,
+ 'pytest==3.4.2'
+ ],
+ test_suite="pytest-runner",
+ packages=find_packages(exclude=['tests', 'examples']),
+ include_package_data=True,
+ package_data={'.':['LICENSE.md'], 'pyrasterframes':['*.jar']},
+ exclude_package_data={'.':['setup.*', 'README.*']},
+ classifiers=[
+ 'Development Status :: 3 - Alpha',
+ 'Environment :: Other Environment',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Natural Language :: English',
+ 'Operating System :: Unix',
+ 'Programming Language :: Python',
+ 'Topic :: Software Development :: Libraries'
+ ],
+ zip_safe=False,
+ cmdclass={
+ 'examples': ExampleCommand,
+ 'minzip': ZipCommand
+ }
+ # entry_points={
+ # "console_scripts": ['pyrasterframes=pyrasterframes:console']
+ # }
+)
+
+if __name__ == "__main__":
+ setup(**setup_args)
diff --git a/pyrasterframes/python/static/L8-B1-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B1-Elkton-VA.tiff
new file mode 100644
index 000000000..0d5a644ea
Binary files /dev/null and b/pyrasterframes/python/static/L8-B1-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B10-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B10-Elkton-VA.tiff
new file mode 100644
index 000000000..947cef3a6
Binary files /dev/null and b/pyrasterframes/python/static/L8-B10-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B11-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B11-Elkton-VA.tiff
new file mode 100644
index 000000000..f57004679
Binary files /dev/null and b/pyrasterframes/python/static/L8-B11-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B2-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B2-Elkton-VA.tiff
new file mode 100644
index 000000000..b287e5180
Binary files /dev/null and b/pyrasterframes/python/static/L8-B2-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B3-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B3-Elkton-VA.tiff
new file mode 100644
index 000000000..61a95250c
Binary files /dev/null and b/pyrasterframes/python/static/L8-B3-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B4-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B4-Elkton-VA.tiff
new file mode 100644
index 000000000..2534d4bd0
Binary files /dev/null and b/pyrasterframes/python/static/L8-B4-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B5-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B5-Elkton-VA.tiff
new file mode 100644
index 000000000..9a574fce8
Binary files /dev/null and b/pyrasterframes/python/static/L8-B5-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B6-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B6-Elkton-VA.tiff
new file mode 100644
index 000000000..1a1ad3c7f
Binary files /dev/null and b/pyrasterframes/python/static/L8-B6-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B7-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B7-Elkton-VA.tiff
new file mode 100644
index 000000000..56616e0f6
Binary files /dev/null and b/pyrasterframes/python/static/L8-B7-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B8-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B8-Elkton-VA.tiff
new file mode 100644
index 000000000..9bbe2c89f
Binary files /dev/null and b/pyrasterframes/python/static/L8-B8-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/static/L8-B8-Robinson-IL.tiff b/pyrasterframes/python/static/L8-B8-Robinson-IL.tiff
new file mode 100644
index 000000000..224ec5ac9
Binary files /dev/null and b/pyrasterframes/python/static/L8-B8-Robinson-IL.tiff differ
diff --git a/pyrasterframes/python/static/L8-B9-Elkton-VA.tiff b/pyrasterframes/python/static/L8-B9-Elkton-VA.tiff
new file mode 100644
index 000000000..3f596e19f
Binary files /dev/null and b/pyrasterframes/python/static/L8-B9-Elkton-VA.tiff differ
diff --git a/pyrasterframes/python/tests/PyRasterFramesTests.py b/pyrasterframes/python/tests/PyRasterFramesTests.py
new file mode 100644
index 000000000..3e60ccee3
--- /dev/null
+++ b/pyrasterframes/python/tests/PyRasterFramesTests.py
@@ -0,0 +1,163 @@
+
+from pyspark.sql import SparkSession, Column
+from pyspark.sql.functions import *
+from pyrasterframes import *
+from pyrasterframes.rasterfunctions import *
+from pathlib import Path
+import os
+import unittest
+
+# version-conditional imports
+import sys
+if sys.version_info[0] > 2:
+ import builtins
+else:
+ import __builtin__ as builtins
+
+
+
+
+def _rounded_compare(val1, val2):
+ print('Comparing {} and {} using round()'.format(val1, val2))
+ return builtins.round(val1) == builtins.round(val2)
+
+
+class RasterFunctionsTest(unittest.TestCase):
+
+
+ @classmethod
+ def setUpClass(cls):
+
+ # gather Scala requirements
+ jarpath = list(Path('../target').resolve().glob('pyrasterframes*.jar'))[0]
+ os.environ["SPARK_CLASSPATH"] = jarpath.as_uri()
+
+ # hard-coded relative path for resources
+ cls.resource_dir = Path('./static').resolve()
+
+ # spark session with RF
+ cls.spark = SparkSession.builder.getOrCreate()
+ cls.spark.sparkContext.setLogLevel('ERROR')
+ print(cls.spark.version)
+ cls.spark.withRasterFrames()
+
+ # load something into a rasterframe
+ rf = cls.spark.read.geotiff(cls.resource_dir.joinpath('L8-B8-Robinson-IL.tiff').as_uri())
+
+ # convert the tile cell type to provide for other operations
+ cls.tileCol = 'tile'
+ cls.rf = rf.withColumn('tile2', convertCellType(cls.tileCol, 'float32')) \
+ .drop(cls.tileCol) \
+ .withColumnRenamed('tile2', cls.tileCol).asRF()
+ cls.rf.show()
+
+
+ def test_identify_columns(self):
+ cols = self.rf.tileColumns()
+ self.assertEqual(len(cols), 1, '`tileColumns` did not find the proper number of columns.')
+ print("Tile columns: ", cols)
+ col = self.rf.spatialKeyColumn()
+ self.assertIsInstance(col, Column, '`spatialKeyColumn` was not found')
+ print("Spatial key column: ", col)
+ col = self.rf.temporalKeyColumn()
+ self.assertIsNone(col, '`temporalKeyColumn` should be `None`')
+ print("Temporal key column: ", col)
+
+
+ def test_tile_operations(self):
+ df1 = self.rf.withColumnRenamed(self.tileCol, 't1').asRF()
+ df2 = self.rf.withColumnRenamed(self.tileCol, 't2').asRF()
+ df3 = df1.spatialJoin(df2).asRF()
+ df3 = df3.withColumn('norm_diff', normalizedDifference('t1', 't2'))
+ df3.printSchema()
+
+ aggs = df3.agg(
+ aggMean('norm_diff'),
+ )
+ aggs.show()
+ row = aggs.first()
+
+ self.assertTrue(_rounded_compare(row['agg_mean(norm_diff)'], 0))
+
+
+ def test_general(self):
+ meta = self.rf.tileLayerMetadata()
+ self.assertIsNotNone(meta['bounds'])
+ df = self.rf.withColumn('dims', tileDimensions(self.tileCol)) \
+ .withColumn('type', cellType(self.tileCol)) \
+ .withColumn('dCells', dataCells(self.tileCol)) \
+ .withColumn('ndCells', noDataCells(self.tileCol)) \
+ .withColumn('min', tileMin(self.tileCol)) \
+ .withColumn('max', tileMax(self.tileCol)) \
+ .withColumn('mean', tileMean(self.tileCol)) \
+ .withColumn('sum', tileSum(self.tileCol)) \
+ .withColumn('stats', tileStats(self.tileCol)) \
+ .withColumn('box2D', box2D('bounds')) \
+ .withColumn('ascii', renderAscii(self.tileCol))
+
+ df.show()
+
+
+ def test_aggregations(self):
+ aggs = self.rf.agg(
+ aggMean(self.tileCol),
+ aggDataCells(self.tileCol),
+ aggNoDataCells(self.tileCol),
+ aggStats(self.tileCol),
+
+ # Not currently working:
+ # aggHistogram(self.tileCol),
+ )
+ aggs.show()
+ row = aggs.first()
+
+ self.assertTrue(_rounded_compare(row['agg_mean(tile)'], 10158))
+ self.assertTrue(row['agg_data_cells(tile)'] == 250000)
+ self.assertTrue(row['agg_nodata_cells(tile)'] == 0)
+ self.assertTrue(row['aggStats(tile)'].dataCells == row['agg_data_cells(tile)'])
+
+
+ def test_sql(self):
+
+ self.rf.createOrReplaceTempView("rf")
+
+ dims = self.rf.withColumn('dims', tileDimensions(self.tileCol)).first().dims
+ dims_str = """{}, {}""".format(dims.cols, dims.rows)
+
+ self.spark.sql("""SELECT tile, rf_makeConstantTile(1, {}, 'uint16') AS One,
+ rf_makeConstantTile(2, {}, 'uint16') AS Two FROM rf""".format(dims_str, dims_str)) \
+ .createOrReplaceTempView("r3")
+
+ ops = self.spark.sql("""SELECT tile, rf_localAdd(tile, One) AS AndOne,
+ rf_localSubtract(tile, One) AS LessOne,
+ rf_localMultiply(tile, Two) AS TimesTwo,
+ rf_localDivide( tile, Two) AS OverTwo
+ FROM r3""")
+
+ ops.printSchema
+ statsRow = ops.select(tileMean(self.tileCol).alias('base'),
+ tileMean("AndOne").alias('plus_one'),
+ tileMean("LessOne").alias('minus_one'),
+ tileMean("TimesTwo").alias('double'),
+ tileMean("OverTwo").alias('half')) \
+ .first()
+
+ self.assertTrue(_rounded_compare(statsRow.base, statsRow.plus_one - 1))
+ self.assertTrue(_rounded_compare(statsRow.base, statsRow.minus_one + 1))
+ self.assertTrue(_rounded_compare(statsRow.base, statsRow.double / 2))
+ self.assertTrue(_rounded_compare(statsRow.base, statsRow.half * 2))
+
+
+def suite():
+ functionTests = unittest.TestSuite()
+ functionTests.addTest(RasterFunctionsTest('test_identify_columns'))
+ functionTests.addTest(RasterFunctionsTest('test_general'))
+ functionTests.addTest(RasterFunctionsTest('test_aggregations'))
+ functionTests.addTest(RasterFunctionsTest('test_sql'))
+ return functionTests
+
+
+unittest.TextTestRunner().run(suite())
+
+
+
diff --git a/pyrasterframes/src/main/scala/astraea/spark/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/astraea/spark/rasterframes/py/PyRFContext.scala
new file mode 100644
index 000000000..eeb5edff5
--- /dev/null
+++ b/pyrasterframes/src/main/scala/astraea/spark/rasterframes/py/PyRFContext.scala
@@ -0,0 +1,100 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2017-2018 Astraea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+package astraea.spark.rasterframes.py
+
+import geotrellis.raster.io.geotiff.reader.GeoTiffReader
+import org.apache.spark.sql._
+import astraea.spark.rasterframes._
+import astraea.spark.rasterframes.util.withAlias
+import com.vividsolutions.jts.geom.Geometry
+import geotrellis.raster.{ArrayTile, CellType, Tile}
+import geotrellis.spark.{SpatialKey, TileLayerMetadata}
+import geotrellis.spark.io._
+import org.locationtech.geomesa.spark.jts.util.WKBUtils
+import spray.json._
+
+
+/**
+ * py4j access wrapper to RasterFrame entry points.
+ *
+ * @author sfitch
+ * @since 11/6/17
+ */
+class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions {
+ sparkSession.withRasterFrames
+
+ /**
+ * Base conversion to RasterFrame
+ */
+ def asRF(df: DataFrame): RasterFrame = {
+ df.asRF
+ }
+
+ /**
+ * Conversion to RasterFrame with spatial key column and TileLayerMetadata specified.
+ */
+ def asRF(df: DataFrame, spatialKey: Column, tlm: String): RasterFrame = {
+ val jtlm = tlm.parseJson.convertTo[TileLayerMetadata[SpatialKey]]
+ df.asRF(spatialKey, jtlm)
+ }
+
+ /**
+ * Convenience function for use in Python
+ */
+ def cellType(name: String): CellType = CellType.fromName(name)
+
+ /** DESERIALIZATION **/
+
+ def generateTile(cellType: String, cols: Int, rows: Int, bytes: Array[Byte]): ArrayTile = {
+ ArrayTile.fromBytes(bytes, this.cellType(cellType), cols, rows)
+ }
+
+ def generateGeometry(obj: Array[Byte]): Geometry = {
+ WKBUtils.read(obj)
+ }
+
+ def tileColumns(df: DataFrame): Array[Column] =
+ df.asRF.tileColumns.toArray
+
+ def spatialKeyColumn(df: DataFrame): Column =
+ df.asRF.spatialKeyColumn
+
+ def temporalKeyColumn(df: DataFrame): Column =
+ df.asRF.temporalKeyColumn.orNull
+
+ def tileToIntArray(col: Column): Column = tileToArray[Int](col)
+
+ def tileToDoubleArray(col: Column): Column = tileToArray[Double](col)
+
+ def tileLayerMetadata(df: DataFrame): String =
+ // The `fold` is required because an `Either` is retured, depending on the key type.
+ df.asRF.tileLayerMetadata.fold(_.toJson, _.toJson).prettyPrint
+
+ def normalizedDifference(left: Column, right: Column): Column = {
+ withAlias("norm_diff", left, right)(
+ localDivide(localSubtract(left, right), localAdd(left, right))
+ ).as[Tile]
+ }
+
+ def spatialJoin(df: DataFrame, right: DataFrame): RasterFrame = {
+ df.asRF.spatialJoin(right.asRF)
+ }
+
+
+}
diff --git a/version.sbt b/version.sbt
index 421f10f23..48fc5fb9a 100644
--- a/version.sbt
+++ b/version.sbt
@@ -1 +1 @@
-version in ThisBuild := "0.6.1"
+version in ThisBuild := "0.6.2-RC2"