Skip to content

Commit 888b673

Browse files
bossiebossie
committed
saveStitchedTileGridTemporal/saveSamples: return items instead of files
Open-EO/openeo-geopyspark-driver#1111
1 parent 7c0dd48 commit 888b673

File tree

4 files changed

+40
-48
lines changed

4 files changed

+40
-48
lines changed

‎openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala

Lines changed: 25 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,10 @@ package object geotiff {
8585
def saveStitched(rdd: SRDD, path: String, cropBounds: Map[String, Double], compression: Compression): Extent =
8686
saveStitched(rdd, path, Some(cropBounds), None, compression)
8787

88-
def saveStitchedTileGrid(rdd: SRDD, path: String, tileGrid: String, compression: Compression): java.util.List[(String, Extent)] =
88+
def saveStitchedTileGrid(rdd: SRDD, path: String, tileGrid: String, compression: Compression): JList[(String, Extent)] =
8989
saveStitchedTileGrid(rdd, path, tileGrid, None, None, compression)
9090

91-
def saveStitchedTileGrid(rdd: SRDD, path: String, tileGrid: String, cropBounds: Map[String, Double], compression: Compression): java.util.List[(String, Extent)] =
91+
def saveStitchedTileGrid(rdd: SRDD, path: String, tileGrid: String, cropBounds: Map[String, Double], compression: Compression): JList[(String, Extent)] =
9292
saveStitchedTileGrid(rdd, path, tileGrid, Some(cropBounds), None, compression)
9393

9494
def saveRDDTiled(rdd:MultibandTileLayerRDD[SpaceTimeKey], path:String,zLevel:Int=6,cropBounds:Option[Extent]=Option.empty[Extent]):Unit = {
@@ -105,7 +105,7 @@ package object geotiff {
105105
zLevel: Int = 6,
106106
cropBounds: Option[Extent] = Option.empty[Extent],
107107
formatOptions: GTiffOptions = new GTiffOptions
108-
): java.util.List[(String, String, Extent)] = {
108+
): JList[(String, String, Extent)] = {
109109
rdd.sparkContext.setCallSite(s"save_result(GTiff, temporal)")
110110
formatOptions.assertNoConflicts()
111111
val ret = saveRDDTemporalAllowAssetPerBand(rdd, path, zLevel, cropBounds, formatOptions)
@@ -204,7 +204,7 @@ package object geotiff {
204204
zLevel: Int = 6,
205205
cropBounds: Option[Extent] = Option.empty[Extent],
206206
formatOptions: GTiffOptions = new GTiffOptions
207-
): java.util.List[Item] = {
207+
): JList[Item] = {
208208
formatOptions.assertNoConflicts()
209209
val preProcessResult: (GridBounds[Int], Extent, RDD[(SpaceTimeKey, MultibandTile)] with Metadata[TileLayerMetadata[SpaceTimeKey]]) = preProcess(rdd,cropBounds)
210210
val gridBounds: GridBounds[Int] = preProcessResult._1
@@ -299,7 +299,7 @@ package object geotiff {
299299
}
300300
.toMap
301301

302-
Item(id = UUID.randomUUID().toString, datetime=timestamp, bbox = croppedExtent, assets.asJava)
302+
Item(id = s"${UUID.randomUUID()}_$timestamp", datetime=timestamp, bbox = croppedExtent, assets.asJava)
303303
}
304304

305305
for ((geotiffResult, _, _, _) <- geotiffResults) {
@@ -318,7 +318,7 @@ package object geotiff {
318318
zLevel: Int = 6,
319319
cropBounds: Option[Extent] = Option.empty[Extent],
320320
formatOptions: GTiffOptions = new GTiffOptions
321-
): java.util.List[String] = {
321+
): JList[String] = {
322322
rdd.sparkContext.setCallSite(s"save_result(GTiff, spatial, $bandCount)")
323323
val tmp = saveRDDAllowAssetPerBand(rdd, bandCount, path, zLevel, cropBounds, formatOptions).asScala
324324
logger.warn("Calling backwards compatibility version for saveRDDAllowAssetPerBand")
@@ -335,7 +335,7 @@ package object geotiff {
335335
zLevel: Int = 6,
336336
cropBounds: Option[Extent] = Option.empty[Extent],
337337
formatOptions: GTiffOptions = new GTiffOptions
338-
): java.util.List[(String, Extent, java.util.List[Int])] = {
338+
): JList[(String, Extent, JList[Int])] = {
339339
formatOptions.assertNoConflicts()
340340
if (formatOptions.separateAssetPerBand) {
341341
val bandLabels = formatOptions.tags.bandTags.map(_("DESCRIPTION"))
@@ -503,7 +503,7 @@ package object geotiff {
503503
def levelFor(extent: Extent, cellSize: CellSize): LayoutLevel = ???
504504
}
505505

506-
def saveRDDGeneric[K: SpatialComponent: Boundable : ClassTag](rdd: MultibandTileLayerRDD[K], bandCount: Int, path: String, zLevel: Int = 6, cropBounds: Option[Extent] = None, formatOptions: GTiffOptions = new GTiffOptions): java.util.List[(String, Extent)] = {
506+
def saveRDDGeneric[K: SpatialComponent: Boundable : ClassTag](rdd: MultibandTileLayerRDD[K], bandCount: Int, path: String, zLevel: Int = 6, cropBounds: Option[Extent] = None, formatOptions: GTiffOptions = new GTiffOptions): JList[(String, Extent)] = {
507507
val preProcessResult: (GridBounds[Int], Extent, RDD[(K, MultibandTile)] with Metadata[TileLayerMetadata[K]]) = preProcess(rdd,cropBounds)
508508
val gridBounds: GridBounds[Int] = preProcessResult._1
509509
val croppedExtent: Extent = preProcessResult._2
@@ -840,7 +840,7 @@ package object geotiff {
840840
cropBounds: Option[Map[String, Double]],
841841
cropDimensions: Option[ArrayList[Int]],
842842
compression: Compression)
843-
: java.util.List[(String, Extent)] = {
843+
: JList[(String, Extent)] = {
844844
val features = TileGrid.computeFeaturesForTileGrid(tileGrid, ProjectedExtent(rdd.metadata.extent,rdd.metadata.crs))
845845

846846
def newFilePath(path: String, tileId: String) = {
@@ -951,47 +951,22 @@ package object geotiff {
951951
polygons: ProjectedPolygons,
952952
sampleNames: JList[String],
953953
compression: Compression,
954-
): JList[(String, String, Extent)] =
955-
saveSamples(rdd, path, polygons, sampleNames, compression, None)
956-
957-
def saveSamples(rdd: MultibandTileLayerRDD[SpaceTimeKey],
958-
path: String,
959-
polygons: ProjectedPolygons,
960-
sampleNames: JList[String],
961-
compression: Compression,
962-
filenamePrefix: Option[String],
963-
): JList[(String, String, Extent)] = {
954+
filenamePrefix: Option[String] = None,
955+
): JList[Item] = {
964956
val reprojected = ProjectedPolygons.reproject(polygons, rdd.metadata.crs)
965957
val features = sampleNames.asScala.zip(reprojected.polygons)
966-
groupByFeatureAndWriteToTiff(rdd, Option.empty, features, path, Option.empty, compression, filenamePrefix)
958+
groupByFeatureAndWriteToTiff(rdd, cropBounds = None, features, path, cropDimensions = None, compression, filenamePrefix)
967959
}
968960

969961
def saveStitchedTileGridTemporal(rdd: MultibandTileLayerRDD[SpaceTimeKey],
970962
path: String,
971963
tileGrid: String,
972964
compression: Compression,
973-
filenamePrefix: Option[String],
974-
): java.util.List[(String, String, Extent)] =
975-
geotrellis.geotiff.saveStitchedTileGridTemporal(rdd, path, tileGrid, Option.empty, Option.empty, compression, filenamePrefix)
976-
977-
def saveStitchedTileGridTemporal(rdd: MultibandTileLayerRDD[SpaceTimeKey],
978-
path: String,
979-
tileGrid: String,
980-
compression: Compression,
981-
): java.util.List[(String, String, Extent)] =
982-
geotrellis.geotiff.saveStitchedTileGridTemporal(rdd, path, tileGrid, Option.empty, Option.empty, compression)
983-
984-
def saveStitchedTileGridTemporal(rdd: MultibandTileLayerRDD[SpaceTimeKey],
985-
path: String,
986-
tileGrid: String,
987-
cropBounds: Option[Map[String, Double]],
988-
cropDimensions: Option[ArrayList[Int]],
989-
compression: Compression,
990965
filenamePrefix: Option[String] = None,
991-
): java.util.List[(String, String, Extent)] = {
966+
): JList[Item] = {
992967
val features = TileGrid.computeFeaturesForTileGrid(tileGrid, ProjectedExtent(rdd.metadata.extent, rdd.metadata.crs))
993-
.map { case (s, extent) => (s, extent.toPolygon()) }
994-
groupByFeatureAndWriteToTiff(rdd, cropBounds, features, path, cropDimensions, compression, filenamePrefix)
968+
.map { case (name, extent) => (name, extent.toPolygon()) }
969+
groupByFeatureAndWriteToTiff(rdd, cropBounds = None, features, path, cropDimensions = None, compression, filenamePrefix)
995970
}
996971

997972
private def groupByFeatureAndWriteToTiff(rdd: MultibandTileLayerRDD[SpaceTimeKey],
@@ -1001,15 +976,15 @@ package object geotiff {
1001976
cropDimensions: Option[ArrayList[Int]],
1002977
compression: Compression,
1003978
filenamePrefix: Option[String] = None,
1004-
): java.util.List[(String, String, Extent)] = {
979+
): JList[Item] = {
1005980
val featuresBC: Broadcast[Seq[(String, Geometry)]] = SparkContext.getOrCreate().broadcast(features)
1006981

1007982
val croppedExtent = cropBounds.map(toExtent)
1008983

1009984
val layout = rdd.metadata.layout
1010985
val crs = rdd.metadata.crs
1011986

1012-
rdd
987+
val ret = rdd
1013988
.flatMap { case (key, tile) => featuresBC.value
1014989
.filter { case (_, geometry) => layout.mapTransform.keysForGeometry(geometry) contains key.spatialKey }
1015990
.map { case (name, geometry) => ((name, (geometry, key.time)), (key.spatialKey, tile)) }
@@ -1020,10 +995,16 @@ package object geotiff {
1020995
val filePath = Paths.get(path).resolve(filename).toString
1021996
val timestamp = time format DateTimeFormatter.ISO_ZONED_DATE_TIME
1022997
(stitchAndWriteToTiff(tiles, filePath, layout, crs, geometry, croppedExtent, cropDimensions, compression).correctPath,
1023-
timestamp, geometry.extent)
998+
timestamp, geometry.extent, name)
1024999
}
10251000
.collect()
1026-
.toList.asJava
1001+
1002+
val items = for {
1003+
(path, timestamp, extent, name) <- ret
1004+
} yield Item(id = f"${UUID.randomUUID()}_${timestamp}_$name", datetime = timestamp, bbox = extent,
1005+
assets = Collections.singletonMap("openEO", Asset(path)))
1006+
1007+
items.toList.asJava
10271008
}
10281009

10291010
private[geotrellis] case class GeoTiffResultObject(correctPath: String, fileExists: Boolean, gdalInfoPath: Option[String])

‎openeo-geotrellis/src/main/scala/org/openeo/geotrellis/stac/Asset.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ package org.openeo.geotrellis.stac
22

33
import java.util
44

5-
case class Asset(path: String, bandIndices: util.List[Int])
5+
case class Asset(path: String, bandIndices: util.List[Int] = null)

‎openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/TileGridTest.scala

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import org.junit.jupiter.api.{BeforeAll, Test}
1212
import org.junit.{AfterClass, Assert}
1313
import org.openeo.geotrellis.LayerFixtures.rgbLayerProvider
1414
import org.openeo.geotrellis.png.PngTest
15+
import org.openeo.geotrellis.stac.Item
1516
import org.openeo.geotrellis.tile_grid.TileGrid
1617
import org.openeo.geotrellis.{LayerFixtures, geotiff}
1718

@@ -148,7 +149,12 @@ class TileGridTest {
148149
("/tmp/openEO_2020-04-05Z_31UDS_2_5.tif", isoFormattedDate)
149150
)
150151

151-
Assert.assertEquals(expectedTiles, tiles.asScala.map { case (path, timestamp, _) => (path, timestamp) }.toSet)
152+
val actualTiles = for {
153+
item <- tiles.asScala
154+
asset <- item.assets.values().asScala
155+
} yield (asset.path, item.datetime)
156+
157+
Assert.assertEquals(expectedTiles, actualTiles.toSet)
152158
}
153159

154160
@Test
@@ -168,7 +174,12 @@ class TileGridTest {
168174
("/tmp/testPrefix_2020-04-05Z_31UDS_2_5.tif", isoFormattedDate)
169175
)
170176

171-
Assert.assertEquals(expectedTiles, tiles.asScala.map { case (path, timestamp, _) => (path, timestamp) }.toSet)
177+
val actualTiles = for {
178+
item <- tiles.asScala
179+
asset <- item.assets.values().asScala
180+
} yield (asset.path, item.datetime)
181+
182+
Assert.assertEquals(expectedTiles, actualTiles.toSet)
172183
}
173184

174185
@Test

‎openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/WriteRDDToGeotiffTest.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -547,7 +547,7 @@ class WriteRDDToGeotiffTest {
547547

548548
val ret = saveSamples(tileLayerRDD, outDir.toString, tiltedRectangle, sampleNames,
549549
DeflateCompression(BEST_COMPRESSION))
550-
assertTrue(ret.get(0)._2.contains("T"))
550+
assertTrue(ret.get(0).datetime.contains("T"))
551551
}
552552

553553
@Test

0 commit comments

Comments
 (0)