Merge pull request #1 in WEBAPPS/latis from feature/LATIS-562-create-…

…a-catalog-dataset-based to dev

* commit 'c495d9b16b09158725a86927b4b321f27f1dae19':
  Named the "ds_name" attribute in CatalogGenerator's output dataset.
  Corrected a comment.
  Add entry: "reader.catalog.class = CatalogGenerator" to latis.properties
  CatalogGenerator now extends DatasetAccessor; organized imports.
  CatalogGenerator.scala now lives in latis.reader (not writer)
  CatalogGenerator can now get a list of datasets for a given mission.
  Starting to create a CatalogGenerator class.

src/main/resources/latis.properties

@@ -15,6 +15,9 @@
 # Access-Control-Allow-Origin
 cors.allow.origin = *
 
+#---- Define Readers ----------------------------------------------------------
+reader.catalog.class = CatalogGenerator
+
 #---- Define Writers ----------------------------------------------------------
 writer.asc.class = latis.writer.AsciiWriter
 

src/main/scala/latis/reader/CatalogGenerator.scala

@@ -0,0 +1,62 @@
+package latis.reader
+
+import scala.collection.mutable.ArrayBuffer
+import latis.dm.Dataset
+import latis.dm.Function
+import latis.dm.Index
+import latis.dm.Sample
+import latis.dm.Text
+import latis.ops.Operation
+import latis.util.FileUtils
+import latis.util.LatisProperties
+import latis.metadata.Metadata
+
+class CatalogGenerator extends DatasetAccessor {  
+
+  private lazy val datasetNames: List[String] = getListOfDatasets   
+  private lazy val catalogDataset: Dataset = generateCatalogDataset
+
+  /**
+   * Return the Dataset that this accessor is responsible for (catalogDataset).
+   */
+  override def getDataset(): Dataset = catalogDataset
+
+  /**
+   * Return the catalogDataset with the given Operations applied to it.
+   */
+  override def getDataset(ops: Seq[Operation]): Dataset = ops.reverse.foldRight(getDataset)(_(_))
+
+  override def close = {}
+
+  //---- Helper Functions ------------------------------------------------------------------------
+
+  /*
+   * Produce a list of the names of all datasets that live in 
+   * the "dataset.dir" as defined in latis.properties.
+   */
+  def getListOfDatasets: List[String] = {
+    val dsdir: String = LatisProperties.getOrElse("dataset.dir", "datasets")
+    val fileList = FileUtils.getListOfFiles(dsdir) 
+    val fileNames = ArrayBuffer[String]()
+
+    for (x <- fileList) {
+      fileNames += x.toString.substring(dsdir.length()+1, x.toString.length-5) //trim leading file path and ".tsml"
+    }
+
+    fileNames.toList
+  }
+
+  def getDatasetNames = datasetNames
+
+  /*
+   * Convert a list of dataset names into
+   * a dataset of the form i -> ds_name
+   */
+  def generateCatalogDataset = {
+    val samples = datasetNames.map(n => {
+      Sample(Index(), Text(Metadata("ds_name"), n))
+    })
+    Dataset(Function(samples))
+  }
+
+}

src/main/scala/latis/util/FileUtils.scala

@@ -66,4 +66,16 @@ object FileUtils {
     input.close
   }
 
+  /*
+   * Given a directory, return a list of files in that directory
+   */
+  def getListOfFiles(dir: String): List[File] = { 
+    val d = new File(dir)
+    if (d.exists && d.isDirectory) {
+      d.listFiles.filter(_.isFile).toList
+    } else {
+      List[File]()
+    }
+  }
+
 }