Add DataRecipe (#18892)

Co-authored-by: thomas <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

src/lightning/data/__init__.py

@@ -1,12 +1,10 @@
 from lightning.data.datasets import LightningDataset, LightningIterableDataset
-from lightning.data.streaming.dataloader import StreamingDataLoader
 from lightning.data.streaming.dataset import StreamingDataset
-from lightning.data.streaming.dataset_optimizer import DatasetOptimizer
+from lightning.data.streaming.map import map
 
 __all__ = [
     "LightningDataset",
     "StreamingDataset",
-    "StreamingDataLoader",
     "LightningIterableDataset",
-    "DatasetOptimizer",
+    "map",
 ]

src/lightning/data/streaming/__init__.py

@@ -12,7 +12,8 @@
 # limitations under the License.
 
 from lightning.data.streaming.cache import Cache
-from lightning.data.streaming.dataloader import StreamingDataLoader
-from lightning.data.streaming.dataset_optimizer import DatasetOptimizer
+from lightning.data.streaming.data_processor import DataChunkRecipe, DataProcessor, DataTransformRecipe
+from lightning.data.streaming.dataset import StreamingDataset
+from lightning.data.streaming.item_loader import TokensLoader
 
-__all__ = ["Cache", "DatasetOptimizer", "StreamingDataLoader"]
+__all__ = ["Cache", "DataProcessor", "StreamingDataset", "DataTransformRecipe", "DataChunkRecipe", "TokensLoader"]