Skip to content

Commit

Permalink
Improve consistency from comments.
Browse files Browse the repository at this point in the history
  • Loading branch information
delucchi-cmu committed Jan 30, 2024
1 parent c9143ee commit a8cb2b9
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 17 deletions.
25 changes: 11 additions & 14 deletions src/hipscat/catalog/association_catalog/partition_join_info.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
"""Container class to hold primary-to-join partition metadata"""
from __future__ import annotations

from typing import Any, Dict, List, Union
from typing import Dict, List

import numpy as np
import pandas as pd
import pyarrow as pa
from typing_extensions import Self

from hipscat.catalog.partition_info import PartitionInfo
from hipscat.io import FilePointer, file_io, paths
Expand Down Expand Up @@ -113,15 +113,12 @@ def write_to_csv(self, catalog_path: FilePointer, storage_options: dict = None):
primary_pixels = self.primary_to_join_map().keys()
partition_info_pointer = paths.get_partition_info_pointer(catalog_path)
partition_info = PartitionInfo.from_healpix(primary_pixels)
file_io.write_dataframe_to_csv(
partition_info.as_dataframe(),
partition_info_pointer,
index=False,
storage_options=storage_options,
partition_info.write_to_file(
partition_info_file=partition_info_pointer, storage_options=storage_options
)

@classmethod
def read_from_dir(cls, catalog_base_dir: FilePointer, storage_options: dict = None) -> Self:
def read_from_dir(cls, catalog_base_dir: FilePointer, storage_options: dict = None) -> PartitionJoinInfo:
"""Read partition join info from a file within a hipscat directory.
This will look for a `_metadata` file, and if not found, will look for
Expand All @@ -132,7 +129,7 @@ def read_from_dir(cls, catalog_base_dir: FilePointer, storage_options: dict = No
storage_options (dict): dictionary that contains abstract filesystem credentials
Returns:
A `PartitionInfo` object with the data from the file
A `PartitionJoinInfo` object with the data from the file
Raises:
FileNotFoundError: if neither desired file is found in the catalog_base_dir
Expand All @@ -155,8 +152,8 @@ def read_from_dir(cls, catalog_base_dir: FilePointer, storage_options: dict = No

@classmethod
def read_from_file(
cls, metadata_file: FilePointer, strict=False, storage_options: Union[Dict[Any, Any], None] = None
) -> Self:
cls, metadata_file: FilePointer, strict=False, storage_options: dict = None
) -> PartitionJoinInfo:
"""Read partition join info from a `_metadata` file to create an object
Args:
Expand Down Expand Up @@ -231,8 +228,8 @@ def read_from_file(

@classmethod
def read_from_csv(
cls, partition_join_info_file: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
) -> Self:
cls, partition_join_info_file: FilePointer, storage_options: dict = None
) -> PartitionJoinInfo:
"""Read partition join info from a `partition_join_info.csv` file to create an object
Args:
Expand All @@ -246,7 +243,7 @@ def read_from_csv(
partition_join_info_file, storage_options=storage_options
):
raise FileNotFoundError(
f"No partition info found where expected: {str(partition_join_info_file)}"
f"No partition join info found where expected: {str(partition_join_info_file)}"
)

data_frame = file_io.load_csv_to_pandas(partition_join_info_file, storage_options=storage_options)
Expand Down
7 changes: 5 additions & 2 deletions src/hipscat/catalog/partition_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,17 @@ def get_highest_order(self) -> int:
max_pixel = np.max(self.pixel_list)
return max_pixel.order

def write_to_file(self, partition_info_file: FilePointer):
def write_to_file(self, partition_info_file: FilePointer, storage_options: dict = None):
"""Write all partition data to CSV file.
Args:
partition_info_file: FilePointer to where the `partition_info.csv`
file will be written
storage_options (dict): dictionary that contains abstract filesystem credentials
"""
file_io.write_dataframe_to_csv(self.as_dataframe(), partition_info_file, index=False)
file_io.write_dataframe_to_csv(
self.as_dataframe(), partition_info_file, index=False, storage_options=storage_options
)

def write_to_metadata_files(self, catalog_path: FilePointer, storage_options: dict = None):
"""Generate parquet metadata, using the known partitions.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def test_read_from_metadata_fail(tmp_path):
PartitionJoinInfo.read_from_file(metadata_filename, strict=True)


def test_load_partition_info_from_dir_fail(tmp_path):
def test_load_partition_join_info_from_dir_fail(tmp_path):
empty_dataframe = pd.DataFrame()
metadata_filename = os.path.join(tmp_path, "empty_metadata.parquet")
empty_dataframe.to_parquet(metadata_filename)
Expand Down

0 comments on commit a8cb2b9

Please sign in to comment.