Skip to content

Commit

Permalink
Remove neptune status
Browse files Browse the repository at this point in the history
  • Loading branch information
kgodlewski committed Jan 15, 2025
1 parent 67463b8 commit 1121313
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 106 deletions.
26 changes: 2 additions & 24 deletions src/neptune_scale/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,40 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path
from typing import Optional

import click

from neptune_scale.cli import (
misc,
sync,
)
from neptune_scale.storage.operations import DATA_DIR
from neptune_scale.cli import sync
from neptune_scale.util.styles import ensure_style_detected


@click.group()
@click.option(
"--path",
type=click.Path(exists=True, file_okay=False),
help="Path containing Neptune data. Defaults to $CWD/.neptune",
)
@click.pass_context
def main(ctx: click.Context, path: Optional[str]) -> None:
def main() -> None:
ensure_style_detected()

ctx.ensure_object(dict)
if path is None:
neptune_dir = Path(".") / DATA_DIR
else:
neptune_dir = Path(path)

ctx.obj["neptune_dir"] = neptune_dir


main.add_command(sync.sync)
main.add_command(misc.status)

if __name__ == "__main__":
main()
37 changes: 0 additions & 37 deletions src/neptune_scale/cli/misc.py

This file was deleted.

56 changes: 17 additions & 39 deletions src/neptune_scale/cli/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
from neptune_scale.cli.util import (
format_duration,
format_local_run,
is_neptune_dir,
)
from neptune_scale.exceptions import (
NeptuneRunDuplicate,
Expand Down Expand Up @@ -224,11 +223,10 @@ def _verify_fork_parent(local_run: LocalRun, parent_must_exist: bool) -> None:
This can happen if the parent run was created in offline mode and is not yet
synced to the Neptune backend.
Run {bash}neptune status -v{end} to list all local runs, and manually sync the parent
run first using {bash}neptune sync <filename>{end}.
Before syncing this run, you need manually sync the parent run first using {bash}neptune sync <filename>{end}.
Alternatively you can run {bash}neptune sync --sync-no-parent{end} to ignore this error,
and proceed with syncing the without the parent run.
and proceed with syncing without the parent run.
"""
raise Exception(msg.format(fork_run_id=local_run.fork_run_id, **STYLES))

Expand Down Expand Up @@ -297,7 +295,6 @@ def sync_file(

@click.command()
@click.argument("filename", type=click.Path(exists=True, dir_okay=False), metavar="<filename>")
@click.option("-k", "--keep", is_flag=True, help="Do not delete the local copy of the data after sync completes")
@click.option("--api-token", type=str, help="Your Neptune API token")
@click.option(
"--allow-non-increasing-step",
Expand All @@ -306,12 +303,9 @@ def sync_file(
"are stuck on a metric being sent multiple times. ",
)
@click.option("--sync-no-parent", is_flag=True, help="Do not require the parent run to exist when syncing forked runs")
@click.pass_context
def sync(
ctx: click.Context,
filename: Optional[str],
filename: str,
api_token: Optional[str],
keep: bool,
allow_non_increasing_step: bool,
sync_no_parent: bool,
) -> None:
Expand All @@ -322,41 +316,25 @@ def sync(
means that the process can be interrupted and resumed later.
"""

neptune_dir = ctx.obj["neptune_dir"]
if not is_neptune_dir(neptune_dir):
logger.error(f"No Neptune data found at {neptune_dir}")
sys.exit(1)

t0 = time.monotonic()
if filename:
files = [Path(filename)]
# For the time being we're not allowing to sync the entire directory, because of the potential for
# users to make mistakes in terms of forked runs and missing parents.
# else:
# files = [run.path for run in list_runs(ctx.obj["neptune_dir"])]

if not files:
logger.info("No data to sync")
sys.exit(1)

logger.info("Starting `neptune sync`")
path = Path(filename)

error = False

for path in files:
try:
sync_file(
path,
api_token=api_token,
allow_non_increasing_step=allow_non_increasing_step,
parent_must_exist=not sync_no_parent,
)
if not keep:
logger.info(f"Removing file {path}")
path.unlink()
except Exception as e:
logger.error("An error occurred during `neptune sync`: %s", e)
error = True
try:
sync_file(
path,
api_token=api_token,
allow_non_increasing_step=allow_non_increasing_step,
parent_must_exist=not sync_no_parent,
)

logger.info(f"Removing file {path}")
path.unlink()
except Exception as e:
logger.error("An error occurred during `neptune sync`: %s", e)
error = True

duration = format_duration(int(time.monotonic() - t0))
if error:
Expand Down
10 changes: 4 additions & 6 deletions src/neptune_scale/cli/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,27 +50,25 @@ def format_duration(seconds: int) -> str:
return " ".join(parts)


def format_local_run(run: LocalRun, with_run_id: bool = True) -> str:
def format_local_run(run: LocalRun) -> str:
if run.operation_count:
pct = round(run.last_synced_operation / run.operation_count * 100, 2)
else:
pct = 100.0

parts = [
f"Run ID: {run.run_id}",
f"Project: {run.project}",
f"Synced: {run.last_synced_operation}/{run.operation_count} ({pct}%)",
f"Synced: {run.last_synced_operation}/{run.operation_count} operations ({pct}%)",
f"Created at: {run.creation_time}",
]

if with_run_id:
parts.insert(0, f"Run ID: {run.run_id}")

if run.experiment_name:
parts.append(f"Experiment: {run.experiment_name}")
if run.fork_run_id:
parts.append(f"Forked from `{run.fork_run_id}` at step {run.fork_step}")

parts.append(f"Path: {run.path}")
# parts.append(f"Path: {run.path}")

text = "\n".join(f" - {part}" for part in parts)
return text

0 comments on commit 1121313

Please sign in to comment.