Add telemetry to API (#451)

Adds telemetry event logging to `mlem.api` commands. Some Qs:

1) Should we log API event if it was used by CLI command (in this PR we
do not)
2) What if one API command calls some nested API command? Should both
events be logged, or only topmost? (kinda the same as 1) if you really
think about it)
3) What parameters we want to log from each command (will post
suggestions as a separate comment)
4) Should `save, load, load_meta` be added too? It can show interesting
stuff like what types of models and datasets are used more frequently
5) Need a framework to enforce same extra fields for similar API/CLI
calls, but that depends on previous Qs

mlem/api/commands.py

@@ -23,7 +23,7 @@
 )
 from mlem.core.import_objects import ImportAnalyzer, ImportHook
 from mlem.core.meta_io import Location, get_fs
-from mlem.core.metadata import load_meta, save
+from mlem.core.metadata import load_meta, log_meta_params, save
 from mlem.core.objects import (
     MlemBuilder,
     MlemData,
@@ -36,6 +36,7 @@
 from mlem.runtime.client import Client
 from mlem.runtime.interface import ModelInterface
 from mlem.runtime.server import Server
+from mlem.telemetry import api_telemetry, telemetry
 from mlem.ui import (
     EMOJI_APPLY,
     EMOJI_COPY,
@@ -47,6 +48,7 @@
 )
 
 
+@api_telemetry
 def apply(
     model: Union[str, MlemModel, Any],
     *data: Union[str, MlemData, Any],
@@ -76,6 +78,7 @@ def apply(
         model = get_model_meta(model)
     else:
         model = MlemModel.from_obj(model)
+    log_meta_params(model, add_object_type=False)
     w = model.model_type
     try:
         resolved_method = w.resolve_method(method)
@@ -103,6 +106,7 @@ def apply(
     return save(res, output, project=target_project)
 
 
+@api_telemetry
 def apply_remote(
     client: Union[str, Client],
     *data: Union[str, MlemData, Any],
@@ -129,6 +133,7 @@ def apply_remote(
             Otherwise returns None.
     """
     client = ensure_mlem_object(Client, client, **client_kwargs)
+    telemetry.log_param("client_type", client.type)
     if method is not None:
         try:
             resolved_method = getattr(client, method)
@@ -148,6 +153,7 @@ def apply_remote(
     return save(res, output, project=target_project)
 
 
+@api_telemetry
 def clone(
     path: str,
     target: str,
@@ -195,6 +201,7 @@ def clone(
     )
 
 
+@api_telemetry
 def init(path: str = ".") -> None:
     """Creates MLEM config in `path`
 
@@ -211,8 +218,6 @@ def init(path: str = ".") -> None:
             f"{posixpath.abspath(path)} already exists, no need to run `mlem init` again"
         )
     else:
-        from mlem.telemetry import telemetry
-
         echo(
             color("███╗   ███╗", "#13ADC7")
             + color("██╗     ", "#945DD5")
@@ -261,6 +266,7 @@ def init(path: str = ".") -> None:
         )
 
 
+@api_telemetry
 def link(
     source: Union[str, MlemObject],
     source_project: Optional[str] = None,
@@ -304,6 +310,7 @@ def link(
     )
 
 
+@api_telemetry
 def build(
     builder: Union[str, MlemBuilder],
     model: Union[str, MlemModel],
@@ -320,11 +327,13 @@ def build(
         The result of the build, different for different builders.
     """
     model = get_model_meta(model, load_value=False)
-    return ensure_mlem_object(MlemBuilder, builder, **builder_kwargs).build(
-        model
-    )
+    builder = ensure_mlem_object(MlemBuilder, builder, **builder_kwargs)
+    log_meta_params(model, add_object_type=False)
+    log_meta_params(builder, add_object_type=False)
+    return builder.build(model)
 
 
+@api_telemetry
 def serve(
     model: Union[str, MlemModel],
     server: Union[Server, str],
@@ -343,10 +352,13 @@ def serve(
     model = get_model_meta(model, load_value=True)
 
     server_obj = ensure_mlem_object(Server, server, **server_kwargs)
+    telemetry.log_param("server_type", server_obj.type)
+    log_meta_params(model)
     echo(f"Starting {server_obj.type} server...")
     server_obj.start(ModelInterface.from_model(model))
 
 
+@api_telemetry
 def import_object(
     path: str,
     project: Optional[str] = None,
@@ -386,6 +398,7 @@ def import_object(
         )
     else:
         meta = ImportAnalyzer.analyze(loc, copy_data=copy_data)
+    log_meta_params(meta, add_object_type=True)
     if target is not None:
         meta.dump(
             target,
@@ -395,6 +408,7 @@ def import_object(
     return meta
 
 
+@api_telemetry
 def deploy(
     deploy_meta_or_path: Union[MlemDeployment, str],
     model: Union[MlemModel, str],
@@ -459,8 +473,10 @@ def deploy(
     if update:
         pass  # todo update from deploy_args and env_args
     # ensuring links are working
+    log_meta_params(deploy_meta)
     deploy_meta.get_env()
     model_meta = get_model_meta(model)
+    log_meta_params(model_meta)
 
     deploy_meta.check_unchanged()
     deploy_meta.deploy(model_meta)

mlem/cli/apply.py

@@ -27,6 +27,7 @@
     for_each_impl,
     lazy_class_docstring,
     make_not_required,
+    pass_api_log_params,
 )
 from mlem.core.data_type import DataAnalyzer
 from mlem.core.errors import UnsupportedDataBatchLoading
@@ -107,7 +108,9 @@ def apply(
             )
         meta = load_meta(model, project, rev, force_type=MlemModel)
 
-        result = apply(
+        result = pass_api_log_params(
+            apply
+        )(  # pylint: disable=too-many-function-args
             meta,
             data,
             method=method,
@@ -256,7 +259,9 @@ def run_apply_remote(
         load_value=True,
         force_type=MlemData,
     )
-    result = apply_remote(
+    result = pass_api_log_params(
+        apply_remote
+    )(  # pylint: disable=too-many-function-args
         client,
         data,
         method=method,

mlem/cli/build.py

@@ -19,6 +19,7 @@
     for_each_impl,
     lazy_class_docstring,
     make_not_required,
+    pass_api_log_params,
 )
 from mlem.core.metadata import load_meta
 from mlem.core.objects import MlemBuilder, MlemModel
@@ -44,7 +45,7 @@ def build_load(
 ):
     from mlem.api.commands import build
 
-    build(
+    pass_api_log_params(build)(
         config_arg(
             MlemBuilder,
             load,
@@ -79,7 +80,7 @@ def build_type(
     ):
         from mlem.api.commands import build
 
-        build(
+        pass_api_log_params(build)(
             config_arg(
                 MlemBuilder,
                 None,

mlem/cli/deployment.py

@@ -27,6 +27,7 @@
     for_each_impl,
     lazy_class_docstring,
     make_not_required,
+    pass_api_log_params,
     wrap_build_error,
 )
 from mlem.core.base import build_mlem_object
@@ -73,7 +74,7 @@ def deploy_run_callback(
     """
     from mlem.api.commands import deploy
 
-    deploy(
+    pass_api_log_params(deploy)(
         load,
         load_meta(
             model, project=model_project, rev=model_rev, force_type=MlemModel
@@ -125,7 +126,7 @@ def deploy_run_command(
                     file_conf=file_conf,
                     **__kwargs__,
                 ).dump(path, project=project)
-        deploy(
+        pass_api_log_params(deploy)(
             meta,
             load_meta(
                 model,

mlem/cli/import_object.py

@@ -8,6 +8,7 @@
     option_rev,
     option_target_project,
 )
+from mlem.cli.utils import pass_api_log_params
 from mlem.core.import_objects import ImportHook
 from mlem.utils.entrypoints import list_implementations
 
@@ -28,7 +29,7 @@ def import_object(
     """Create a `.mlem` metafile for a model or data in any file or directory."""
     from mlem.api.commands import import_object
 
-    import_object(
+    pass_api_log_params(import_object)(
         uri,
         project=project,
         rev=rev,

mlem/cli/init.py

@@ -1,6 +1,7 @@
 from typer import Argument
 
 from mlem.cli.main import PATH_METAVAR, mlem_command
+from mlem.cli.utils import pass_api_log_params
 
 
 @mlem_command("init", section="common")
@@ -15,4 +16,4 @@ def init(
     """Initialize a MLEM project."""
     from mlem.api.commands import init
 
-    init(path)
+    pass_api_log_params(init)(path)

mlem/cli/link.py

@@ -8,6 +8,7 @@
     option_rev,
     option_target_project,
 )
+from mlem.cli.utils import pass_api_log_params
 
 
 @mlem_command("link", section="object")
@@ -43,7 +44,7 @@ def link(
     """
     from mlem.api.commands import link
 
-    link(
+    pass_api_log_params(link)(
         source=source,
         source_project=source_project,
         rev=rev,