diff --git a/docker-compose.yml b/docker-compose.yml index 304b90f..5b5a0e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -175,7 +175,7 @@ services: aliases: - "nfcomposes3" ports: - - "127.0.0.1:${SEAWEEDFS_DEV_OUTSIDE_PORT:-8001}:8000" + - "127.0.0.1:${SEAWEEDFS_DEV_OUTSIDE_PORT:-8000}:8000" volumes: - "./devenv/s3_config/s3.json:/etc/seaweedfs/s3.json" - "seaweedfs_buckets:/data" diff --git a/skipper/skipper/dataseries/storage/contract/base.py b/skipper/skipper/dataseries/storage/contract/base.py index 3883624..7d53600 100644 --- a/skipper/skipper/dataseries/storage/contract/base.py +++ b/skipper/skipper/dataseries/storage/contract/base.py @@ -16,7 +16,7 @@ from rest_framework.generics import get_object_or_404 from rest_framework.request import Request from rest_framework.serializers import SerializerMetaclass -from typing import Dict, Any, Optional, Union, Tuple +from typing import Dict, Any, Optional, Union, Tuple, List from uuid import UUID from skipper.core.serializers.base import BaseSerializer @@ -244,7 +244,7 @@ def impl_internal_to_representation( self, data_point: Any, data_series: DataSeries, - external_id_as_dimension_identifier: bool + external_id_as_dimension_identifier: bool, ) -> Dict[str, Any]: raise NotImplementedError() diff --git a/skipper/skipper/dataseries/storage/contract/view.py b/skipper/skipper/dataseries/storage/contract/view.py index 04f5714..c550082 100644 --- a/skipper/skipper/dataseries/storage/contract/view.py +++ b/skipper/skipper/dataseries/storage/contract/view.py @@ -53,6 +53,8 @@ def get_changes_since(self) -> Optional[datetime.datetime]: ... def should_include_versions(self) -> bool: ... + def get_include_in_payload(self) -> Optional[List[str]]: ... + class StorageViewAdapter(metaclass=ABCMeta): """ diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/queries/create_view.py b/skipper/skipper/dataseries/storage/dynamic_sql/queries/create_view.py index 81cfafb..54bec8c 100644 --- a/skipper/skipper/dataseries/storage/dynamic_sql/queries/create_view.py +++ b/skipper/skipper/dataseries/storage/dynamic_sql/queries/create_view.py @@ -86,6 +86,7 @@ def create_view_as( data_sql = data_series_as_sql_table( data_series, + include_in_payload=None, data_series_query_info=data_series_query_info, resolve_dimension_external_ids=identify_dimensions_by_external_id ) diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/queries/display.py b/skipper/skipper/dataseries/storage/dynamic_sql/queries/display.py index f1ad0fa..3bf93dc 100644 --- a/skipper/skipper/dataseries/storage/dynamic_sql/queries/display.py +++ b/skipper/skipper/dataseries/storage/dynamic_sql/queries/display.py @@ -80,6 +80,7 @@ def render_pagination_data_select(include_pagination_data: bool) -> str: def _data_series_as_sql_table( + include_in_payload: Optional[List[str]], payload_as_json: bool, point_in_time: bool, changes_since: bool, @@ -101,6 +102,9 @@ def _data_series_as_sql_table( and len(_data_series_query_info.dimensions.keys()) > 0 all_select_infos = select_infos(_data_series_query_info) + if include_in_payload is not None: + all_select_infos = [select_info for select_info in all_select_infos if select_info.unescaped_display_id in include_in_payload] + central_table_sql = single_data_series_as_sql_table( all_select_infos=all_select_infos, data_series_query_info=_data_series_query_info, @@ -222,6 +226,7 @@ def _data_series_as_sql_table( def data_series_as_sql_table( data_series: DataSeries, + include_in_payload: List[str], payload_as_json: bool = False, point_in_time: bool = False, changes_since: bool = False, @@ -229,7 +234,7 @@ def data_series_as_sql_table( filter_str: str = '', resolve_dimension_external_ids: bool = False, data_series_query_info: Optional[DataSeriesQueryInfo] = None, - use_materialized: Optional[bool] = None + use_materialized: Optional[bool] = None, ) -> str: _data_series_query_info: DataSeriesQueryInfo if data_series_query_info is None: @@ -238,6 +243,7 @@ def data_series_as_sql_table( _data_series_query_info = data_series_query_info sql = _data_series_as_sql_table( + include_in_payload=include_in_payload, payload_as_json=payload_as_json, point_in_time=point_in_time, changes_since=changes_since, diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/queries/materialize_view.py b/skipper/skipper/dataseries/storage/dynamic_sql/queries/materialize_view.py index 742d21b..20d9294 100644 --- a/skipper/skipper/dataseries/storage/dynamic_sql/queries/materialize_view.py +++ b/skipper/skipper/dataseries/storage/dynamic_sql/queries/materialize_view.py @@ -56,7 +56,7 @@ def materialize_view_as(schema_name: str, overwrite: bool, data_series_id: str, ) if not exists_already: query = f"""CREATE MATERIALIZED VIEW IF NOT EXISTS {schema_name}.{escaped_view_name} - AS {data_series_as_sql_table(data_series)}""" + AS {data_series_as_sql_table(data_series, include_in_payload=None)}""" query_params: Dict[str, Any] = {select_info.payload_variable_name: select_info.unescaped_display_id for select_info in select_infos(compute_data_series_query_info(data_series))} cursor.execute( diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/serializers/modification.py b/skipper/skipper/dataseries/storage/dynamic_sql/serializers/modification.py index 10264f9..73722ac 100644 --- a/skipper/skipper/dataseries/storage/dynamic_sql/serializers/modification.py +++ b/skipper/skipper/dataseries/storage/dynamic_sql/serializers/modification.py @@ -41,10 +41,10 @@ def impl_internal_to_representation( query_params: Dict[str, Any] = {select_info.payload_variable_name: select_info.unescaped_display_id for select_info in select_infos(self.get_data_series_children_query_info())} - _view = self.context.get('view') query_str = data_series_as_sql_table( data_series=data_series, + include_in_payload=None, payload_as_json=True, point_in_time=self.point_in_time is not None, include_versions=self.should_include_versions, diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/storage_view_adapter.py b/skipper/skipper/dataseries/storage/dynamic_sql/storage_view_adapter.py index 7413e77..8823a4a 100644 --- a/skipper/skipper/dataseries/storage/dynamic_sql/storage_view_adapter.py +++ b/skipper/skipper/dataseries/storage/dynamic_sql/storage_view_adapter.py @@ -93,6 +93,7 @@ def precompute_filter_part( def raw_display_data_point_query( + include_in_payload: Optional[List[str]], filter_value: Dict[str, Any], data_series: DataSeries, external_id_as_dimension_identifier: bool, @@ -205,6 +206,7 @@ def raw_display_data_point_query( query_params['changes_since'] = changes_since query_str = data_series_as_sql_table( + include_in_payload=include_in_payload, data_series=data_series_obj, payload_as_json=True, point_in_time=is_point_in_time, @@ -255,6 +257,7 @@ def access_object( return obj raw_query = raw_display_data_point_query( + include_in_payload=view.get_include_in_payload(), data_point_id=data_point_id, filter_value=view.get_filter_value(), data_series=view.access_data_series(), @@ -332,6 +335,7 @@ def encode_last_id_for_pagination(self, view: BaseDataSeries_DataPointViewSet, d def get_next_page_query_for_pagination(self, view: BaseDataSeries_DataPointViewSet, last_query: str, limit: int, request: HttpRequest) -> RawQuerySet: # type: ignore return raw_display_data_point_query( + include_in_payload=view.get_include_in_payload(), # we actually don't need anything in the payload here start_object=last_query, limit=limit, filter_value=view.get_filter_value(), @@ -353,6 +357,7 @@ def get_prev_page_query_for_pagination( ) -> Optional[RawQuerySet]: # type: ignore # FIXME: this could be a simpler query without as many joins return raw_display_data_point_query( + include_in_payload=view.get_include_in_payload(), # we actually don't need anything in the payload here start_object=last_query, reverse=True, limit=limit, diff --git a/skipper/skipper/dataseries/tests/data_point/test_data_point_non_historical_lifecycle.py b/skipper/skipper/dataseries/tests/data_point/test_data_point_non_historical_lifecycle.py index 133a829..33ff69e 100644 --- a/skipper/skipper/dataseries/tests/data_point/test_data_point_non_historical_lifecycle.py +++ b/skipper/skipper/dataseries/tests/data_point/test_data_point_non_historical_lifecycle.py @@ -156,6 +156,7 @@ class FakedFile: ) _sql = data_series_as_sql_table( + include_in_payload=None, data_series=_data_series_obj, payload_as_json=True, point_in_time=False, diff --git a/skipper/skipper/dataseries/views/datapoint/crud.py b/skipper/skipper/dataseries/views/datapoint/crud.py index b90e0a3..6184a5b 100644 --- a/skipper/skipper/dataseries/views/datapoint/crud.py +++ b/skipper/skipper/dataseries/views/datapoint/crud.py @@ -198,6 +198,17 @@ def get_filter_value(self) -> Dict[str, Any]: else: filter_value = {} return filter_value + + def get_include_in_payload(self) -> Optional[List[str]]: + if 'include_in_payload' in self.request.GET: + _include_in_payload_str = self.request.GET['include_in_payload'] + if _include_in_payload_str != None: + return list(_include_in_payload_str.split(',')) + else: + return None + else: + return None + def get_external_ids(self) -> Optional[List[str]]: return self.request.GET.getlist('external_id') if 'external_id' in self.request.GET else None # type: ignore @@ -275,6 +286,7 @@ def get_description_string(self) -> str: - count[=true]
- external_id= (repeatable)
- identify_dimensions_by_external_id[=true]
+ - include_in_payload= (comma separated list of dimension/fact external ids to include in the payload)
""" if _history: doc_string = f""" diff --git a/skipper/skipper/dataseries/views/metamodel/cube_sql.py b/skipper/skipper/dataseries/views/metamodel/cube_sql.py index f80b3bc..4895ce1 100644 --- a/skipper/skipper/dataseries/views/metamodel/cube_sql.py +++ b/skipper/skipper/dataseries/views/metamodel/cube_sql.py @@ -121,6 +121,7 @@ def get(self, request: Request, **kwargs: str) -> HttpResponse: sql_template = data_series_as_sql_table( data_series, + include_in_payload=None, payload_as_json=payload_as_json(request), resolve_dimension_external_ids=use_external_id_as_dimension_identifier( cast(Dict[str, Any], request.GET)