diff --git a/docker-compose.yml b/docker-compose.yml
index 304b90f..5b5a0e2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -175,7 +175,7 @@ services:
aliases:
- "nfcomposes3"
ports:
- - "127.0.0.1:${SEAWEEDFS_DEV_OUTSIDE_PORT:-8001}:8000"
+ - "127.0.0.1:${SEAWEEDFS_DEV_OUTSIDE_PORT:-8000}:8000"
volumes:
- "./devenv/s3_config/s3.json:/etc/seaweedfs/s3.json"
- "seaweedfs_buckets:/data"
diff --git a/skipper/skipper/dataseries/storage/contract/base.py b/skipper/skipper/dataseries/storage/contract/base.py
index 3883624..7d53600 100644
--- a/skipper/skipper/dataseries/storage/contract/base.py
+++ b/skipper/skipper/dataseries/storage/contract/base.py
@@ -16,7 +16,7 @@
from rest_framework.generics import get_object_or_404
from rest_framework.request import Request
from rest_framework.serializers import SerializerMetaclass
-from typing import Dict, Any, Optional, Union, Tuple
+from typing import Dict, Any, Optional, Union, Tuple, List
from uuid import UUID
from skipper.core.serializers.base import BaseSerializer
@@ -244,7 +244,7 @@ def impl_internal_to_representation(
self,
data_point: Any,
data_series: DataSeries,
- external_id_as_dimension_identifier: bool
+ external_id_as_dimension_identifier: bool,
) -> Dict[str, Any]:
raise NotImplementedError()
diff --git a/skipper/skipper/dataseries/storage/contract/view.py b/skipper/skipper/dataseries/storage/contract/view.py
index 04f5714..c550082 100644
--- a/skipper/skipper/dataseries/storage/contract/view.py
+++ b/skipper/skipper/dataseries/storage/contract/view.py
@@ -53,6 +53,8 @@ def get_changes_since(self) -> Optional[datetime.datetime]: ...
def should_include_versions(self) -> bool: ...
+ def get_include_in_payload(self) -> Optional[List[str]]: ...
+
class StorageViewAdapter(metaclass=ABCMeta):
"""
diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/queries/create_view.py b/skipper/skipper/dataseries/storage/dynamic_sql/queries/create_view.py
index 81cfafb..54bec8c 100644
--- a/skipper/skipper/dataseries/storage/dynamic_sql/queries/create_view.py
+++ b/skipper/skipper/dataseries/storage/dynamic_sql/queries/create_view.py
@@ -86,6 +86,7 @@ def create_view_as(
data_sql = data_series_as_sql_table(
data_series,
+ include_in_payload=None,
data_series_query_info=data_series_query_info,
resolve_dimension_external_ids=identify_dimensions_by_external_id
)
diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/queries/display.py b/skipper/skipper/dataseries/storage/dynamic_sql/queries/display.py
index f1ad0fa..3bf93dc 100644
--- a/skipper/skipper/dataseries/storage/dynamic_sql/queries/display.py
+++ b/skipper/skipper/dataseries/storage/dynamic_sql/queries/display.py
@@ -80,6 +80,7 @@ def render_pagination_data_select(include_pagination_data: bool) -> str:
def _data_series_as_sql_table(
+ include_in_payload: Optional[List[str]],
payload_as_json: bool,
point_in_time: bool,
changes_since: bool,
@@ -101,6 +102,9 @@ def _data_series_as_sql_table(
and len(_data_series_query_info.dimensions.keys()) > 0
all_select_infos = select_infos(_data_series_query_info)
+ if include_in_payload is not None:
+ all_select_infos = [select_info for select_info in all_select_infos if select_info.unescaped_display_id in include_in_payload]
+
central_table_sql = single_data_series_as_sql_table(
all_select_infos=all_select_infos,
data_series_query_info=_data_series_query_info,
@@ -222,6 +226,7 @@ def _data_series_as_sql_table(
def data_series_as_sql_table(
data_series: DataSeries,
+ include_in_payload: List[str],
payload_as_json: bool = False,
point_in_time: bool = False,
changes_since: bool = False,
@@ -229,7 +234,7 @@ def data_series_as_sql_table(
filter_str: str = '',
resolve_dimension_external_ids: bool = False,
data_series_query_info: Optional[DataSeriesQueryInfo] = None,
- use_materialized: Optional[bool] = None
+ use_materialized: Optional[bool] = None,
) -> str:
_data_series_query_info: DataSeriesQueryInfo
if data_series_query_info is None:
@@ -238,6 +243,7 @@ def data_series_as_sql_table(
_data_series_query_info = data_series_query_info
sql = _data_series_as_sql_table(
+ include_in_payload=include_in_payload,
payload_as_json=payload_as_json,
point_in_time=point_in_time,
changes_since=changes_since,
diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/queries/materialize_view.py b/skipper/skipper/dataseries/storage/dynamic_sql/queries/materialize_view.py
index 742d21b..20d9294 100644
--- a/skipper/skipper/dataseries/storage/dynamic_sql/queries/materialize_view.py
+++ b/skipper/skipper/dataseries/storage/dynamic_sql/queries/materialize_view.py
@@ -56,7 +56,7 @@ def materialize_view_as(schema_name: str, overwrite: bool, data_series_id: str,
)
if not exists_already:
query = f"""CREATE MATERIALIZED VIEW IF NOT EXISTS {schema_name}.{escaped_view_name}
- AS {data_series_as_sql_table(data_series)}"""
+ AS {data_series_as_sql_table(data_series, include_in_payload=None)}"""
query_params: Dict[str, Any] = {select_info.payload_variable_name: select_info.unescaped_display_id for
select_info in select_infos(compute_data_series_query_info(data_series))}
cursor.execute(
diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/serializers/modification.py b/skipper/skipper/dataseries/storage/dynamic_sql/serializers/modification.py
index 10264f9..73722ac 100644
--- a/skipper/skipper/dataseries/storage/dynamic_sql/serializers/modification.py
+++ b/skipper/skipper/dataseries/storage/dynamic_sql/serializers/modification.py
@@ -41,10 +41,10 @@ def impl_internal_to_representation(
query_params: Dict[str, Any] = {select_info.payload_variable_name: select_info.unescaped_display_id for
select_info in
select_infos(self.get_data_series_children_query_info())}
- _view = self.context.get('view')
query_str = data_series_as_sql_table(
data_series=data_series,
+ include_in_payload=None,
payload_as_json=True,
point_in_time=self.point_in_time is not None,
include_versions=self.should_include_versions,
diff --git a/skipper/skipper/dataseries/storage/dynamic_sql/storage_view_adapter.py b/skipper/skipper/dataseries/storage/dynamic_sql/storage_view_adapter.py
index 7413e77..8823a4a 100644
--- a/skipper/skipper/dataseries/storage/dynamic_sql/storage_view_adapter.py
+++ b/skipper/skipper/dataseries/storage/dynamic_sql/storage_view_adapter.py
@@ -93,6 +93,7 @@ def precompute_filter_part(
def raw_display_data_point_query(
+ include_in_payload: Optional[List[str]],
filter_value: Dict[str, Any],
data_series: DataSeries,
external_id_as_dimension_identifier: bool,
@@ -205,6 +206,7 @@ def raw_display_data_point_query(
query_params['changes_since'] = changes_since
query_str = data_series_as_sql_table(
+ include_in_payload=include_in_payload,
data_series=data_series_obj,
payload_as_json=True,
point_in_time=is_point_in_time,
@@ -255,6 +257,7 @@ def access_object(
return obj
raw_query = raw_display_data_point_query(
+ include_in_payload=view.get_include_in_payload(),
data_point_id=data_point_id,
filter_value=view.get_filter_value(),
data_series=view.access_data_series(),
@@ -332,6 +335,7 @@ def encode_last_id_for_pagination(self, view: BaseDataSeries_DataPointViewSet, d
def get_next_page_query_for_pagination(self, view: BaseDataSeries_DataPointViewSet, last_query: str, limit: int,
request: HttpRequest) -> RawQuerySet: # type: ignore
return raw_display_data_point_query(
+ include_in_payload=view.get_include_in_payload(), # we actually don't need anything in the payload here
start_object=last_query,
limit=limit,
filter_value=view.get_filter_value(),
@@ -353,6 +357,7 @@ def get_prev_page_query_for_pagination(
) -> Optional[RawQuerySet]: # type: ignore
# FIXME: this could be a simpler query without as many joins
return raw_display_data_point_query(
+ include_in_payload=view.get_include_in_payload(), # we actually don't need anything in the payload here
start_object=last_query,
reverse=True,
limit=limit,
diff --git a/skipper/skipper/dataseries/tests/data_point/test_data_point_non_historical_lifecycle.py b/skipper/skipper/dataseries/tests/data_point/test_data_point_non_historical_lifecycle.py
index 133a829..33ff69e 100644
--- a/skipper/skipper/dataseries/tests/data_point/test_data_point_non_historical_lifecycle.py
+++ b/skipper/skipper/dataseries/tests/data_point/test_data_point_non_historical_lifecycle.py
@@ -156,6 +156,7 @@ class FakedFile:
)
_sql = data_series_as_sql_table(
+ include_in_payload=None,
data_series=_data_series_obj,
payload_as_json=True,
point_in_time=False,
diff --git a/skipper/skipper/dataseries/views/datapoint/crud.py b/skipper/skipper/dataseries/views/datapoint/crud.py
index b90e0a3..6184a5b 100644
--- a/skipper/skipper/dataseries/views/datapoint/crud.py
+++ b/skipper/skipper/dataseries/views/datapoint/crud.py
@@ -198,6 +198,17 @@ def get_filter_value(self) -> Dict[str, Any]:
else:
filter_value = {}
return filter_value
+
+ def get_include_in_payload(self) -> Optional[List[str]]:
+ if 'include_in_payload' in self.request.GET:
+ _include_in_payload_str = self.request.GET['include_in_payload']
+ if _include_in_payload_str != None:
+ return list(_include_in_payload_str.split(','))
+ else:
+ return None
+ else:
+ return None
+
def get_external_ids(self) -> Optional[List[str]]:
return self.request.GET.getlist('external_id') if 'external_id' in self.request.GET else None # type: ignore
@@ -275,6 +286,7 @@ def get_description_string(self) -> str:
- count[=true]
- external_id= (repeatable)
- identify_dimensions_by_external_id[=true]
+ - include_in_payload= (comma separated list of dimension/fact external ids to include in the payload)
"""
if _history:
doc_string = f"""
diff --git a/skipper/skipper/dataseries/views/metamodel/cube_sql.py b/skipper/skipper/dataseries/views/metamodel/cube_sql.py
index f80b3bc..4895ce1 100644
--- a/skipper/skipper/dataseries/views/metamodel/cube_sql.py
+++ b/skipper/skipper/dataseries/views/metamodel/cube_sql.py
@@ -121,6 +121,7 @@ def get(self, request: Request, **kwargs: str) -> HttpResponse:
sql_template = data_series_as_sql_table(
data_series,
+ include_in_payload=None,
payload_as_json=payload_as_json(request),
resolve_dimension_external_ids=use_external_id_as_dimension_identifier(
cast(Dict[str, Any], request.GET)