-
Notifications
You must be signed in to change notification settings - Fork 1
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat : add parameter of type connection (IASO,GCS..) #86
Changes from 2 commits
9324b65
34e617c
fbe3f00
f3946d6
d263c2c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,12 @@ | ||
import re | ||
import typing | ||
from openhexa.sdk.workspaces.connection import ( | ||
DHIS2Connection, | ||
IASOConnection, | ||
PostgreSQLConnection, | ||
S3Connection, | ||
GCSConnection, | ||
) | ||
|
||
|
||
class ParameterValueError(Exception): | ||
|
@@ -125,7 +132,179 @@ def normalize(value: typing.Any) -> typing.Any: | |
return value | ||
|
||
|
||
TYPES_BY_PYTHON_TYPE = {str: String, bool: Boolean, int: Integer, float: Float} | ||
class PostgreSQLConnectionType(ParameterType): | ||
@property | ||
def spec_type(self) -> str: | ||
return "postgresql" | ||
|
||
@property | ||
def expected_type(self) -> typing.Type: | ||
return PostgreSQLConnectionType | ||
|
||
@property | ||
def accepts_choice(self) -> bool: | ||
return False | ||
|
||
@property | ||
def accepts_multiple(self) -> bool: | ||
return False | ||
|
||
def validate(self, value: typing.Optional[typing.Any], *, allow_empty: bool = True) -> typing.Optional[str]: | ||
if not allow_empty and value == "": | ||
raise ParameterValueError("Empty values are not accepted.") | ||
|
||
if not isinstance(value, str): | ||
raise ParameterValueError(f"Invalid type for value {value} (expected {str}, got {type(value)})") | ||
|
||
return value | ||
|
||
|
||
class S3ConnectionType(ParameterType): | ||
@property | ||
def spec_type(self) -> str: | ||
return "s3" | ||
|
||
@property | ||
def expected_type(self) -> typing.Type: | ||
return S3ConnectionType | ||
|
||
@property | ||
def accepts_choice(self) -> bool: | ||
return False | ||
|
||
@property | ||
def accepts_multiple(self) -> bool: | ||
return False | ||
|
||
def validate(self, value: typing.Optional[typing.Any], *, allow_empty: bool = True) -> typing.Optional[str]: | ||
if not allow_empty and value == "": | ||
raise ParameterValueError("Empty values are not accepted.") | ||
|
||
if not isinstance(value, str): | ||
raise ParameterValueError(f"Invalid type for value {value} (expected {str}, got {type(value)})") | ||
|
||
return value | ||
|
||
|
||
class GCSConnectionType(ParameterType): | ||
@property | ||
def spec_type(self) -> str: | ||
return "gcs" | ||
|
||
@property | ||
def expected_type(self) -> typing.Type: | ||
return GCSConnectionType | ||
|
||
@property | ||
def accepts_choice(self) -> bool: | ||
return False | ||
|
||
@property | ||
def accepts_multiple(self) -> bool: | ||
return False | ||
|
||
def validate(self, value: typing.Optional[typing.Any], *, allow_empty: bool = True) -> typing.Optional[str]: | ||
if not allow_empty and value == "": | ||
raise ParameterValueError("Empty values are not accepted.") | ||
|
||
if not isinstance(value, str): | ||
raise ParameterValueError(f"Invalid type for value {value} (expected {str}, got {type(value)})") | ||
|
||
return value | ||
|
||
|
||
class DHIS2ConnectionType(ParameterType): | ||
@property | ||
def spec_type(self) -> str: | ||
return "dhis2" | ||
|
||
@property | ||
def expected_type(self) -> typing.Type: | ||
return DHIS2ConnectionType | ||
|
||
@property | ||
def accepts_choice(self) -> bool: | ||
return False | ||
|
||
@property | ||
def accepts_multiple(self) -> bool: | ||
return False | ||
|
||
def validate(self, value: typing.Optional[typing.Any], *, allow_empty: bool = True) -> typing.Optional[str]: | ||
if not allow_empty and value == "": | ||
raise ParameterValueError("Empty values are not accepted.") | ||
|
||
if not isinstance(value, str): | ||
raise ParameterValueError(f"Invalid type for value {value} (expected {str}, got {type(value)})") | ||
|
||
return value | ||
|
||
|
||
class IASOConnectionType(ParameterType): | ||
@property | ||
def spec_type(self) -> str: | ||
return "iaso" | ||
|
||
@property | ||
def expected_type(self) -> typing.Type: | ||
return IASOConnectionType | ||
|
||
@property | ||
def accepts_choice(self) -> bool: | ||
return False | ||
|
||
@property | ||
def accepts_multiple(self) -> bool: | ||
return False | ||
|
||
def validate(self, value: typing.Optional[typing.Any], *, allow_empty: bool = True) -> typing.Optional[str]: | ||
if not allow_empty and value == "": | ||
raise ParameterValueError("Empty values are not accepted.") | ||
|
||
if not isinstance(value, str): | ||
raise ParameterValueError(f"Invalid type for value {value} (expected {str}, got {type(value)})") | ||
|
||
return value | ||
|
||
|
||
class CustomConnectionType(ParameterType): | ||
@property | ||
def spec_type(self) -> str: | ||
return "custom" | ||
|
||
@property | ||
def expected_type(self) -> typing.Type: | ||
return str | ||
|
||
@property | ||
def accepts_choice(self) -> bool: | ||
return False | ||
|
||
@property | ||
def accepts_multiple(self) -> bool: | ||
return False | ||
|
||
def validate(self, value: typing.Optional[typing.Any], *, allow_empty: bool = True) -> typing.Optional[str]: | ||
if not allow_empty and value == "": | ||
raise ParameterValueError("Empty values are not accepted.") | ||
|
||
if not isinstance(value, str): | ||
raise ParameterValueError(f"Invalid type for value {value} (expected {str}, got {type(value)})") | ||
|
||
return super().validate(value, allow_empty) | ||
|
||
|
||
TYPES_BY_PYTHON_TYPE = { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do you think that it would be a lot of added work to rename There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let me try |
||
str: String, | ||
bool: Boolean, | ||
int: Integer, | ||
float: Float, | ||
DHIS2Connection: DHIS2ConnectionType, | ||
PostgreSQLConnection: PostgreSQLConnectionType, | ||
IASOConnection: IASOConnectionType, | ||
S3Connection: S3ConnectionType, | ||
GCSConnection: GCSConnectionType, | ||
} | ||
|
||
|
||
class InvalidParameterError(Exception): | ||
|
@@ -343,3 +522,19 @@ def get_all_parameters(self): | |
return [self.parameter, *self.function.get_all_parameters()] | ||
|
||
return [self.parameter] | ||
|
||
|
||
def is_connection_parameter(param: Parameter): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is fine - but we could do something cleaner, ideally encapsulating the logic that builds a connection instance in Do you think this would be possible? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, I've first tried that approach but for unknow (still searching) reason I couldn't load the connection config. But yes, I'll try to make it works. |
||
return any( | ||
[ | ||
isinstance(param.type, type) | ||
for type in [ | ||
DHIS2ConnectionType, | ||
PostgreSQLConnectionType, | ||
IASOConnectionType, | ||
S3ConnectionType, | ||
GCSConnectionType, | ||
CustomConnectionType, | ||
] | ||
] | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,9 +16,14 @@ | |
|
||
from openhexa.sdk.utils import Environments, get_environment | ||
|
||
from .parameter import FunctionWithParameter, Parameter, ParameterValueError | ||
from .parameter import ( | ||
FunctionWithParameter, | ||
Parameter, | ||
ParameterValueError, | ||
is_connection_parameter, | ||
) | ||
from .task import PipelineWithTask | ||
from .utils import get_local_workspace_config | ||
from .utils import get_local_workspace_config, get_connection_by_type | ||
|
||
logger = getLogger(__name__) | ||
|
||
|
@@ -97,7 +102,10 @@ def run(self, config: typing.Dict[str, typing.Any]): | |
for parameter in self.parameters: | ||
value = config.pop(parameter.code, None) | ||
validated_value = parameter.validate(value) | ||
validated_config[parameter.code] = validated_value | ||
if is_connection_parameter(parameter): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. See remark above, we might be able to put everything in |
||
validated_config[parameter.code] = get_connection_by_type(parameter.type, validated_value) | ||
else: | ||
validated_config[parameter.code] = validated_value | ||
|
||
if len(config) > 0: | ||
raise ParameterValueError(f"The provided config contains invalid key(s): {', '.join(list(config.keys()))}") | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,16 @@ | |
import stringcase | ||
import yaml | ||
|
||
from openhexa.sdk.workspaces import workspace | ||
from .parameter import ( | ||
DHIS2ConnectionType, | ||
PostgreSQLConnectionType, | ||
IASOConnectionType, | ||
S3ConnectionType, | ||
GCSConnectionType, | ||
CustomConnectionType, | ||
) | ||
|
||
|
||
class LocalWorkspaceConfigError(Exception): | ||
pass | ||
|
@@ -148,3 +158,23 @@ def get_local_workspace_config(path: Path): | |
if key != "type": | ||
env_vars[stringcase.constcase(f"{slug}_{key.lower()}")] = str(value) | ||
return env_vars | ||
|
||
|
||
def get_connection_by_type(type: any, identifier: str): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Might be unnecessary |
||
if isinstance(type, DHIS2ConnectionType): | ||
return workspace.dhis2_connection(identifier) | ||
|
||
if isinstance(type, PostgreSQLConnectionType): | ||
return workspace.postgresql_connection(identifier) | ||
|
||
if isinstance(type, IASOConnectionType): | ||
return workspace.iaso_connection(identifier) | ||
|
||
if isinstance(type, S3ConnectionType): | ||
return workspace.s3_connection(identifier) | ||
|
||
if isinstance(type, GCSConnectionType): | ||
return workspace.gcs_connection(identifier) | ||
|
||
if isinstance(type, CustomConnectionType): | ||
return workspace.custom_connection(identifier) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This feels duplicated between connection parameter types. We might need an intermediary class or helper methods to avoid copy pasting that kind of code.