Skip to content

Commit

Permalink
Add sh:in ConstraintComponent to Flink and SQLite SHACL to SQL transl…
Browse files Browse the repository at this point in the history
…ation

sh:in feature restricts values to a set of allowed values. This is used for instance
for state values (when people do not do it the preferred way and manage state by IRIs)
e.g. the SHACL sh:in ("ON" "OFF") restricts a Literal only to the values "ON" or "OFF".
In addition, unit tests and kms tests have been extended.

Epic: #427
User Story: #454
Task: #455

Signed-off-by: marcel <[email protected]>
  • Loading branch information
wagmarcel authored and abhijith-hr committed Nov 17, 2023
1 parent bd700cb commit c39f9c4
Show file tree
Hide file tree
Showing 15 changed files with 274 additions and 2 deletions.
57 changes: 56 additions & 1 deletion semantic-model/shacl2flink/lib/shacl_properties_to_sql.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
from rdflib import Graph, Namespace
import os
import sys
import csv
from io import StringIO
import ruamel.yaml
from jinja2 import Template

Expand Down Expand Up @@ -52,6 +54,7 @@
SELECT
?nodeshape ?targetclass ?propertypath ?mincount ?maxcount ?attributeclass ?nodekind
?minexclusive ?maxexclusive ?mininclusive ?maxinclusive ?minlength ?maxlength ?pattern ?severitycode
(GROUP_CONCAT(CONCAT('"',?in, '"'); separator=',') as ?ins)
where {
?nodeshape a sh:NodeShape .
?nodeshape sh:targetClass ?targetclass .
Expand All @@ -72,9 +75,12 @@
OPTIONAL { ?nodeshape sh:property [ sh:path ?propertypath ; sh:property [sh:path ngsild:hasValue ; sh:minLength ?minlength ;] ; ] }
OPTIONAL { ?nodeshape sh:property [ sh:path ?propertypath ; sh:property [sh:path ngsild:hasValue ; sh:maxLength ?maxlength ;] ; ] }
OPTIONAL { ?nodeshape sh:property [ sh:path ?propertypath ; sh:property [sh:path ngsild:hasValue ; sh:pattern ?pattern ;] ; ] }
OPTIONAL { ?nodeshape sh:property [ sh:path ?propertypath ; sh:property [sh:path ngsild:hasValue ; sh:in/(rdf:rest*/rdf:first)+ ?in ;] ; ] }
OPTIONAL { ?nodeshape sh:property [ sh:path ?propertypath ; sh:property [sh:path ngsild:hasValue ; sh:class ?attributeclass ;] ; ] }
OPTIONAL { ?nodeshape sh:property [ sh:path ?propertypath; sh:severity ?severity ; ] . ?severity iff:severityCode ?severitycode .}
}
GROUP BY ?nodeshape ?targetclass ?propertypath ?mincount ?maxcount ?attributeclass ?nodekind
?minexclusive ?maxexclusive ?mininclusive ?maxinclusive ?minlength ?maxlength ?pattern ?severitycode
order by ?targetclass
""" # noqa: E501
Expand Down Expand Up @@ -325,6 +331,28 @@
FROM A1
""" # noqa: E501

sql_check_literal_in = """
SELECT this AS resource,
'{{constraintname}}({{property_path}}[' || CAST( `index` AS STRING) || '])' AS event,
'Development' AS environment,
{%- if sqlite -%}
'[SHACL Validator]' AS service,
{%- else %}
ARRAY ['SHACL Validator'] AS service,
{%- endif %}
CASE WHEN typ IS NOT NULL AND attr_typ IS NOT NULL AND val NOT IN ({% for elem in ins %}'{{ elem }}'{{ ", " if not loop.last else "" }}{% endfor %})
THEN '{{severity}}'
ELSE 'ok' END AS severity,
'customer' customer,
CASE WHEN typ IS NOT NULL AND attr_typ IS NOT NULL AND val NOT IN ({% for elem in ins %}'{{ elem }}'{{ ", " if not loop.last else "" }}{% endfor %})
THEN 'Model validation for Property {{property_path}} failed for ' || this || '. Value ' || IFNULL(val, 'NULL') || ' is not allowed.'
ELSE 'All ok' END as `text`
{% if sqlite %}
,CURRENT_TIMESTAMP
{% endif %}
FROM A1
""" # noqa: E501


def translate(shaclefile, knowledgefile):
"""
Expand Down Expand Up @@ -484,6 +512,11 @@ def translate(shaclefile, knowledgefile):
max_length = row.maxlength.toPython() if row.maxlength is not None \
else None
pattern = row.pattern.toPython() if row.pattern is not None else None
ins = row.ins.toPython() if row.ins is not None else None
if ins is not None and ins != '':
reader = csv.reader(StringIO(ins))
parsed_list = next(reader)
ins = [element.replace("'", "\\'") for element in parsed_list]
if (nodekind == sh.IRI):
sql_command_yaml = Template(sql_check_property_iri_base).render(
alerts_bulk_table=alerts_bulk_table,
Expand Down Expand Up @@ -539,7 +572,6 @@ def translate(shaclefile, knowledgefile):
property_class=property_class,
severity=severitycode,
sqlite=True)

elif (nodekind == sh.Literal):
sql_command_yaml = Template(sql_check_property_iri_base).render(
alerts_bulk_table=alerts_bulk_table,
Expand Down Expand Up @@ -598,6 +630,29 @@ def translate(shaclefile, knowledgefile):
severity=severitycode,
minmaxname="MinExclusive",
sqlite=True)
if ins is not None and len(ins) != 0:
sql_command_yaml += "\nUNION ALL"
sql_command_sqlite += "\nUNION ALL"
sql_command_yaml += \
Template(sql_check_literal_in).render(
alerts_bulk_table=alerts_bulk_table,
target_class=target_class,
property_path=property_path,
property_class=property_class,
severity=severitycode,
sqlite=False,
constraintname="InConstraintComponent",
ins=ins)
sql_command_sqlite += \
Template(sql_check_literal_in).render(
alerts_bulk_table=alerts_bulk_table,
target_class=target_class,
property_path=property_path,
property_class=property_class,
severity=severitycode,
sqlite=True,
constraintname="InConstraintComponent",
ins=ins)
if max_exclusive is not None:
sql_command_yaml += "\nUNION ALL"
sql_command_sqlite += "\nUNION ALL"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','warning'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:plasmacutter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','warning'
'urn:plasmacutter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
[
{
"@context": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld",
"id": "urn:plasmacutter:1",
"type": "https://industry-fusion.com/types/v0.9/plasmacutter",
"https://industry-fusion.com/types/v0.9/state": {
"type": "Property",
"value": {
"@id": "https://industry-fusion.com/types/v0.9/state_OFF"
}
},
"https://industry-fusion.com/types/v0.9/stringState": {
"type": "Property",
"value": "T\"ST"
},
"https://industry-fusion.com/types/v0.9/hasWorkpiece": {
"type": "Relationship",
"https://uri.etsi.org/ngsi-ld/hasObject": "urn:workpiece:1"
},
"https://industry-fusion.com/types/v0.9/hasFilter": {
"type": "Relationship",
"object": "urn:filter:1"
}
},
{
"@context": "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld",
"id": "urn:filter:1",
"type": "https://industry-fusion.com/types/v0.9/filter",
"https://industry-fusion.com/types/v0.9/state": {
"type": "Property",
"value": {
"@id": "https://industry-fusion.com/types/v0.9/state_PREPARING"
}
},
"https://industry-fusion.com/types/v0.9/stringState": {
"type": "Property",
"value": "t1,t2"
},
"https://industry-fusion.com/types/v0.9/strength": {
"type": "Property",
"value": 0.5
},
"https://industry-fusion.com/types/v0.9/hasCartridge": {
"type": "Relationship",
"https://uri.etsi.org/ngsi-ld/hasObject": "urn:filterCartridge:1"
}
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
'urn:filter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge[0])','warning'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:filter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:filter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:filter:1','MaxInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','MinInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','warning'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:plasmacutter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter[0])','ok'
'urn:plasmacutter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece[0])','warning'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:plasmacutter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','warning'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','warning'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:plasmacutter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
Original file line number Diff line number Diff line change
@@ -1,18 +1,24 @@
'urn:filter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge[0])','warning'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:filter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:filter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:filter:1','MaxInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','MinInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:plasmacutter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter[0])','ok'
'urn:plasmacutter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece[0])','warning'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:plasmacutter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
Original file line number Diff line number Diff line change
@@ -1,21 +1,27 @@
'urn:filter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge[0])','warning'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:filter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:filter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:filter:1','MaxInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','MinInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:plasmacutter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter[0])','ok'
'urn:plasmacutter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece[0])','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:plasmacutter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:workpiece:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/material)','ok'
'urn:workpiece:1','CountConstraintComponent(https://schema.org/depth)','ok'
'urn:workpiece:1','MaxInclusiveConstraintComponent(https://schema.org/depth[0])','ok'
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
'urn:filter:1','ClassConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge[0])','ok'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:filter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:filter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:filter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:filter:1','MaxInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','MinInclusiveConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasCartridge)','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/strength[0])','ok'
'urn:filter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:filterCartridge:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/inUseFrom)','ok'
'urn:filterCartridge:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/inUseUntil)','ok'
'urn:filterCartridge:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/wasteClass)','ok'
Expand All @@ -19,10 +22,13 @@
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/state)','ok'
'urn:plasmacutter:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/stringState)','ok'
'urn:plasmacutter:1','DatatypeConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','InConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasFilter)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/hasWorkpiece)','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/state[0])','ok'
'urn:plasmacutter:1','NodeKindConstraintComponent(https://industry-fusion.com/types/v0.9/stringState[0])','ok'
'urn:workpiece:1','CountConstraintComponent(https://industry-fusion.com/types/v0.9/material)','ok'
'urn:workpiece:1','CountConstraintComponent(https://schema.org/depth)','ok'
'urn:workpiece:1','MaxInclusiveConstraintComponent(https://schema.org/depth[0])','warning'
Expand Down
Loading

0 comments on commit c39f9c4

Please sign in to comment.