Distance measure implementation

modules/fbs-sql-checker/distance/.gitignore

@@ -0,0 +1 @@
+__pycache__/

modules/fbs-sql-checker/distance/attribute_check.py

@@ -0,0 +1,102 @@
+import sqlparse
+import constants as c
+import attribute_distance as att_dist
+import format as f
+
+ref_pro_att: list[str] = []
+query_pro_att: list[str] = []
+
+ref_cmd_list: list[str] = []
+query_cmd_list: list[str] = []
+
+ref_map: dict[str, dict[str, str]] = {}
+query_map: dict[str, dict[str, str]] = {}
+
+
+def extract_attributes(ref, query):
+    _token_iteration(ref, ref_map, ref_pro_att, ref_cmd_list)
+    _token_iteration(query, query_map, query_pro_att, query_cmd_list)
+
+    print(f"REF MAP: {ref_map}\nQuery Map: {query_map}\n")
+    print("Projection attributes before order: ", ref_pro_att, query_pro_att)
+
+    print(f"COMMAND LIST HERE {ref_cmd_list}, QUERY {query_cmd_list}")
+
+    attribute_distance = att_dist.get_attributes_distance(ref_pro_att, query_pro_att)
+
+    print("Projection attributes after order: ", ref_pro_att, query_pro_att, "\n")
+
+    command_distance = att_dist.get_command_distance(ref_cmd_list, query_cmd_list)
+
+    keyword_distance = att_dist.get_keyword_distance(ref_map, query_map)
+
+    print(f"attributes: {attribute_distance}, command: {command_distance}, keywordw: {keyword_distance}")
+
+    return attribute_distance + command_distance + keyword_distance
+
+
+def _token_iteration(tokens, map_dict, pro_att_list, cmd_list):
+    for i, token in enumerate(tokens):
+        if isinstance(token, sqlparse.sql.Token):
+            if isinstance(token, sqlparse.sql.Token):
+                if token.ttype == sqlparse.tokens.Whitespace or token.ttype == sqlparse.tokens.Newline:
+                    continue
+                if token.ttype == sqlparse.tokens.Keyword and token.value == c.DISTINCT:
+                    _extract_keyword(tokens[i + 2], map_dict)
+                if token.ttype == sqlparse.tokens.Wildcard:
+                    pro_att_list.append(token.value)
+                    break
+                if isinstance(token, sqlparse.sql.IdentifierList):
+                    for t in token.get_identifiers():
+                        _extract_att_and_cmds(t, map_dict, pro_att_list, cmd_list)
+                if isinstance(token, (
+                        sqlparse.sql.Identifier, sqlparse.sql.Function, sqlparse.sql.Operation,
+                        sqlparse.sql.Parenthesis)):
+                    _extract_att_and_cmds(token, map_dict, pro_att_list, cmd_list)
+                if token.ttype == sqlparse.tokens.Keyword and token.value == c.FROM:
+                    break
+
+
+def _extract_att_and_cmds(token, map_dict, pro_att_list, cmd_list):
+    _extract_alias(token, map_dict)
+    if isinstance(token, (sqlparse.sql.Operation, sqlparse.sql.Parenthesis)):
+        pro_att_list.append(token.value)
+    if isinstance(token, sqlparse.sql.Function):
+        params = [p.value for p in token.get_parameters()]
+        cmd_list.append(token.get_real_name())
+        pro_att_list.append(params[0])
+        if token.value.__contains__(c.DISTINCT):
+            _extract_keyword(f.format_command(token), map_dict)
+    if isinstance(token, sqlparse.sql.Identifier):
+        if str(token.get_real_name()).upper() in [cmd for cmd in c.SELECT_CMDS]:
+            cmd_list.append(token.get_real_name())
+            pro_att_list.append(f.format_alias(f.format_distinct(f.format_db_name(f.format_command(token)))))
+            if token.value.__contains__(c.DISTINCT):
+                _extract_keyword(f.format_command(token), map_dict)
+        else:
+            pro_att_list.append(f.format_distinct(f.format_db_name(f.format_alias(token.value))))
+
+
+def _extract_alias(ident: sqlparse.sql.Identifier, map_dict):
+    if ident.has_alias():
+        updated_ident = f.format_alias(f.format_db_name(ident.value))
+        # token first will extract the attribute without its alias
+        _add_to_map(map_dict, c.ALIAS, updated_ident, ident.get_alias())
+
+
+def _extract_keyword(ident, map_dict):
+    if isinstance(ident, sqlparse.sql.IdentifierList):
+        ident_list = list[sqlparse.sql.Identifier](ident.get_identifiers())
+        # get all the identifiers that are referred to the distinct keyword. (alias gets formatted and removed)
+        result = ", ".join(f.format_db_name(f.format_alias(i.value)) for i in ident_list)
+        _add_to_map(map_dict, c.KEYWORD, c.DISTINCT, result)
+    else:
+        # remove trailing alias or distinct keyword to add only the attribute to the map
+        updated_value = f.format_distinct(f.format_db_name(f.format_alias(ident)))
+        _add_to_map(map_dict, c.KEYWORD, c.DISTINCT, updated_value)
+
+
+def _add_to_map(map_dict, key, inner_key, value):
+    if key not in map_dict:
+        map_dict[key] = {}
+    map_dict[key][inner_key] = value

modules/fbs-sql-checker/distance/attribute_distance.py

@@ -0,0 +1,91 @@
+import re
+import constants as c
+import uuid
+
+operation_map: dict[str, str, str] = {}
+
+
+def get_attributes_distance(ref: list[str], query: list[str]):
+    moves = 0
+    # ignore wildcard 
+    if ref.__contains__("*"):
+        moves = 0
+    elif set(ref) == set(query):
+        for r, q in zip(ref, query):
+            if r != q:
+                moves += c.ORDER_MULT
+                # Rearrange the query to match the order of reference
+                query.remove(r)
+                query.insert(ref.index(r), r)
+    else:
+        for r in ref:
+            if r not in query:
+                moves += c.OBJECT_MULT
+
+    op_dist = _get_operation_distance(ref, query)
+    moves += op_dist
+    print(f"\nOP MAP: {operation_map}\n")
+    return moves
+
+
+def get_command_distance(ref: list[str], query: list[str]):
+    moves = 0
+    if len(ref) != len(query):
+        moves += c.OBJECT_MULT
+    elif set(ref) != set(query):
+        moves += c.STRUCT_MULT
+    return moves
+
+
+def get_keyword_distance(ref_map: dict, query_map: dict):
+    moves = 0
+    ref_kws: dict = ref_map.get(c.KEYWORD)
+    query_kws: dict = query_map.get(c.KEYWORD)
+
+    if ref_kws is not None and query_kws is not None:
+        if set(ref_kws.values()) == set(query_kws.values()):
+            moves = 0
+        else:
+            moves += c.OBJECT_MULT
+    return moves
+
+
+def _get_operation_distance(ref_list: list[str], query_list: list[str]):
+    ref_op_list = []
+    query_op_list = []
+
+    # using a regex pattern to extract the operations contained in both attribute lists
+    for exp in ref_list:
+        if re.findall(c.MATH_EXP_REGEX, exp):
+            ref_op_list.append(exp)
+    for exp in query_list:
+        if re.findall(c.MATH_EXP_REGEX, exp):
+            query_op_list.append(exp)
+
+    return _calculate_expression_similarity(ref_op_list, query_op_list)
+
+# Jaccard index may not be the best method to measure the distance of two mathematical expressions
+def _calculate_expression_similarity(ref_exp: list[str], query_exp: list[str]):
+    diff = 0
+    for r, q in zip(ref_exp, query_exp):
+        ref_set = set(r.replace("(", "").replace(")", ""))
+        query_set = set(q.replace("(", "").replace(")", ""))
+        intersection = len(ref_set.intersection(query_set))
+        union = len(ref_set.union(query_set))  # len(ref_set) + len(query_set) - intersection
+        if union != 0:
+            # Jaccard Index / Similarity Coefficient
+            diff_val = 1 - (intersection / union)
+            _add_to_op_map(operation_map, r, q, diff_val)
+            diff += diff_val
+    return diff
+
+
+def _add_to_op_map(op_map, ref, query, sim):
+    generated_uuid = str(uuid.uuid4())
+    short_id = generated_uuid[:4]  # Take the first 8 characters as the short ID
+    new_entry_key = f"{short_id}"
+    op_map[new_entry_key] = {
+        "ref": ref,
+        "query": query,
+        "difference": sim
+    }

modules/fbs-sql-checker/distance/constants.py

@@ -0,0 +1,60 @@
+# BASIC SQL KEYWORDS
+DISTINCT = "DISTINCT"
+FROM = "FROM"
+TABLE = "TABLE"
+ALIAS = "ALIAS"
+ON = "ON"
+WHERE = "WHERE"
+GROUP_BY = "GROUP BY"
+HAVING = "HAVING"
+ORDER_BY = "ORDER BY"
+DESC = r'(?i)DESC'
+ASC = r'(?i)ASC'
+KEYWORD = "KEYWORD"
+COMMAND = "COMMAND"
+
+# SELECT COMMANDS
+SELECT_CMDS = [
+    "SUM",
+    "COUNT",
+    "ROUND",
+    "SEC_TO_TIME",
+    "AVG",
+    "MAX",
+    "MIN",
+    "ABS",
+    "TIME_TO_SEC",
+    "YEAR",
+    "UPPER",
+    "LOWER",
+    "LENGTH"
+]
+
+# JOIN TYPES
+JOIN_TYPES = [
+    "INNER JOIN",
+    "LEFT JOIN",
+    "RIGHT JOIN",
+    "FULL JOIN",
+    "SELF JOIN",
+    "JOIN"
+]
+
+# REGULAR EXPRESSIONS
+ALIAS_REGEX = r"\sas\s+\"(.+?)\"|\sas\s+(\w+)"
+DB_NAME_REGEX = r"^[^.]+\.(.*)$"
+DB_COMP_REGEX = r'(\s*(?:<=|>=|!=|=|<|>)\s*)'
+MATH_EXP_REGEX = r"[\d()+\-*\/]"
+EQ_COMP_REGEX = r'\s*\w+\s*=\s*\w+\s*'
+
+# DATABASE DATA
+HOSTNAME = 'localhost'
+DB_NAME = 'postgres'
+USERNAME = 'postgres'
+PASSWORD = 'admin'
+PORT_ID = 5432
+
+# MULTIPLIERS
+ORDER_MULT = 5
+STRUCT_MULT = 20
+OBJECT_MULT = 50

modules/fbs-sql-checker/distance/db_connection.py

@@ -0,0 +1,46 @@
+import psycopg2
+import constants as c
+
+connection = None
+
+
+def setup_db(att_list):
+    with psycopg2.connect(
+            host=c.HOSTNAME,
+            dbname=c.DB_NAME,
+            user=c.USERNAME,
+            password=c.PASSWORD,
+            port=c.PORT_ID
+    ) as connection:
+        cursor = connection.cursor()
+
+        for i, att in enumerate(att_list):
+            sql_script = f"CREATE TABLE IF NOT EXISTS {chr(65 + i)} (x INTEGER, CONSTRAINT {chr(97 + i)}_unique_x UNIQUE (x));"
+            cursor.execute(sql_script)
+
+        bits = len(att_list)
+        for i in range(2 ** bits):
+            binary_number = format(i, f'0{bits}b')
+            decimal_number = int(binary_number, 2)
+            offset = -1
+            for j in range(len(binary_number)):
+                if binary_number[offset] == '1':
+                    select_query = f"SELECT * FROM {chr(65 + j)} WHERE x = {decimal_number};"
+                    cursor.execute(select_query)
+                    result = cursor.fetchone()
+
+                    if result is None:
+                        insert_query = f"INSERT INTO {chr(65 + j)} (x) VALUES ({decimal_number});"
+                        cursor.execute(insert_query)
+                offset -= 1
+        connection.commit()
+    return connection
+
+
+def execute_query(query, connection: psycopg2):
+    cursor = connection.cursor()
+    cursor.execute(query)
+    result = cursor.fetchall()
+    res_set = set(frozenset(s) for s in result)
+    connection.commit()
+    return res_set

modules/fbs-sql-checker/distance/docker-compose.yml

@@ -0,0 +1,16 @@
+version: '3.8'
+
+services:
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: admin
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+volumes:
+  postgres_data:

modules/fbs-sql-checker/distance/equation_checker.py

@@ -0,0 +1,21 @@
+import re
+import constants as c
+import sympy as s
+
+
+def check_equation(ref: list[str], query: list[str]):
+    moves = 0
+    for r, q in zip(ref, query):
+        if re.match(c.EQ_COMP_REGEX, r) and re.match(c.EQ_COMP_REGEX, q):
+            if set(r) != set(q):
+                moves += c.OBJECT_MULT
+        elif not _check_if_equal(r, q):
+            moves += c.OBJECT_MULT
+    return moves
+
+
+def _check_if_equal(eq1, eq2):
+    eq1 = s.simplify(s.sympify(eq1))
+    eq2 = s.simplify(s.sympify(eq2))
+
+    return eq1.equals(eq2)

modules/fbs-sql-checker/distance/format.py

@@ -0,0 +1,54 @@
+import re
+import sqlparse
+import constants as c
+
+
+def format_alias(ident: str):
+    # check ident with pattern to get the alias keyword and alias name
+    regex = re.compile(c.ALIAS_REGEX, re.IGNORECASE)
+    match = regex.search(ident)
+    if match:
+        # flag is used for case sensitivity
+        ident = re.sub(c.ALIAS_REGEX, "", ident, flags=re.IGNORECASE).strip()
+    return ident
+
+
+def format_db_name(ident: str):
+    # check ident with pattern to get the alias keyword and alias name
+    regex = re.compile(c.DB_NAME_REGEX)
+
+    # Check if the ident matches the pattern
+    match = regex.search(ident)
+    if match:
+        ident = match.group(1)
+
+    return ident
+
+
+def format_distinct(ident: str):
+    if ident.__contains__(c.DISTINCT):
+        ident = ident.replace(c.DISTINCT, "").strip()
+    return ident
+
+
+def format_command(ident: sqlparse.sql.Identifier):
+    # get_real_name() function returns the select command and will get removed along the alias and the 'as' keyword
+    formatted = ident.value.replace(ident.get_real_name(), "").replace("(", "").replace(")", "")
+    return formatted
+
+
+def format_comp_db_name(ident: str):
+    # Split the input string using the regex pattern to find the operator
+    parts = re.split(c.DB_COMP_REGEX, ident)
+
+    # Get the left and right sides of the equation after removing whitespace
+    left_substring = parts[0].rsplit('.', 1)[-1].strip()
+    right_substring = parts[2].rsplit('.', 1)[-1].strip()
+
+    # Check if the operator is "LIKE" and replace it with "="
+    operator = parts[1].strip()
+
+    # Join the substrings back together with the operator
+    result = f"{left_substring} {operator} {right_substring}"
+
+    return result