Improve handling of column types

Support the following previously unhandled forms - schema.enum_type - schema."enum_type" - "schema"."enum_type" - schema.enum_type[] - schema."enum_type"[] - "schema"."enum_type"[] - char[] Also handle ambiguity between array and settings (both using []) by making the col_type atomic (to not allow whitespace and comments between tokens). Which matches upstream behavior. (all of which are supported by the upstream JS DBML parser)
apskhem · Dec 5, 2024 · e1ba154 · e1ba154
1 parent fe71764
commit e1ba154
Show file tree

Hide file tree

Showing 28 changed files with 10,986 additions and 369 deletions.
diff --git a/src/dbml.pest b/src/dbml.pest
@@ -73,9 +73,9 @@ col_attribute = { ref_inline | attribute }
 col_settings = { "[" ~ (col_attribute ~ ("," ~ col_attribute)*)? ~ "]" }
 col_type_arg = { "(" ~ (value ~ ("," ~ value)*)? ~ ")" }
 col_type_array = { "[" ~ integer? ~ "]" }
-col_type_unquoted = { var ~ col_type_arg? }
-col_type_quoted = { "\"" ~ spaced_var ~ col_type_arg? ~ col_type_array* ~ "\"" }
-col_type = { col_type_unquoted | col_type_quoted }
+col_type_unquoted = ${ var ~ col_type_arg? ~ col_type_array* }
+col_type_quoted = ${ "\"" ~ spaced_var ~ col_type_arg? ~ col_type_array* ~ "\"" ~ col_type_array* }
+col_type = ${ (ident ~ ".")? ~ (col_type_quoted | col_type_unquoted ) }
 table_col = { ident ~ col_type ~ col_settings? }
 table_block = { "{" ~ (table_col | note_decl | indexes_decl)* ~ "}" }
 table_alias = { ^"as " ~ ident }

diff --git a/src/parser/mod.rs b/src/parser/mod.rs
@@ -229,19 +229,34 @@ fn parse_table_col(pair: Pair<Rule>) -> ParserResult<TableColumn> {
   })
 }
 
+fn build_type_name_with_schema(schema: Option<&Ident>, type_name: Pair<Rule>) -> String {
+  let mut type_name = type_name.as_str().to_string();
+  if let Some(schema) = schema {
+    type_name = format!("{}.{}", schema.to_string, type_name);
+  }
+  type_name
+}
+
 fn parse_col_type(pair: Pair<Rule>) -> ParserResult<ColumnType> {
   let mut out = ColumnType {
     span_range: s2r(pair.as_span()),
     raw: pair.as_str().to_string(),
     ..Default::default()
   };
 
+  let mut schema = None;
+
   for p1 in pair.into_inner() {
     match p1.as_rule() {
+      Rule::ident => {
+        schema = Some(parse_ident(p1)?);
+      }
       Rule::col_type_quoted | Rule::col_type_unquoted => {
         for p2 in p1.into_inner() {
           match p2.as_rule() {
-            Rule::var | Rule::spaced_var => out.type_name = ColumnTypeName::Raw(p2.as_str().to_string()),
+            Rule::var | Rule::spaced_var => {
+              out.type_name = ColumnTypeName::Raw(build_type_name_with_schema(schema.as_ref(), p2))
+            }
             Rule::col_type_arg => out.args = parse_col_type_arg(p2)?,
             Rule::col_type_array => {
               let val = p2.into_inner().try_fold(None, |_, p3| {

diff --git a/tests/dbml/array_type.in.dbml b/tests/dbml/array_type.in.dbml
@@ -2,8 +2,10 @@ Table sal_emp {
   name text
   pay_by_quarter "int[]" [not null]
   schedule "text[][]" [null]
+  unquoted text[] [not null]
+  mixed "character varying"[]
 }
 
 Table tictactoe {
   squares "integer[3][3]"
-}
+}
diff --git a/tests/dbml/index_tables.in.dbml b/tests/dbml/index_tables.in.dbml
@@ -17,4 +17,4 @@ Table users {
     (`getdate()`, `upper(gender)`)
     (`reverse(country_code)`)
   }
-}
+}
diff --git a/tests/dbml/postgres_importer/multiple_schema.out.dbml b/tests/dbml/postgres_importer/multiple_schema.out.dbml
@@ -17,6 +17,8 @@ Table "users" {
   "pjs2" job_status
   "pg" schemaB.gender
   "pg2" gender
+  "pg3" schemaB."gender"
+  "pg4" "schemaB"."gender"
 }
 
 Table "products" {
@@ -66,4 +68,4 @@ Table "schemaA"."locations" {
   "id" int [pk]
   "name" varchar
   Note: 'This is a note in table "locations"'
-}
+}
diff --git a/tests/mod.rs b/tests/mod.rs
@@ -22,22 +22,42 @@ fn read_dbml_dir<P: AsRef<Path>>(dir_path: P) -> Result<Vec<PathBuf>> {
   Ok(out)
 }
 
-fn create_out_dir() -> Result<()> {
+fn create_out_dir(sub_dir: Option<&PathBuf>) -> Result<()> {
   if fs::metadata(OUT_DIR).is_err() {
     fs::create_dir(OUT_DIR)?;
   }
+  if let Some(sub_dir) = sub_dir {
+    let path = Path::new(OUT_DIR).join(sub_dir);
+    if fs::metadata(&path).is_err() {
+      fs::create_dir(path)?;
+    }
+  }
 
   Ok(())
 }
 
 /// Run with UPDATE_DBML_OUTPUT=1 to update the expected output files
 /// e.g. (on Linux or macOS)
 /// UPDATE_DBML_OUTPUT=1 cargo test
-#[test]
-fn parse_dbml_unchecked() -> Result<()> {
-  create_out_dir()?;
+fn update_expected() -> bool {
+  match std::env::var("UPDATE_DBML_OUTPUT") {
+    Ok(v) => v == "1",
+    _ => false,
+  }
+}
 
-  let testing_dbml_paths = read_dbml_dir("tests/dbml")?;
+fn compare_parsed_with_expected(sub_dir: Option<impl Into<PathBuf>>, update: bool) -> Result<()> {
+  let sub_dir = sub_dir.map(Into::into);
+
+  create_out_dir(sub_dir.as_ref())?;
+  let mut source_dir = Path::new("tests/dbml").to_path_buf();
+  let mut out_file_dir = Path::new(OUT_DIR).to_path_buf();
+  if let Some(sub_dir) = sub_dir {
+    source_dir.push(&sub_dir);
+    out_file_dir.push(&sub_dir);
+  }
+
+  let testing_dbml_paths = read_dbml_dir(source_dir)?;
 
   for path in testing_dbml_paths {
     let content = fs::read_to_string(&path)?;
@@ -49,21 +69,15 @@ fn parse_dbml_unchecked() -> Result<()> {
     let mut out_file_path = path.clone();
     out_file_path.set_extension("ron");
     let out_file_name = out_file_path.file_name().unwrap().to_str().unwrap();
-    let out_file_path = format!("{}/{}", OUT_DIR, out_file_name);
+    let out_file_path = out_file_dir.join(out_file_name);
 
-    let update = match std::env::var("UPDATE_DBML_OUTPUT") {
-      Ok(v) => v == "1",
-      _ => false
-    };
     if update {
       fs::write(out_file_path, out_content)?;
     } else {
       let expected = fs::read_to_string(&out_file_path).or_else(|e| {
         match e.kind() {
-          std::io::ErrorKind::NotFound => {
-            Ok("no output file".to_string())
-          },
-          e => Err(e)
+          std::io::ErrorKind::NotFound => Ok("no output file".to_string()),
+          e => Err(e),
         }
       })?;
       assert_eq!(out_content, expected, "Unexpected output for {:?}", path);
@@ -72,7 +86,27 @@ fn parse_dbml_unchecked() -> Result<()> {
 
   Ok(())
 }
-
+
+#[test]
+fn parse_dbml_root() -> Result<()> {
+  compare_parsed_with_expected(None::<PathBuf>, update_expected())
+}
+
+#[test]
+fn parse_dbml_mysql_importer() -> Result<()> {
+  compare_parsed_with_expected(Some("mysql_importer"), update_expected())
+}
+
+#[test]
+fn parse_dbml_mssql_importer() -> Result<()> {
+  compare_parsed_with_expected(Some("mssql_importer"), update_expected())
+}
+
+#[test]
+fn parse_dbml_pgsql_importer() -> Result<()> {
+  compare_parsed_with_expected(Some("postgres_importer"), update_expected())
+}
+
 #[test]
 fn parse_dbml_validator() -> Result<()> {
   let testing_dbml_paths = read_dbml_dir("tests/dbml/validator")?;

diff --git a/tests/out/array_type.in.ron b/tests/out/array_type.in.ron
@@ -1,10 +1,10 @@
 SchemaBlock {
-    span_range: 0..141,
-    input: "Table sal_emp {\n  name text\n  pay_by_quarter \"int[]\" [not null]\n  schedule \"text[][]\" [null]\n}\n\nTable tictactoe {\n  squares \"integer[3][3]\"\n}",
+    span_range: 0..201,
+    input: "Table sal_emp {\n  name text\n  pay_by_quarter \"int[]\" [not null]\n  schedule \"text[][]\" [null]\n  unquoted text[] [not null]\n  mixed \"character varying\"[]\n}\n\nTable tictactoe {\n  squares \"integer[3][3]\"\n}\n",
     blocks: [
         Table(
             TableBlock {
-                span_range: 0..94,
+                span_range: 0..153,
                 cols: [
                     TableColumn {
                         span_range: 18..30,
@@ -14,8 +14,8 @@ SchemaBlock {
                             to_string: "name",
                         },
                         type: ColumnType {
-                            span_range: 23..30,
-                            raw: "text\n  ",
+                            span_range: 23..27,
+                            raw: "text",
                             type_name: Raw(
                                 "text",
                             ),
@@ -113,6 +113,70 @@ SchemaBlock {
                             },
                         ),
                     },
+                    TableColumn {
+                        span_range: 95..121,
+                        name: Ident {
+                            span_range: 95..103,
+                            raw: "unquoted",
+                            to_string: "unquoted",
+                        },
+                        type: ColumnType {
+                            span_range: 104..110,
+                            raw: "text[]",
+                            type_name: Raw(
+                                "text",
+                            ),
+                            args: [],
+                            arrays: [
+                                None,
+                            ],
+                        },
+                        settings: Some(
+                            ColumnSettings {
+                                span_range: 111..121,
+                                attributes: [
+                                    Attribute {
+                                        span_range: 112..120,
+                                        key: Ident {
+                                            span_range: 112..120,
+                                            raw: "not null",
+                                            to_string: "not null",
+                                        },
+                                        value: None,
+                                    },
+                                ],
+                                is_pk: false,
+                                is_unique: false,
+                                nullable: Some(
+                                    NotNull,
+                                ),
+                                is_incremental: false,
+                                note: None,
+                                default: None,
+                                refs: [],
+                            },
+                        ),
+                    },
+                    TableColumn {
+                        span_range: 124..152,
+                        name: Ident {
+                            span_range: 124..129,
+                            raw: "mixed",
+                            to_string: "mixed",
+                        },
+                        type: ColumnType {
+                            span_range: 130..151,
+                            raw: "\"character varying\"[]",
+                            type_name: Raw(
+                                "character varying",
+                            ),
+                            args: [],
+                            arrays: [
+                                None,
+                            ],
+                        },
+                        settings: None,
+                    },
                 ],
                 ident: TableIdent {
                     span_range: 6..14,
@@ -131,17 +195,17 @@ SchemaBlock {
         ),
         Table(
             TableBlock {
-                span_range: 96..141,
+                span_range: 155..200,
                 cols: [
                     TableColumn {
-                        span_range: 116..140,
+                        span_range: 175..199,
                         name: Ident {
-                            span_range: 116..123,
+                            span_range: 175..182,
                             raw: "squares",
                             to_string: "squares",
                         },
                         type: ColumnType {
-                            span_range: 124..139,
+                            span_range: 183..198,
                             raw: "\"integer[3][3]\"",
                             type_name: Raw(
                                 "integer",
@@ -160,9 +224,9 @@ SchemaBlock {
                     },
                 ],
                 ident: TableIdent {
-                    span_range: 102..112,
+                    span_range: 161..171,
                     name: Ident {
-                        span_range: 102..111,
+                        span_range: 161..170,
                         raw: "tictactoe",
                         to_string: "tictactoe",
                     },

diff --git a/tests/out/comment.in.ron b/tests/out/comment.in.ron
@@ -14,8 +14,8 @@ SchemaBlock {
                             to_string: "id",
                         },
                         type: ColumnType {
-                            span_range: 27..31,
-                            raw: "int ",
+                            span_range: 27..30,
+                            raw: "int",
                             type_name: Raw(
                                 "int",
                             ),
@@ -54,8 +54,8 @@ SchemaBlock {
                             to_string: "user_id",
                         },
                         type: ColumnType {
-                            span_range: 61..65,
-                            raw: "int ",
+                            span_range: 61..64,
+                            raw: "int",
                             type_name: Raw(
                                 "int",
                             ),
@@ -105,8 +105,8 @@ SchemaBlock {
                             to_string: "status",
                         },
                         type: ColumnType {
-                            span_range: 93..101,
-                            raw: "varchar ",
+                            span_range: 93..100,
+                            raw: "varchar",
                             type_name: Raw(
                                 "varchar",
                             ),
@@ -155,8 +155,8 @@ SchemaBlock {
                             to_string: "created_at",
                         },
                         type: ColumnType {
-                            span_range: 143..151,
-                            raw: "varchar ",
+                            span_range: 143..150,
+                            raw: "varchar",
                             type_name: Raw(
                                 "varchar",
                             ),
-Original file line number
+Diff line change
@@ Expand Up / @@ -17,4 +17,4 @@ Table users { @@
         (`getdate()`, `upper(gender)`)
         (`reverse(country_code)`)
       }
-    }
+    }