Update entity import to use insertMany

lib/model/frames/entity.js

@@ -9,19 +9,26 @@
 
 /* eslint-disable no-multi-spaces */
 
-const { embedded, Frame, readable, table } = require('../frame');
+const { embedded, fieldTypes, Frame, readable, table } = require('../frame');
 const { extractEntity, normalizeUuid, extractLabelFromSubmission, extractBaseVersionFromSubmission } = require('../../data/entity');
 
 // These Frames don't interact with APIs directly, hence no readable/writable
 class Entity extends Frame.define(
   table('entities', 'entity'),
   'id',                     'uuid', readable,
-  'datasetId',
-  'createdAt', readable,    'creatorId', readable,
-  'updatedAt', readable,    'deletedAt', readable,
+  'datasetId',              'creatorId', readable,
   'conflict', readable,
+  'createdAt', readable,
+  'updatedAt', readable,    'deletedAt', readable,
   embedded('creator'),
-  embedded('currentVersion')
+  embedded('currentVersion'),
+  fieldTypes([
+    'int4', 'varchar',
+    'int4', 'int4',
+    'conflictType',
+    'timestamptz',
+    'timestamptz', 'timestamptz',
+  ])
 ) {
   get def() { return this.aux.def; }
 
@@ -75,14 +82,25 @@ Entity.Extended = class extends Frame.define(
 Entity.Def = Frame.define(
   table('entity_defs', 'def'),
   'id',                             'entityId',
-  'createdAt',    readable,         'current',      readable,
+  'current',      readable,
   'sourceId',                       'label',        readable,
   'creatorId',    readable,         'userAgent',    readable,
   'data',         readable,         'root',
   'version',      readable,         'baseVersion',  readable,
   'dataReceived', readable,         'conflictingProperties', readable,
+  'createdAt',    readable,
   embedded('creator'),
-  embedded('source')
+  embedded('source'),
+  fieldTypes([
+    'int4', 'int4',
+    'bool',
+    'int4', 'varchar',
+    'int4', 'varchar',
+    'jsonb', 'bool',
+    'int4', 'int4',
+    'jsonb', 'jsonb',
+    'timestamptz'
+  ])
 );
 
 Entity.Def.Metadata = class extends Entity.Def {

lib/model/query/entities.js

@@ -9,7 +9,7 @@
 
 const { sql } = require('slonik');
 const { Actor, Entity, Submission, Form } = require('../frames');
-const { equals, extender, unjoiner, page, markDeleted } = require('../../util/db');
+const { equals, extender, unjoiner, page, markDeleted, insertMany } = require('../../util/db');
 const { map, mergeRight, pickAll } = require('ramda');
 const { blankStringToNull, construct } = require('../../util/util');
 const { QueryOptions } = require('../../util/db');
@@ -90,49 +90,29 @@ createNew.audit.withResult = true;
 // it could be used in places of createNew() but createNew uses a single query so it may be faster
 // in single entity situations (eg. processing submissions to make entities)
 // Note: if the entity schema changes, createMany and createNew would both need to change.
-const createMany = (dataset, entities, sourceId, userAgentIn) => async ({ all, context }) => {
+const createMany = (dataset, rawEntities, sourceId, userAgentIn) => async ({ all, context }) => {
   const creatorId = context.auth.actor.map((actor) => actor.id).orNull();
   const userAgent = blankStringToNull(userAgentIn);
 
-  const entityRows = entities.map(e => [dataset.id, e.uuid, creatorId]);
-  const entityColumnTypes = ['int4', 'uuid', 'int4'];
-  const newEntities = await all(sql`
-  INSERT INTO entities ("createdAt", "datasetId", "uuid", "creatorId")
-  SELECT clock_timestamp(), * FROM ${sql.unnest(entityRows, entityColumnTypes)} as t
-  RETURNING id`);
-
-  const defRows = entities.map((e, i) => [
-    newEntities[i].id,
-    e.def.label,
-    JSON.stringify(e.def.data),
-    JSON.stringify(e.def.dataReceived),
-    sourceId,
+  // Augment parsed entity data with dataset and creator IDs
+  const entitiesForInsert = rawEntities.map(e => new Entity({ datasetId: dataset.id, creatorId, ...e }));
+
+  const entities = await all(insertMany(entitiesForInsert));
+
+  // Augment defs with IDs of freshly inserted entities and
+  // other default values
+  const defsForInsert = rawEntities.map((e, i) => new Entity.Def({
+    entityId: entities[i].id,
     creatorId,
+    root: true,
+    current: true,
+    sourceId,
+    version: 1,
     userAgent,
-    'true',
-    'true',
-    '1'
-  ]);
-  const defColumnTypes = [
-    'int4',
-    'text',
-    'json',
-    'json',
-    'int4',
-    'int4',
-    'text',
-    'bool',
-    'bool',
-    'int4'
-  ];
-
-  const defs = await all(sql`
-  INSERT INTO entity_defs ("createdAt", "entityId", "label", "data", "dataReceived",
-    "sourceId", "creatorId", "userAgent", "root", "current", "version")
-  SELECT clock_timestamp(), * FROM ${sql.unnest(defRows, defColumnTypes)} as t
-  RETURNING *`);
-
-  return defs;
+    ...e.def,
+    data: { ...e.def.data }, // was [Object: null prototype]
+  }));
+  return all(insertMany(defsForInsert));
 };
 
 createMany.audit = (dataset, entities, sourceId) => (log) =>

lib/util/db.js

@@ -179,7 +179,8 @@ const insertMany = (objs) => {
 
   return sql`
   INSERT INTO ${raw(Type.table)} (${columns})
-  SELECT ${selectExp} FROM ${sql.unnest(rows, columnTypes)} AS t`;
+  SELECT ${selectExp} FROM ${sql.unnest(rows, columnTypes)} AS t
+  RETURNING *`;
 };
 
 // generic update utility

test/unit/util/db.js

@@ -319,7 +319,8 @@ returning *`);
       const query = insertMany([ new T({ x: 2 }), new T({ y: 3 }) ]);
       query.sql.should.be.eql(`
   INSERT INTO dogs ("x","y")
-  SELECT * FROM unnest($1::"text"[], $2::"text"[]) AS t`);
+  SELECT * FROM unnest($1::"text"[], $2::"text"[]) AS t
+  RETURNING *`);
       query.values.should.be.eql([
         [2, null],
         [null, 3]
@@ -331,7 +332,8 @@ returning *`);
       const query = insertMany([ new U({ x: new Date('2000-01-01') }), new U() ]);
       query.sql.should.be.eql(`
   INSERT INTO dogs ("createdAt", "x")
-  SELECT clock_timestamp(), * FROM unnest($1::"timestamptz"[]) AS t`);
+  SELECT clock_timestamp(), * FROM unnest($1::"timestamptz"[]) AS t
+  RETURNING *`);
       query.values.should.be.eql([
         ['2000-01-01T00:00:00.000Z', null]
       ]);