Entities Integrity URL

docs/api.yaml

@@ -40,14 +40,22 @@ info:
 
     Here major and breaking changes to the API are listed by version.
 
+    ## ODK Central v2025.1
+
+    **Added**:
+    - [RESTORE](/central-api-entity-management/#restoring-a-deleted-entity) endpoint for Entities.
+    - Entities that have been soft-deleted for 30 days will automatically be purged. 
+    - [Entities Odata](/central-api-odata-endpoints/#id3) now returns `__system/deletedAt`. It can also be used in $filter, $sort and $select query parameters.
+    - [Integrity](/central-api-openrosa-endpoints/#openrosa-dataset-integrity-api) endpoint for the Entity list.
+
     ## ODK Central v2024.3
 
     **Added**:
     - Endpoints for managing [User Preferences](/central-api-accounts-and-users/#user-preferences), mainly to be used by the frontend.
 
     **Changed**:
 
-    - [Submissions Odata]() now returns `__system/deletedAt`. It can also be used in $filter, $sort and $select query parameters.
+    - [Submissions Odata](/central-api-odata-endpoints/#data-document) now returns `__system/deletedAt`. It can also be used in $filter, $sort and $select query parameters.
     - Dataset (Entity List) properties with the same name but different capitalization are not allowed.
     - Form Attachments for both [published Forms](/central-api-form-management/#listing-form-attachments) and [draft Forms](/central-api-form-management/#listing-expected-draft-form-attachments) now return a property representing the hash of the attachment file.
 
@@ -9155,51 +9163,6 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error403'
-    delete:
-      tags:
-      - Entity Management
-      summary: Deleting an Entity
-      description: Use this API to delete an Entity. With this API, Entity is soft-deleted,
-        which means it is still in the database and you can retreive it by passing
-        `?deleted=true` to [GET /projects/:id/datasets/:name/entities](/central-api-entity-management/#entities-metadata).
-        In the future, we will provide a way to restore deleted entities and purge
-        deleted entities.
-      operationId: Deleting an Entity
-      parameters:
-      - name: projectId
-        in: path
-        description: The numeric ID of the Project
-        required: true
-        schema:
-          type: number
-        example: "16"
-      - name: name
-        in: path
-        description: Name of the Dataset
-        required: true
-        schema:
-          type: string
-        example: people
-      - name: uuid
-        in: path
-        description: UUID of the Entity
-        required: true
-        schema:
-          type: string
-        example: 54a405a0-53ce-4748-9788-d23a30cc3afa
-      responses:
-        200:
-          description: OK
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Success'
-        403:
-          description: Forbidden
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Error403'
     patch:
       tags:
       - Entity Management
@@ -9314,6 +9277,100 @@ paths:
               schema:
                 $ref: '#/components/schemas/Error403'
       x-codegen-request-body-name: body
+    delete:
+      tags:
+      - Entity Management
+      summary: Deleting an Entity
+      description: Use this API to delete an Entity. With this API, Entity is soft-deleted,
+        which means it is still in the database and you can retreive it by passing
+        `?deleted=true` to [GET /projects/:id/datasets/:name/entities](/central-api-entity-management/#entities-metadata).
+        In the future, we will provide a way to restore deleted entities and purge
+        deleted entities.
+      operationId: Deleting an Entity
+      parameters:
+      - name: projectId
+        in: path
+        description: The numeric ID of the Project
+        required: true
+        schema:
+          type: number
+        example: "16"
+      - name: name
+        in: path
+        description: Name of the Dataset
+        required: true
+        schema:
+          type: string
+        example: people
+      - name: uuid
+        in: path
+        description: UUID of the Entity
+        required: true
+        schema:
+          type: string
+        example: 54a405a0-53ce-4748-9788-d23a30cc3afa
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Success'
+        403:
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error403'
+
+  /projects/{projectId}/datasets/{name}/entities/{uuid}/restore:
+    post:
+      tags:
+      - Entity Management
+      summary: Restoring a deleted Entity
+      description: Entities that have been recently soft-deleted and not yet purged can be restored using this endpoint.
+      operationId: Restoring a deleted Entity
+      parameters:
+      - name: projectId
+        in: path
+        description: The numeric ID of the Project
+        required: true
+        schema:
+          type: number
+        example: "16"
+      - name: name
+        in: path
+        description: Name of the Dataset
+        required: true
+        schema:
+          type: string
+        example: people
+      - name: uuid
+        in: path
+        description: UUID of the Entity
+        required: true
+        schema:
+          type: string
+        example: 54a405a0-53ce-4748-9788-d23a30cc3afa
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Success'
+        403:
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error403'
+        404:
+          description: Not Found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error404'
   /projects/{projectId}/datasets/{name}/entities/{uuid}/versions:
     get:
       tags:
@@ -9734,6 +9791,12 @@ paths:
         * The Manifest will only output information for files the server actually has in its possession. Any missing expected files will be omitted, as we cannot provide a `hash` or `downloadUrl` for them.
 
         * For Attachments that are linked to a Dataset, the value of `hash` is calculated using the MD5 of the last updated timestamp of the Dataset, instead of the content of the Dataset.
+
+        [Offline Entities support](https://forum.getodk.org/t/openrosa-spec-proposal-support-offline-entities/48052):
+
+        * If an attachemnt is linked to a Dataset, then `type="entityList"` attribute is added to the `mediaFile` element.
+
+        * `integrityUrl` is also returned for the attachments that are linked to a Dataset.
       operationId: OpenRosa Form Manifest API
       parameters:
       - name: projectId
@@ -9778,6 +9841,12 @@ paths:
                     <hash>md5:a6fdc426037143cf71cced68e2532e3c</hash>
                     <downloadUrl>https://your.odk.server/v1/projects/7/forms/basic/attachments/question2.jpg</downloadUrl>
                   </mediaFile>
+                  <mediaFile type="entityList">
+                    <filename>people.csv</filename>
+                    <hash>md5:9fd39ac868eccdc0c134b3b7a6a25eb7</hash>
+                    <downloadUrl>https://your.odk.server/v1/projects/7/forms/basic/attachments/people.csv</downloadUrl>
+                    <integrityUrl>https://your.odk.server/v1/projects/7/datasets/people/integrity</integrityUrl>
+                  </mediaFile>
                 </manifest>
         403:
           description: Forbidden
@@ -9791,6 +9860,79 @@ paths:
                 <OpenRosaResponse xmlns="http://openrosa.org/http/response" items="0">
                   <message nature="error">The authenticated actor does not have rights to perform that action.</message>
                 </OpenRosaResponse>
+  /v1/projects/{projectId}/datasets/{name}/integrity?id={UUIDs}:
+    get:
+      tags:
+      - OpenRosa Endpoints
+      summary: OpenRosa Dataset Integrity API
+      description: |-
+        _(introduced: version 2025.1)_
+
+        This is the fully standards-compliant implementation of the Entities Integrity API as described in [OpenRosa spec proposal: support offline Entities](https://forum.getodk.org/t/openrosa-spec-proposal-support-offline-entities/48052).
+
+        This returns the `deleted` flag of the Entities requested through `id` query parameter. If no `id` is provided then all Entities are return.
+      operationId: OpenRosa Form Manifest API
+      parameters:
+      - name: projectId
+        in: path
+        description: The numeric ID of the Project
+        required: true
+        schema:
+          type: number
+        example: "7"
+      - name: name
+        in: path
+        description: The `name` of the dataset being referenced.
+        required: true
+        schema:
+          type: string
+        example: people
+      - name: id
+        in: query
+        description: The comma separated UUIDs of the Entities
+        required: true
+        schema:
+          type: string
+        example: 6fdfa3b6-64fb-46cf-b98c-c92b57f914b1,97717278-2bf8-4565-88b2-711c88d66e75
+      - name: X-OpenRosa-Version
+        in: header
+        description: e.g. 1.0
+        schema:
+          type: string
+        example: "1.0"
+      responses:
+        200:
+          description: OK
+          headers:
+            X-OpenRosa-Version:
+              schema:
+                type: string
+          content:
+            text/xml:
+              example: |
+                <?xml version="1.0" encoding="UTF-8"?>
+                <data>
+                  <entities>
+                    <entity id="6fdfa3b6-64fb-46cf-b98c-c92b57f914b1">
+                      <deleted>true</deleted>
+                    </entity>
+                    <entity id="97717278-2bf8-4565-88b2-711c88d66e75">
+                      <deleted>false</deleted>
+                    </entity>
+                  </entities>
+                </data>
+        403:
+          description: Forbidden
+          headers:
+            X-OpenRosa-Version:
+              schema:
+                type: string
+          content:
+            text/xml:
+              example: |
+                <OpenRosaResponse xmlns="http://openrosa.org/http/response" items="0">
+                  <message nature="error">The authenticated actor does not have rights to perform that action.</message>
+                </OpenRosaResponse>
   /v1/test/{token}/projects/{projectId}/forms/{xmlFormId}/draft/formList:
     get:
       tags:
@@ -10912,8 +11054,9 @@ paths:
         | Entity Timestamp        | `__system/createdAt` |
         | Entity Update Timestamp | `__system/updatedAt` |
         | Entity Conflict         | `__system/conflict`  |
+        | Entity Delete Timestamp | `__system/deletedAt` |
 
-        Note that `createdAt` and `updatedAt` are time components. This means that any comparisons you make need to account for the full time of the entity. It might seem like `$filter=__system/createdAt le 2020-01-31` would return all results on or before 31 Jan 2020, but in fact only entities made before midnight of that day would be accepted. To include all of the month of January, you need to filter by either `$filter=__system/createdAt le 2020-01-31T23:59:59.999Z` or `$filter=__system/createdAt lt 2020-02-01`. Remember also that you can [query by a specific timezone](https://en.wikipedia.org/wiki/ISO_8601#Time_offsets_from_UTC).
+        Note that `createdAt`, `updatedAt` and `deletedAt` are time components. This means that any comparisons you make need to account for the full time of the entity. It might seem like `$filter=__system/createdAt le 2020-01-31` would return all results on or before 31 Jan 2020, but in fact only entities made before midnight of that day would be accepted. To include all of the month of January, you need to filter by either `$filter=__system/createdAt le 2020-01-31T23:59:59.999Z` or `$filter=__system/createdAt lt 2020-02-01`. Remember also that you can [query by a specific timezone](https://en.wikipedia.org/wiki/ISO_8601#Time_offsets_from_UTC).
 
         Please see the [OData documentation](http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#_Toc31358948) on `$filter` [operations](http://docs.oasis-open.org/odata/odata/v4.01/cs01/part1-protocol/odata-v4.01-cs01-part1-protocol.html#sec_BuiltinFilterOperations) and [functions](http://docs.oasis-open.org/odata/odata/v4.01/cs01/part1-protocol/odata-v4.01-cs01-part1-protocol.html#sec_BuiltinQueryFunctions) for more information.
 
@@ -11007,6 +11150,7 @@ paths:
                     creatorName: Tree surveyor
                     updates: 1,
                     updatedAt: '2023-04-31T19:41:16.478Z'
+                    deletedAt: null
                     version: 1
                     conflict: null
                 - __id: aeebd746-3b1e-4a24-ba9d-ed6547bd5ff1
@@ -11017,6 +11161,7 @@ paths:
                     creatorName: Tree surveyor
                     updates: 1,
                     updatedAt: '2023-04-31T19:41:16.478Z'
+                    deletedAt: null
                     version: 2
                     conflict: null
                   geometry: 47.722581 18.562111 0 0,

lib/bin/purge.js

@@ -21,11 +21,13 @@ const { purgeTask } = require('../task/purge');
 
 const { program } = require('commander');
 program.option('-f, --force', 'Force any soft-deleted form to be purged right away.');
-program.option('-m, --mode <value>', 'Mode of purging. Can be "forms", "submissions", or "all". Default is "all".', 'all');
+program.option('-m, --mode <value>', 'Mode of purging. Can be "forms", "submissions", "entities" or "all". Default is "all".', 'all');
 program.option('-i, --formId <integer>', 'Purge a specific form based on its id.', parseInt);
 program.option('-p, --projectId <integer>', 'Restrict purging to a specific project.', parseInt);
 program.option('-x, --xmlFormId <value>', 'Restrict purging to specific form based on xmlFormId (must be used with projectId).');
 program.option('-s, --instanceId <value>', 'Restrict purging to a specific submission based on instanceId (use with projectId and xmlFormId).');
+program.option('-d, --datasetName <value>', 'Restrict purging to specific dataset/entity-list based on its name (must be used with projectId).');
+program.option('-e, --entityUuid <value>', 'Restrict purging to a specific entitiy based on its UUID (use with projectId and datasetName).');
 
 program.parse();
 

lib/data/entity.js

@@ -24,6 +24,7 @@ const { sanitizeOdataIdentifier, blankStringToNull, isBlank } = require('../util
 const odataToColumnMap = new Map([
   ['__system/createdAt', 'entities.createdAt'],
   ['__system/updatedAt', 'entities.updatedAt'],
+  ['__system/deletedAt', 'entities.deletedAt'],
   ['__system/creatorId', 'entities.creatorId'],
   ['__system/conflict', 'entities.conflict']
 ]);
@@ -284,6 +285,7 @@ const selectFields = (entity, properties, selectedProperties) => {
     creatorName: entity.aux.creator.displayName,
     updates: entity.updates,
     updatedAt: entity.updatedAt,
+    deletedAt: entity.deletedAt,
     version: entity.def.version,
     conflict: entity.conflict
   };

lib/formats/odata.js

@@ -233,6 +233,7 @@ const edmxTemplaterForEntities = template(`<?xml version="1.0" encoding="UTF-8"?
         <Property Name="creatorName" Type="Edm.String"/>        
         <Property Name="updates" Type="Edm.Int64"/>
         <Property Name="updatedAt" Type="Edm.DateTimeOffset"/>
+        <Property Name="deletedAt" Type="Edm.DateTimeOffset"/>
         <Property Name="version" Type="Edm.Int64"/>
         <Property Name="conflict" Type="Edm.String"/>
       </ComplexType>

lib/formats/openrosa.js

@@ -66,6 +66,9 @@ const formManifestTemplate = template(200, `<?xml version="1.0" encoding="UTF-8"
       <filename>{{name}}</filename>
       <hash>md5:{{openRosaHash}}</hash>
       <downloadUrl>{{{domain}}}{{{basePath}}}/attachments/{{urlName}}</downloadUrl>
+      {{#integrityUrl}}
+      <integrityUrl>{{{integrityUrl}}}</integrityUrl>
+      {{/integrityUrl}}
     </mediaFile>
     {{/hasSource}}
   {{/attachments}}
@@ -77,7 +80,10 @@ const formManifest = (data) => formManifestTemplate(mergeRight(data, {
     attachment.with({
       hasSource: attachment.blobId || attachment.datasetId,
       urlName: encodeURIComponent(attachment.name),
-      isDataset: attachment.datasetId != null
+      isDataset: attachment.datasetId != null,
+      integrityUrl: attachment.datasetId ?
+        `${data.domain}${data.projectPath}/datasets/${encodeURIComponent(attachment.name.replace(/\.[^/.]+$/, ''))}/integrity`
+        : null
     }))
 }));
 
@@ -87,5 +93,16 @@ const openRosaErrorTemplate = openRosaMessageBase('error');
 parse(openRosaErrorTemplate);
 const openRosaError = (message) => render(openRosaErrorTemplate, { message });
 
-module.exports = { createdMessage, formList, formManifest, openRosaError };
+const entityListTemplate = template(200, `<?xml version="1.0" encoding="UTF-8"?>
+  <data>
+    <entities>
+    {{#entities}}
+      <entity id="{{uuid}}">
+        <deleted>{{deleted}}</deleted>
+      </entity>
+    {{/entities}}
+    </entities>
+  </data>`);
+const entityList = (data) => entityListTemplate(data);
+module.exports = { createdMessage, formList, formManifest, openRosaError, entityList };