Merge pull request #194 from dhis2/filtering

feat(filtering): support for filtering

server/package.json

@@ -18,6 +18,7 @@
     },
     "dependencies": {
         "@hapi/boom": "^9.0.0",
+        "@hapi/bounce": "^2.0.0",
         "@hapi/hapi": "^19.0.5",
         "@hapi/inert": "^6.0.1",
         "@hapi/joi": "^17.1.1",

server/src/models/v2/Default.js

@@ -37,24 +37,7 @@ const definition = joi
         stripUnknown: true,
     })
 
-/**
- * Creates a default function to validating
- * and transform results the given definition (a joi schema).
- * @param {Joi.Schema} defintion - The joi schema to use
- *
- * @returns {function(dbResult): []} - A function taking some data (may be an array or object) to be validated by the schema.
- * The function throws a ValidationError if mapping fails
- */
-function createDefaultValidator(schema) {
-    return function(dbResult) {
-        return Array.isArray(dbResult)
-            ? dbResult.map(v => joi.attempt(v, schema))
-            : joi.attempt(dbResult, schema)
-    }
-}
-
 module.exports = {
     def: definition,
     definition,
-    createDefaultValidator,
 }

server/src/models/v2/Organisation.js

@@ -1,9 +1,7 @@
-const joi = require('@hapi/joi')
+const joi = require('../../utils/CustomJoi')
 const User = require('./User')
-const {
-    definition: defaultDefinition,
-    createDefaultValidator,
-} = require('./Default')
+const { definition: defaultDefinition } = require('./Default')
+const { createDefaultValidator } = require('./helpers')
 
 const definition = defaultDefinition
     .append({
@@ -25,18 +23,12 @@ const definition = defaultDefinition
     .rename('created_by_user_id', 'owner', {
         ignoreUndefined: true,
     })
-
-const defWithUsers = definition.append({
-    users: joi
-        .array()
-        .items(User.definition)
-        .required(),
-})
+    .label('Organisation')
 
 const dbDefinition = definition.tailor('db')
 
 // internal -> external
-const externalDefintion = definition.tailor('external')
+const externalDefinition = definition.tailor('external')
 
 // database -> internal
 const parseDatabaseJson = createDefaultValidator(definition)
@@ -48,8 +40,7 @@ module.exports = {
     def: definition,
     definition,
     dbDefinition,
-    externalDefintion,
-    defWithUsers,
+    externalDefinition,
     parseDatabaseJson,
     formatDatabaseJson,
 }

server/src/models/v2/User.js

@@ -1,13 +1,13 @@
 const joi = require('@hapi/joi')
-const {
-    definition: defaultDefinition,
-    createDefaultValidator,
-} = require('./Default')
+const { definition: defaultDefinition } = require('./Default')
+const { createDefaultValidator } = require('./helpers')
 
-const definition = defaultDefinition.append({
-    name: joi.string(),
-    email: joi.string(),
-})
+const definition = defaultDefinition
+    .append({
+        name: joi.string(),
+        email: joi.string(),
+    })
+    .label('User')
 
 const parseDatabaseJson = createDefaultValidator(definition)
 

server/src/models/v2/helpers.js

@@ -0,0 +1,20 @@
+const Joi = require('@hapi/joi')
+
+/**
+ * Creates a default function to validating
+ * and transform results the given definition (a joi schema).
+ * @param {Joi.Schema} defintion - The joi schema to use
+ *
+ * @returns {function(dbResult): []} - A function taking some data (may be an array or object) to be validated by the schema.
+ * The function throws a ValidationError if mapping fails
+ */
+const createDefaultValidator = schema => {
+    return dbResult =>
+        Array.isArray(dbResult)
+            ? dbResult.map(v => Joi.attempt(v, schema))
+            : Joi.attempt(dbResult, schema)
+}
+
+module.exports = {
+    createDefaultValidator,
+}

server/src/plugins/apiRoutes.js

@@ -66,7 +66,9 @@ const apiRoutesPlugin = {
                     'Running without authentication requires to setup mapping to a user to use for requests requiring a current user id (e.g. creating apps for example). Set process.env.NO_AUTH_MAPPED_USER_ID',
                     '\x1b[m'
                 )
-                throw new Error("No auth set up. Set process.env.NO_AUTH_MAPPED_USER_ID to a valid user ID.")
+                throw new Error(
+                    'No auth set up. Set process.env.NO_AUTH_MAPPED_USER_ID to a valid user ID.'
+                )
             }
         }
 

server/src/plugins/errorMapper.js

@@ -11,6 +11,12 @@ const {
     wrapError,
 } = require('db-errors')
 
+/**
+ * A plugin that parses database-errors thrown in handlers and maps them to the correct boom-error.
+ * This means that in many cases we do not need to handle or catch db-query errors
+ * unless we need to do something else with the error.
+ */
+
 const dbErrorMap = {
     badRequest: [
         CheckViolationError,
@@ -27,7 +33,7 @@ const dbErrorMap = {
 const onPreResponseHandler = function(request, h) {
     const { response: error } = request
     // not error or joi-error - ignore
-    // By default validation errors thrown in handler are transformed to 500-error, as they are not Boom-errors.
+    // By default validation errors thrown in har are transformed to 500-error, as they are not Boom-errors.
     // This makes sense, as internal validation errors (outside validation-handlers) are developer errors
     if (!error.isBoom || error.isJoi) {
         return h.continue

server/src/plugins/queryFilter.js

@@ -0,0 +1,139 @@
+const Boom = require('@hapi/boom')
+const Bounce = require('@hapi/bounce')
+const Joi = require('@hapi/joi')
+const { Filters } = require('../utils/Filter')
+const { parseFilterString } = require('../utils/filterUtils')
+
+/**
+ * This plugin hooks into onPrehandler, processing all requests it's enabled for.
+ * It's designed to be used in conjunction with CustomJoi.js that extends Joi
+ * with a filter type. The filters are validated through Hapi's validate.query.
+ *
+ * The purpose of the plugin is to group all filters in `request.plugins.queryFilter.filters`.
+ * The value of this property is an instance of Filters.
+ *
+ * All options can be overwritten by each route.
+ * Options:
+ *  enabled - enable the queryFilter
+ *  ignoreKeys - ignore the key. Note that only keys of type Joi.filter() in the validate.query object are used.
+ *  rename - A Joi-schema with .rename() functions that are used to get renames of the keys.
+ *           Can also be a plain Object with keys being the query-name, and the value being the database-name.
+ *      This is used to rename API-facing filter-names to database-column names.
+ */
+
+const FILTER_TYPE = 'filter'
+
+const defaultOptions = {
+    enabled: false, // default disabled for all routes
+    method: ['GET'],
+    ignoreKeys: ['paging'],
+}
+
+// Holds Joi-schema descriptions, as these can be quite expensive to generate and this is a hot-path
+// cached by path as key
+
+// validateDescriptions is used to get queryParams with Filter-type
+const validateDescriptions = {}
+// renameDescriptions is used to rename the filter-Keys to the correct DB-column
+const renameDescriptions = {}
+
+const onPreHandler = function(request, h) {
+    const routeOptions = request.route.settings.plugins.queryFilter || {}
+
+    const options = {
+        ...this.options,
+        ...routeOptions,
+        ignoreKeys: this.options.ignoreKeys.concat(
+            routeOptions.ignoreKeys || []
+        ),
+    }
+
+    if (
+        !options.enabled ||
+        !options.method.includes(request.method.toUpperCase())
+    ) {
+        return h.continue
+    }
+
+    const routeQueryValidation = request.route.settings.validate.query
+    const rename = routeOptions.rename
+    const queryFilterKeys = new Set()
+    let renameMap = rename
+    let validateDescription = validateDescriptions[request.path]
+    let renameDescription = renameDescriptions[request.path]
+
+    if (rename && Joi.isSchema(rename)) {
+        if (!renameDescription) {
+            renameDescription = renameDescriptions[
+                request.path
+            ] = rename.describe()
+        }
+        renameMap = renameDescription.renames.reduce((acc, curr) => {
+            const { from, to } = curr
+            acc[from] = to
+            return acc
+        }, {})
+    }
+
+    if (Joi.isSchema(routeQueryValidation)) {
+        if (!validateDescription) {
+            validateDescription = validateDescriptions[
+                request.path
+            ] = routeQueryValidation.describe()
+        }
+        // only add validations with .filter()
+        Object.keys(validateDescription.keys).forEach(k => {
+            const keyDesc = validateDescription.keys[k]
+            if (keyDesc.type === FILTER_TYPE) {
+                queryFilterKeys.add(k)
+            }
+        })
+    } else {
+        // add all keys if no validation
+        Object.keys(request.query).forEach(key => {
+            const val = request.query[key]
+            const parsed = parseFilterString(val)
+            request.query[key] = parsed
+            queryFilterKeys.add(key)
+        })
+    }
+
+    const queryFilters = Object.keys(request.query).reduce((acc, curr) => {
+        if (
+            queryFilterKeys.has(curr) &&
+            options.ignoreKeys.indexOf(curr) === -1
+        ) {
+            acc[curr] = request.query[curr]
+        }
+        return acc
+    }, {})
+
+    try {
+        const filters = new Filters(queryFilters, {
+            renameMap,
+        })
+        request.plugins.queryFilter = filters
+    } catch (e) {
+        Bounce.rethrow(e, 'system')
+        throw Boom.boomify(e, { statusCode: 400 })
+    }
+
+    return h.continue
+}
+
+const filterPlugin = {
+    name: 'FilterPlugin',
+    register: async (server, options) => {
+        const opts = {
+            ...defaultOptions,
+            ...options,
+        }
+        server.bind({
+            options: opts,
+        })
+
+        server.ext('onPreHandler', onPreHandler)
+    },
+}
+
+module.exports = filterPlugin

server/src/plugins/staticFrontendRoutes.js

@@ -25,13 +25,7 @@ const staticFrontendRoutes = {
                 path: '/js/{param*}',
                 handler: {
                     directory: {
-                        path: path.join(
-                            __dirname,
-                            '..',
-                            '..',
-                            'static',
-                            'js'
-                        ),
+                        path: path.join(__dirname, '..', '..', 'static', 'js'),
                     },
                 },
             },

server/src/routes/v2/index.js

@@ -1,3 +1,7 @@
 const { flatten } = require('../../utils')
 
-module.exports = [require('./apps.js'), require('./channels.js'), require('./organisations')]
+module.exports = [
+    require('./apps.js'),
+    require('./channels.js'),
+    require('./organisations'),
+]