Display camera by name (#1928)

server/config/brain/camera/answers.en.json

@@ -1,15 +1,15 @@
 [
   {
-    "label": "camera.get-image-room.success",
+    "label": "camera.get-image.success",
     "answers": ["Here is what I see:"]
   },
 
   {
-    "label": "camera.get-image-room.no-image-found",
+    "label": "camera.get-image.no-image-found",
     "answers": ["Sorry, I can't get a recent image."]
   },
   {
-    "label": "camera.get-image-room.fail",
+    "label": "camera.get-image.fail",
     "answers": ["Sorry, something wrong happened while getting the camera"]
   }
 ]

server/config/brain/camera/answers.fr.json

@@ -1,15 +1,15 @@
 [
   {
-    "label": "camera.get-image-room.success",
+    "label": "camera.get-image.success",
     "answers": ["Voilà ce que je vois:"]
   },
 
   {
-    "label": "camera.get-image-room.no-image-found",
+    "label": "camera.get-image.no-image-found",
     "answers": ["Désolé, je n'ai pas trouvé d'image récente."]
   },
   {
-    "label": "camera.get-image-room.fail",
+    "label": "camera.get-image.fail",
     "answers": ["Désolé, je n'ai pas réussi à récupérer cette image de caméra"]
   }
 ]

server/config/brain/camera/questions.en.json

@@ -1,6 +1,11 @@
 [
   {
-    "label": "camera.get-image-room",
-    "questions": ["Show me a view of the camera in %room%", "Display camera image in %room%"]
+    "label": "camera.get-image",
+    "questions": [
+      "Show me a view of the camera in %room%",
+      "Display camera image in %room%",
+      "Show me a view of the camera %device%",
+      "Display camera image of %device%"
+    ]
   }
 ]

server/config/brain/camera/questions.fr.json

@@ -1,6 +1,13 @@
 [
   {
-    "label": "camera.get-image-room",
-    "questions": ["Montre moi la caméra du %room%", "Montre moi le %room%", "Montre moi la caméra de la %room%"]
+    "label": "camera.get-image",
+    "questions": [
+      "Montre moi la caméra du %room%",
+      "Montre moi le %room%",
+      "Montre moi la caméra de la %room%",
+      "Montre moi %device%",
+      "Montre moi la caméra %device%",
+      "Montre moi %device%"
+    ]
   }
 ]

server/lib/brain/index.js

@@ -19,6 +19,7 @@ const Brain = function Brain() {
   this.namedEntities = {
     room: new Map(),
     scene: new Map(),
+    device: new Map(),
   };
 };
 

server/lib/device/camera/camera.command.js

@@ -10,27 +10,35 @@ const { NotFoundError } = require('../../../utils/coreErrors');
  * light.command(message, classification, context);
  */
 async function command(message, classification, context) {
-  let cameraImage;
+  const cameraImages = [];
+
   const roomEntity = classification.entities.find((entity) => entity.entity === 'room');
+  const deviceEntity = classification.entities.find((entity) => entity.entity === 'device');
+
   try {
     switch (classification.intent) {
-      case 'camera.get-image-room':
-        if (!roomEntity) {
+      case 'camera.get-image':
+        if (roomEntity) {
+          cameraImages.push(...(await this.getImagesInRoom(roomEntity.option)));
+        } else if (deviceEntity) {
+          cameraImages.push(await this.getImage(deviceEntity.option));
+        } else {
           throw new NotFoundError('Room not found');
         }
-        cameraImage = await this.getImageInRoom(roomEntity.option);
-        if (cameraImage) {
-          this.messageManager.replyByIntent(message, 'camera.get-image-room.success', context, cameraImage);
+        if (cameraImages.length) {
+          cameraImages.forEach((cameraImage) => {
+            this.messageManager.replyByIntent(message, 'camera.get-image.success', context, cameraImage);
+          });
         } else {
-          this.messageManager.replyByIntent(message, 'camera.get-image-room.no-image-found', context);
+          this.messageManager.replyByIntent(message, 'camera.get-image.no-image-found', context);
         }
         break;
       default:
         throw new Error('Not found');
     }
   } catch (e) {
     logger.debug(e);
-    this.messageManager.replyByIntent(message, 'camera.get-image-room.fail', context);
+    this.messageManager.replyByIntent(message, 'camera.get-image.fail', context);
   }
   return null;
 }

server/lib/device/camera/camera.getImageInRoom.js → server/lib/device/camera/camera.getImagesInRoom.js

@@ -5,11 +5,11 @@ const { DEVICE_FEATURE_CATEGORIES } = require('../../../utils/constants');
 /**
  * @description Get image in room.
  * @param {string} roomId - Id of the room.
- * @returns {Promise} Resolve with the image.
+ * @returns {Promise} Resolve array with images.
  * @example
- * getImageInRoom('f0dea887-d14f-4344-a57b-795c16e0abda');
+ * getImagesInRoom('f0dea887-d14f-4344-a57b-795c16e0abda');
  */
-async function getImageInRoom(roomId) {
+async function getImagesInRoom(roomId) {
   const oneHourAgo = new Date(new Date().getTime() - 1 * 60 * 60 * 1000);
   const deviceFeatures = await db.DeviceFeature.findAll({
     attributes: ['last_value_string'],
@@ -34,13 +34,9 @@ async function getImageInRoom(roomId) {
     },
   });
 
-  if (deviceFeatures.length === 0) {
-    return null;
-  }
-
-  return deviceFeatures[0].last_value_string;
+  return deviceFeatures.map((deviceFeature) => deviceFeature.last_value_string);
 }
 
 module.exports = {
-  getImageInRoom,
+  getImagesInRoom,
 };

server/lib/device/camera/index.js

@@ -2,7 +2,7 @@ const { command } = require('./camera.command');
 const { setImage } = require('./camera.setImage');
 const { getImage } = require('./camera.getImage');
 const { get } = require('./camera.get');
-const { getImageInRoom } = require('./camera.getImageInRoom');
+const { getImagesInRoom } = require('./camera.getImagesInRoom');
 const { INTENTS } = require('../../../utils/constants');
 const { getLiveImage } = require('./camera.getLiveImage');
 
@@ -12,14 +12,14 @@ const Camera = function Camera(stateManager, messageManager, eventManager, servi
   this.eventManager = eventManager;
   this.serviceManager = serviceManager;
   this.deviceManager = deviceManager;
-  this.eventManager.on(INTENTS.CAMERA.GET_IMAGE_ROOM, this.command.bind(this));
+  this.eventManager.on(INTENTS.CAMERA.GET_IMAGE, this.command.bind(this));
 };
 
 Camera.prototype.command = command;
 Camera.prototype.setImage = setImage;
 Camera.prototype.getImage = getImage;
 Camera.prototype.get = get;
-Camera.prototype.getImageInRoom = getImageInRoom;
+Camera.prototype.getImagesInRoom = getImagesInRoom;
 Camera.prototype.getLiveImage = getLiveImage;
 
 module.exports = Camera;

server/lib/device/device.create.js

@@ -82,6 +82,8 @@ async function create(device) {
     if (deviceInDb === null) {
       deviceInDb = await db.Device.create(device, { transaction });
     } else {
+      this.brain.removeNamedEntity('device', deviceInDb.identifier, deviceInDb.name);
+
       actionEvent = EVENTS.DEVICE.UPDATE;
       oldPollFrequency = deviceInDb.poll_frequency;
 
@@ -107,6 +109,8 @@ async function create(device) {
     deviceToReturn.features = deviceToReturn.features || [];
     deviceToReturn.params = deviceToReturn.params || [];
 
+    this.brain.addNamedEntity('device', deviceToReturn.selector, deviceToReturn.name);
+
     // if we need to create features
     const newFeatures = await Promise.map(features, async (feature) => {
       // if the device feature already exist

server/lib/device/device.init.js

@@ -35,6 +35,7 @@ async function init(startDeviceStateAggregate = true) {
   const plainDevices = devices.map((device) => {
     const plainDevice = device.get({ plain: true });
     this.add(plainDevice);
+    this.brain.addNamedEntity('device', plainDevice.selector, plainDevice.name);
     return plainDevice;
   });
   if (startDeviceStateAggregate) {

server/lib/device/index.js

@@ -43,6 +43,7 @@ const DeviceManager = function DeviceManager(
   roomManager,
   variable,
   job,
+  brain,
 ) {
   this.eventManager = eventManager;
   this.messageManager = messageManager;
@@ -51,6 +52,7 @@ const DeviceManager = function DeviceManager(
   this.roomManager = roomManager;
   this.variable = variable;
   this.job = job;
+  this.brain = brain;
 
   this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH = 1000;
   this.WAIT_TIME_BETWEEN_DEVICE_FEATURE_CLEAN_BATCH = 100;

server/lib/gateway/gateway.forwardMessageToOpenAI.js

@@ -1,7 +1,7 @@
 const { Error429 } = require('../../utils/httpErrors');
 
 const intentTranslation = {
-  SHOW_CAMERA: 'camera.get-image-room',
+  SHOW_CAMERA: 'camera.get-image',
   TURN_ON: 'light.turn-on',
   TURN_OFF: 'light.turn-off',
   GET_TEMPERATURE: 'temperature-sensor.get-in-room',
@@ -44,6 +44,12 @@ async function forwardMessageToOpenAI({ message, previousQuestions, context }) {
       classification.entities = [{ entity: 'scene', option: sceneSelector, sourceText: response.scene }];
     }
 
+    // add device entity
+    if (response.device) {
+      const deviceSelector = this.brain.getEntityIdByName('device', response.device);
+      classification.entities = [{ entity: 'device', option: deviceSelector, sourceText: response.device }];
+    }
+
     classification.intent = intentTranslation[response.type];
 
     // Reply with OpenAI response

server/lib/index.js

@@ -66,7 +66,7 @@ function Gladys(params = {}) {
   const message = new MessageHandler(event, brain, service, stateManager, variable);
   const user = new User(session, stateManager, variable);
   const location = new Location(user, event);
-  const device = new Device(event, message, stateManager, service, room, variable, job);
+  const device = new Device(event, message, stateManager, service, room, variable, job, brain);
   const calendar = new Calendar(service);
   const scheduler = new Scheduler(event);
   const weather = new Weather(service, event, message, house);

server/test/lib/device/camera/camera.command.test.js

@@ -3,6 +3,7 @@ const { assert, fake } = require('sinon');
 const Device = require('../../../../lib/device');
 const StateManager = require('../../../../lib/state');
 const Job = require('../../../../lib/job');
+const { DEVICE_FEATURE_CATEGORIES, DEVICE_FEATURE_TYPES } = require('../../../../utils/constants');
 
 const event = new EventEmitter();
 
@@ -20,14 +21,14 @@ const RANDOM_IMAGE =
   'image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg==';
 
 describe('Camera.command', () => {
-  it('should respond with image from camera', async () => {
+  it('should respond with image from camera of room', async () => {
     const stateManager = new StateManager(event);
     const deviceManager = new Device(event, messageManager, stateManager, {}, {}, {}, job);
     const context = {};
     await deviceManager.camera.command(
       message,
       {
-        intent: 'camera.get-image-room',
+        intent: 'camera.get-image',
         entities: [
           {
             start: 25,
@@ -45,7 +46,44 @@ describe('Camera.command', () => {
       },
       context,
     );
-    assert.calledWith(messageManager.replyByIntent, message, 'camera.get-image-room.success', context, RANDOM_IMAGE);
+    assert.calledWith(messageManager.replyByIntent, message, 'camera.get-image.success', context, RANDOM_IMAGE);
+  });
+  it('should respond with image from camera', async () => {
+    const stateManager = new StateManager(event);
+    stateManager.setState('device', 'camera-1', {
+      features: [
+        {
+          category: DEVICE_FEATURE_CATEGORIES.CAMERA,
+          type: DEVICE_FEATURE_TYPES.CAMERA.IMAGE,
+          last_value_changed: new Date().toISOString(),
+          last_value_string: RANDOM_IMAGE,
+        },
+      ],
+    });
+    const deviceManager = new Device(event, messageManager, stateManager, {}, {}, {}, job);
+    const context = {};
+    await deviceManager.camera.command(
+      message,
+      {
+        intent: 'camera.get-image',
+        entities: [
+          {
+            start: 25,
+            end: 31,
+            len: 7,
+            levenshtein: 0,
+            accuracy: 1,
+            entity: 'device',
+            type: 'enum',
+            option: 'camera-1',
+            sourceText: 'camera-1',
+            utteranceText: 'camera-1',
+          },
+        ],
+      },
+      context,
+    );
+    assert.calledWith(messageManager.replyByIntent, message, 'camera.get-image.success', context, RANDOM_IMAGE);
   });
   it('should respond camera not found', async () => {
     const stateManager = new StateManager(event);
@@ -54,7 +92,7 @@ describe('Camera.command', () => {
     await deviceManager.camera.command(
       message,
       {
-        intent: 'camera.get-image-room',
+        intent: 'camera.get-image',
         entities: [
           {
             start: 25,
@@ -72,7 +110,7 @@ describe('Camera.command', () => {
       },
       context,
     );
-    assert.calledWith(messageManager.replyByIntent, message, 'camera.get-image-room.no-image-found', context);
+    assert.calledWith(messageManager.replyByIntent, message, 'camera.get-image.no-image-found', context);
   });
   it('should respond camera not found', async () => {
     const stateManager = new StateManager(event);
@@ -81,11 +119,11 @@ describe('Camera.command', () => {
     await deviceManager.camera.command(
       message,
       {
-        intent: 'camera.get-image-room',
+        intent: 'camera.get-image',
         entities: [],
       },
       context,
     );
-    assert.calledWith(messageManager.replyByIntent, message, 'camera.get-image-room.no-image-found', context);
+    assert.calledWith(messageManager.replyByIntent, message, 'camera.get-image.no-image-found', context);
   });
 });

server/test/lib/device/camera/camera.getImageInRoom.test.js

@@ -10,7 +10,7 @@ const job = new Job(event);
 const RANDOM_IMAGE =
   'image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg==';
 
-describe('Camera.getImageInRoom', () => {
+describe('Camera.getImagesInRoom', () => {
   it('should return image', async () => {
     const stateManager = new StateManager(event);
     const deviceManager = new Device(event, {}, stateManager, {}, {}, {}, job);
@@ -26,7 +26,7 @@ describe('Camera.getImageInRoom', () => {
       ],
     });
     await deviceManager.camera.setImage('test-camera', RANDOM_IMAGE);
-    const cameraImage = await deviceManager.camera.getImageInRoom('2398c689-8b47-43cc-ad32-e98d9be098b5');
-    expect(cameraImage).to.equal(RANDOM_IMAGE);
+    const cameraImage = await deviceManager.camera.getImagesInRoom('2398c689-8b47-43cc-ad32-e98d9be098b5');
+    expect(cameraImage[0]).to.equal(RANDOM_IMAGE);
   });
 });