PoC gui test automation

Create gui test suite with common setup. Create
generic keywords for launching and closing any app
with gui test automation.

Create example test cases for chromium and firefox.
Added test case for logout and login, and test for
booting to login screen.

Test case for GUI reboot is added. This test case
verifies also that the device boots to login screen,
not directly to desktop.

Login is included in the suite setup because nearly
all gui tests depend on login. However, login is also
added as a separate test case (after reboot test case).

Log out is added as a separate test case. It should
be taken care that this test will be the last one
in the suite.

Gui tests are expected to be run from logged out
state but the login keyword is designed so that it
does not fail even when run in already logged in state.

Graphical details and icons used in image recognition
are copied from run-time ghaf gui-vm.

Added new ssh keyword 'Connect to VM if not already
connected' to speed up testing by reducing unnecessary
re-connect cycles.

Fixed also a bug in iperf test report message showing
only tx results.

Signed-off-by: Samuli Leivo <[email protected]>

Robot-Framework/lib/gui_testing.py

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII)
+# SPDX-License-Identifier: Apache-2.0
+
+from pyscreeze import locate, center
+import logging
+import subprocess
+
+
+def locate_image(image, confidence):
+    screenshot = "./screenshot.png"
+    image_box = locate(image, screenshot, confidence=confidence)
+    image_center = center(image_box)
+    logging.info(image_box)
+    logging.info(image_center)
+    image_center_in_mouse_coordinates = convert_resolution(image_center, screenshot)
+    logging.info(image_center_in_mouse_coordinates)
+    return image_center_in_mouse_coordinates
+
+def convert_resolution(coordinates, screenshot):
+    # Currently default screenshot image resolution is 1920x1200
+    # but ydotool mouse movement resolution was tested to be 960x600.
+    # Testing shows that this scaling ratio stays fixed even if changing the display resolution:
+    # ydotool mouse resolution changes in relation to display resolution.
+    # Hence we can use the hardcoded value.
+    scaling_factor = 2
+    mouse_coordinates = {
+        'x': int(coordinates[0] / scaling_factor),
+        'y': int(coordinates[1] / scaling_factor)
+    }
+    return mouse_coordinates
+
+def convert_app_icon(crop, background, input_file='icon.svg', output_file='icon.png'):
+    if background != "none":
+        subprocess.run(['magick', '-background', background, input_file, '-gravity', 'center', '-extent',
+                        '{}x{}'.format(crop, crop), output_file])
+    else:
+        subprocess.run(['magick', input_file, '-gravity', 'center', '-extent',
+                        '{}x{}'.format(crop, crop), output_file])
+    return
+
+def negate_app_icon(input_file, output_file):
+    subprocess.run(['magick', input_file, '-negate', output_file])

Robot-Framework/resources/common_keywords.resource

@@ -19,8 +19,64 @@ Check that the application was started
     Should Not Be Empty  ${app_pids}  ${app_name} is not started
     Log To Console       ${app_name} is started
 
+Check that the application is not running
+    [Arguments]          ${app_name}  ${range}=2
+    ${pids}=  Set Variable  ${EMPTY}
+    FOR   ${i}   IN RANGE  ${range}
+        ${keyword_status}  ${pids}  Run Keyword And Ignore Error   Find pid by name    ${app_name}
+        ${status}    Run Keyword And Return Status   Should Be Empty  ${pids}
+        IF    ${status}    BREAK
+        Sleep   1
+    END
+    Should Be Empty   ${pids}   ${app_name} is still running
+    Log To Console    ${app_name} not running
+
 Check If Ping Fails
     [Documentation]  Check that ping is not getting response from host
-    # ${out}   Run and Return RC   ping ${DEVICE_IP_ADDRESS} -c 1
     ${result}  Run Process  ping  ${DEVICE_IP_ADDRESS}  -c1  timeout=1s
-    Should Not Be Equal   ${result.rc}  ${0}
+    Should Not Be Equal   ${result.rc}  ${0}
+
+Run journalctl recording
+    ${output}     Execute Command    journalctl > jrnl.txt
+    ${output}     Execute Command    nohup journalctl -f >> jrnl.txt 2>&1 &
+
+Log journctl
+    ${output}     Execute Command    cat jrnl.txt
+    Log           ${output}
+    @{pid}        Find pid by name   journalctl
+    Kill process  @{pid}
+
+Check If Device Is Up
+    [Arguments]    ${range}=20
+    Set Global Variable    ${IS_AVAILABLE}       False
+    ${start_time}=    Get Time	epoch
+    FOR    ${i}    IN RANGE    ${range}
+        ${ping}=    Ping Host   ${DEVICE_IP_ADDRESS}
+        IF    ${ping}
+            Log To Console    Ping ${DEVICE_IP_ADDRESS} successfull
+            BREAK
+        END
+        Sleep  1
+    END
+
+    IF    ${ping}
+        ${port_22_is_available}     Check if ssh is ready on device
+        IF  ${port_22_is_available}
+            Set Global Variable    ${IS_AVAILABLE}       True
+        ELSE
+            Set Global Variable    ${IS_AVAILABLE}       False
+        END
+    END
+
+    ${diff}=    Evaluate    int(time.time()) - int(${start_time})
+
+    IF  ${IS_AVAILABLE}    Log To Console    Device woke up after ${diff} sec.
+
+    IF  ${IS_AVAILABLE} == False
+        Log To Console    Device is not available after reboot via SSH, waited for ${diff} sec!
+        IF  "${SERIAL_PORT}" == "NONE"
+            Log To Console    There is no address for serial connection
+        ELSE
+            Check Serial Connection
+        END
+    END

Robot-Framework/resources/gui_keywords.resource

@@ -0,0 +1,128 @@
+# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII)
+# SPDX-License-Identifier: Apache-2.0
+
+*** Settings ***
+Resource            ../config/variables.robot
+Library             ../lib/gui_testing.py
+Library             Collections
+
+*** Variables ***
+
+${start_menu}         ./launcher.png
+${LOGGED_IN_STATUS}   ${True}
+
+*** Keywords ***
+
+GUI Log in
+    [Documentation]   Login and verify that task bar is available
+    Verify logout
+    IF  ${LOGGED_IN_STATUS}
+        Log To Console          Already logged in. Skipping login.
+    ELSE
+        Log To Console          Logging in
+    END
+    Start ydotoold
+    Log To Console    Typing username and password to login
+    Type string and press enter  ${LOGIN}
+    Type string and press enter  ${PASSWORD}
+
+GUI Log out
+    [Documentation]   Log out and optionally verify that desktop is not available
+    [Arguments]        ${log_out_icon}=./logout.png
+    Start ydotoold
+    Get icon           ghaf-artwork  power.svg  crop=0  background=black
+    Locate and click   ./icon.png  0.95  5
+    Get icon           ghaf-artwork  logout.svg  crop=0  background=black
+    Locate and click   ./icon.png  0.95  5
+
+Type string and press enter
+    [Arguments]   ${string}
+    Log To Console    Typing
+    Execute Command   ydotool type ${string}  sudo=True  sudo_password=${PASSWORD}
+    Log To Console    Pressing Enter
+    Execute Command   ydotool key -d 0 28:1 28:0  sudo=True  sudo_password=${PASSWORD}
+
+Locate image on screen
+    [Documentation]    Take a screenshot. Locate given image on the screenshot.
+    ...                Return center coordinates of the image in mouse coordinate system
+    [Arguments]        ${image_to_be_searched}  ${confidence}=0.999   ${iterations}=5
+    ${coordinates}=  Set Variable  ${EMPTY}
+    ${pass_status}=  Set Variable  FAIL
+    FOR   ${i}   IN RANGE  ${iterations}
+        Log To Console    Taking screenshot
+        Execute Command   rm screenshot.png
+        ${rc} =    Execute Command  grim screenshot.png  return_stdout=False  return_rc=${true}
+        IF  "${rc}" == "0"
+            SSHLibrary.Get File  screenshot.png   screenshot.png
+            Log To Console    Locating image ${image_to_be_searched} on screenshot
+            ${pass_status}  ${coordinates}  Run Keyword And Ignore Error  Locate image  ${image_to_be_searched}   ${confidence}
+        END
+        IF    $pass_status=='PASS'    BREAK
+        Sleep  0.5
+    END
+    IF    $pass_status=='FAIL'    FAIL  Image recognition failure: ${image_to_be_searched}
+    Log To Console    Coordinates: ${coordinates}
+    ${mouse_x}  Get From Dictionary   ${coordinates}  x
+    ${mouse_y}  Get From Dictionary   ${coordinates}  y
+    RETURN  ${mouse_x}  ${mouse_y}
+
+Locate and click
+    [Arguments]   ${image_to_be_searched}  ${confidence}=0.99  ${iterations}=5
+    ${mouse_x}  ${mouse_y}  Locate image on screen  ${image_to_be_searched}   ${confidence}
+    Execute Command   ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y}  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   ydotool click 0xC0  sudo=True  sudo_password=${PASSWORD}
+
+Start ydotoold
+    [Documentation]    Start ydotool daemon if it is not already running.
+    ${ydotoold_state}=    Execute Command    sh -c 'ps aux | grep ydotoold | grep -v grep'
+    IF  $ydotoold_state == '${EMPTY}'
+        Log To Console    Starting ydotool daemon
+        Run Keyword And Ignore Error  Execute Command   -b /run/current-system/sw/bin/ydotoold --socket-path /tmp/.ydotool_socket  sudo=True  sudo_password=${PASSWORD}  timeout=3
+        ${ydotoold_state}=    Execute Command    sh -c 'ps aux | grep ydotoold | grep -v grep'
+        Should Not Be Empty  ${ydotoold_state}  failed to start ydotool daemon
+    ELSE
+        Log To Console    Check: ydotool daemon running
+    END
+
+Stop ydotoold
+    [Documentation]    Kill ydotool daemon
+    Log To Console    Stopping ydotool daemon
+    Execute Command   pkill ydotoold  sudo=True  sudo_password=${PASSWORD}
+
+Move cursor to corner
+    [Documentation]    Move the cursor to the upper left corner so that it will not block searching further gui screenshots
+    Log To Console    Moving cursor to corner from blocking further image detection
+    Start ydotoold
+    Execute Command   ydotool mousemove --absolute -x 50 -y 50  sudo=True  sudo_password=${PASSWORD}
+
+Verify login
+    [Documentation]    Check that launcher icon is available on desktop
+    Log To Console          Verifying login by trying to detect the launcher icon
+    Locate image on screen  ${start_menu}  0.95  15
+
+Verify logout
+    [Documentation]    Check that dekstop is not available by running 'grim' which should have return code 1 in this case
+    [Arguments]        ${iterations}=5
+    ${status}=  Set Variable  ${EMPTY}
+    FOR   ${i}   IN RANGE  ${iterations}
+        ${rc}=    Execute Command  grim check.png  return_stdout=False  return_rc=${true}
+        IF  "${rc}" == "1"
+            Set Global Variable  ${LOGGED_IN_STATUS}  ${False}
+            BREAK
+        ELSE
+            Set Global Variable  ${LOGGED_IN_STATUS}  ${True}
+        END
+        Sleep  1
+    END
+
+Get icon
+    [Documentation]    Copy icon svg file to test agent machine. Crop and convert the svg file to png.
+    [Arguments]        ${path}   ${icon_name}   ${crop}   ${background}=none   ${output_filename}=icon.png
+    IF  $path == "app"
+        SSHLibrary.Get File  ${APP_ICON_PATH}/${icon_name}   icon.svg
+    ELSE IF  $path == "ghaf-artwork"
+        SSHLibrary.Get File  ${ARTWORK_PATH}/${icon_name}   icon.svg
+    ELSE
+        SSHLibrary.Get File  ${path}/${icon_name}   icon.svg
+    END
+    Convert app icon  ${crop}   ${background}   input_file=icon.svg  output_file=${output_filename}

Robot-Framework/resources/ssh_keywords.resource

@@ -458,3 +458,23 @@ Check Storagevm Size
     Log  ${storage}
     ${size}  Get Regexp Matches  ${storage}  (?im)(storagevm\\s*)(\\d{1,3}G)(\\s*\\d{1,3}.\\s*)(\\d{1,3})  4
     RETURN  ${size}[0]
+
+Check ssh connection status
+    [Documentation]    Check if there is already an active ssh connection to vm_name
+    [Arguments]        ${vm_name}
+    ${status}   ${hostname}      Run Keyword And Ignore Error  Execute Command  hostname
+    IF  $vm_name in $hostname
+        RETURN  ${True}
+    ELSE
+        RETURN  ${False}
+    END
+
+Connect to VM if not already connected
+    [Documentation]    Connect only if there is not already an active connection
+    [Arguments]        ${vm_name}
+    ${connected}   Check ssh connection status   ${vm_name}
+    IF  not ${connected}
+        Connect
+        Connect to netvm
+        Connect to VM   ${vm_name}
+    END

Robot-Framework/test-suites/bat-tests/__init__.robot

@@ -5,6 +5,7 @@
 Documentation       BAT tests
 Resource            ../../resources/ssh_keywords.resource
 Resource            ../../resources/serial_keywords.resource
+Resource            ../../resources/common_keywords.resource
 Suite Setup         Common Setup
 Suite Teardown      Common Teardown
 
@@ -33,16 +34,6 @@ Common Teardown
     END
     Close All Connections
 
-Run journalctl recording
-    ${output}     Execute Command    journalctl > jrnl.txt
-    ${output}     Execute Command    nohup journalctl -f >> jrnl.txt 2>&1 &
-
-Log journctl
-    ${output}     Execute Command    cat jrnl.txt
-    Log           ${output}
-    @{pid}        Find pid by name   journalctl
-    Kill process  @{pid}
-
 Log versions
     ${ghaf_version}     Execute Command   ghaf-version
     Log to console      Ghaf version: ${ghaf_version}

Robot-Framework/test-suites/bat-tests/apps.robot

@@ -7,6 +7,9 @@ Force Tags          apps
 Resource            ../../resources/ssh_keywords.resource
 Resource            ../../config/variables.robot
 Resource            ../../resources/common_keywords.resource
+Library             ../../lib/gui_testing.py
+Library             Collections
+Library             BuiltIn
 Suite Teardown      Close All Connections
 
 

Robot-Framework/test-suites/boot-test/boot_test.robot

@@ -118,41 +118,3 @@ Teardown
     END
     Close All Connections
     Delete All Ports
-
-Check If Device Is Up
-    [Arguments]    ${range}=20
-    Set Global Variable    ${IS_AVAILABLE}       False
-    ${start_time}=    Get Time	epoch
-    FOR    ${i}    IN RANGE    ${range}
-        ${ping}=    Ping Host   ${DEVICE_IP_ADDRESS}
-        IF    ${ping}
-            Log To Console    Ping ${DEVICE_IP_ADDRESS} successfull
-            BREAK
-        END
-        Sleep  1
-    END
-
-    IF    ${ping}
-        ${port_22_is_available}     Check if ssh is ready on device
-        IF  ${port_22_is_available}
-            Set Global Variable    ${IS_AVAILABLE}       True
-        ELSE
-            Set Global Variable    ${IS_AVAILABLE}       False
-        END
-    END
-
-    ${diff}=    Evaluate    int(time.time()) - int(${start_time})
-
-    IF  ${IS_AVAILABLE}    Log To Console    Device woke up after ${diff} sec.
-
-    IF  ${IS_AVAILABLE} == False
-        Log To Console    Device is not available after reboot via SSH, waited for ${diff} sec!
-        IF  "${SERIAL_PORT}" == "NONE"
-            Log To Console    There is no address for serial connection
-        ELSE
-            Check Serial Connection
-        END
-    END
-
-
-