Skip to content
This repository has been archived by the owner on Jun 14, 2023. It is now read-only.

ruby perfmon collector does not send metrics #9

Open
root360-StefanHeitmueller opened this issue Nov 27, 2020 · 5 comments
Open

ruby perfmon collector does not send metrics #9

root360-StefanHeitmueller opened this issue Nov 27, 2020 · 5 comments

Comments

@root360-StefanHeitmueller
Copy link

Operating System

Ubuntu Bionic Beaver 18.04

Installation

sh install_cht_perfmon.sh 20 <key> aws disable-update

Result

perfmon collector

I've tapped the instance with mitmproxy to see why metrics are not being sent to https://chapi.cloudhealthtech.com/metrics/v1 (according to the docs)

There are no requests to the endpoint:

Screenshot from 2020-11-27 10-18-39
@root360-StefanHeitmueller
Copy link
Author

Agent status in dashboard reports as late upload

Screenshot from 2020-11-27 11-05-20

@root360-StefanHeitmueller
Copy link
Author

https://support.cloudhealthtech.com/hc/en-us/articles/360015713131-The-CloudHealth-Agent-has-a-status-of-Late-upload-What-does-this-mean- mentions to check /opt/cht_perfmon/cht_perfmon_collector.output which does not exist on the system.

@root360-StefanHeitmueller
Copy link
Author

ls -l /var/lib/chtcollectd/* does show active metrics collected by collectd:

root@i-xxx:~# ls -l /var/lib/chtcollectd/*
total 80
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 cpu-average
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-dev
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-dev-shm
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-root
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run-lock
drwxrwsr-x 2 root cht_agent 4096 Nov 25 06:25 df-run-user-2002
drwxrwsr-x 2 root cht_agent 4096 Nov 25 06:25 df-run-user-2003
drwxrwsr-x 2 root cht_agent 4096 Nov 27 07:56 df-run-user-2007
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run-user-2013
drwxrwsr-x 2 root cht_agent 4096 Nov 25 06:25 df-run-user-2203
drwxr-sr-x 2 root cht_agent 4096 Nov 27 07:56 df-run-user-2207
drwxr-sr-x 2 root cht_agent 4096 Nov 26 06:25 df-run-user-2213
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run-user-3001
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-sys-fs-cgroup
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-var-log-remote
drwxr-sr-x 2 root cht_agent 4096 Nov 27 10:00 interface-ens5
drwxr-sr-x 2 root cht_agent 4096 Nov 27 10:00 interface-lo
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 memory

@root360-StefanHeitmueller
Copy link
Author

error

@root360-StefanHeitmueller
Copy link
Author

Ah, okay, got another instance which is sending data, but to another path at https://api.cloudhealthtech.com/v1/agent/upload with a JSON structure like

{
  "instance": "i-xxx",
  "namespace": "CloudHealth/Perfmon",
  "cloud_name": "aws",
  "start_time": "2020-11-27T10:00:00Z",
  "stop_time": "2020-11-27T11:00:00Z",
  "facter": null,
  "perf_data": [
    {
      "monitor_name": "df",
      "value": {
        "/etc/hosts": {
          "sample_specific_stats": {
            "fs_src": "/dev/nvme0n1p1",
            "fs_type": "ext4",
            "fs_options": "rw,relatime,discard"
          },
          "used": {
            "min": 4373340160,
            "max": 4374106112,
            "min_perc": 21.076533531943472,
            "max_perc": 21.080224901107826,
            "avg": 4373716718.933333,
            "avg_perc": 21.07834829061642
          },
          "free": {
            "min": 16375701504,
            "max": 16376467456,
            "min_perc": 78.91977509889217,
            "max_perc": 78.92346646805653,
            "avg": 16376090897.066675,
            "avg_perc": 78.92165170938362
          }
        },
        "/": {
          "sample_specific_stats": {
            "fs_src": "overlay",
            "fs_type": "overlay",
            "fs_options": "rw,relatime,lowerdir=/var/lib/docker/overlay2/l/BGPWVDQKFNCX6MCF52TUKXXI4C:/var/lib/docker/overlay2/l/37CWUGOOKJMVFCT4LZWOG3UX6R:/var/lib/docker/overlay2/l/4KAVPSAZWW3MUFLXSVWFT2O5AM:/var/lib/docker/overlay2/l/LAJUGTRHGVXWM4YNI7I465F3TK:/var/lib/docker/overlay2/l/V663TUYTSHF7E4FOZQIGIYU5MY:/var/lib/docker/overlay2/l/PAZYZAK4XNQJ3GY7525HSUZZOF:/var/lib/docker/overlay2/l/DNPA4DSYTZJAXO6ABVGSSXBVQA:/var/lib/docker/overlay2/l/AHXBYO6TZTCGU6J2OIZ23EYJA6,upperdir=/var/lib/docker/overlay2/ca900efa9b02b89c277afe0fa6f87b427b2dc1d8132c83c4be490ea789c15ce1/diff,workdir=/var/lib/docker/overlay2/ca900efa9b02b89c277afe0fa6f87b427b2dc1d8132c83c4be490ea789c15ce1/work,xino=off"
          },
          "used": {
            "min": 4373340160,
            "max": 4374106112,
            "min_perc": 21.076533531943472,
            "max_perc": 21.080224901107826,
            "avg": 4373716377.599999,
            "avg_perc": 21.07834664562125
          },
          "free": {
            "min": 16375701504,
            "max": 16376467456,
            "min_perc": 78.91977509889217,
            "max_perc": 78.92346646805653,
            "avg": 16376091238.40001,
            "avg_perc": 78.92165335437879
          }
        },
        "/etc/hostname": {
          "sample_specific_stats": {
            "fs_src": "/dev/nvme0n1p1",
            "fs_type": "ext4",
            "fs_options": "rw,relatime,discard"
          },
          "used": {
            "min": 4373340160,
            "max": 4374106112,
            "min_perc": 21.076533531943472,
            "max_perc": 21.080224901107826,
            "avg": 4373716718.933333,
            "avg_perc": 21.07834829061642
          },
          "free": {
            "min": 16375701504,
            "max": 16376467456,
            "min_perc": 78.91977509889217,
            "max_perc": 78.92346646805653,
            "avg": 16376090897.066675,
            "avg_perc": 78.92165170938362
          }
        },
        "/etc/resolv.conf": {
          "sample_specific_stats": {
            "fs_src": "/dev/nvme0n1p1",
            "fs_type": "ext4",
            "fs_options": "rw,relatime,discard"
          },
          "used": {
            "min": 4373340160,
            "max": 4374106112,
            "min_perc": 21.076533531943472,
            "max_perc": 21.080224901107826,
            "avg": 4373716718.933333,
            "avg_perc": 21.07834829061642
          },
          "free": {
            "min": 16375701504,
            "max": 16376467456,
            "min_perc": 78.91977509889217,
            "max_perc": 78.92346646805653,
            "avg": 16376090897.066675,
            "avg_perc": 78.92165170938362
          }
        }
      }
    },
    {
      "monitor_name": "cpu",
      "value": {
        "average": {
          "sample_specific_stats": {},
          "unused": {
            "min": 96.165178,
            "max": 98.01659099999999,
            "min_perc": 96.56903840984343,
            "max_perc": 98.37711159237382,
            "avg": 97.7576948392857,
            "avg_perc": 98.10920257351773
          },
          "used": {
            "min": 1.616666,
            "max": 3.416613,
            "min_perc": 1.6228884076261711,
            "max_perc": 3.4309615901565778,
            "avg": 1.8839269642857142,
            "avg_perc": 1.890797426482287
          }
        }
      }
    },
    {
      "monitor_name": "memory",
      "value": {
        "memory": {
          "sample_specific_stats": {},
          "used": {
            "min": 1085796352,
            "max": 1212633088,
            "min_perc": 53.105129674221814,
            "max_perc": 59.30857777047683,
            "avg": 1134323097.6,
            "avg_perc": 55.47852051589226
          },
          "free": {
            "min": 831983616,
            "max": 958820352,
            "min_perc": 40.69142222952317,
            "max_perc": 46.894870325778186,
            "avg": 910293606.4000001,
            "avg_perc": 44.52147948410776
          }
        }
      }
    },
    {
      "monitor_name": "interface",
      "value": {
        "eth0": {
          "sample_specific_stats": {},
          "packets_rx": {
            "min": 0,
            "max": 2.799998,
            "min_perc": 0,
            "max_perc": 1.4084501091055877,
            "avg": 0.0736119888888889,
            "avg_perc": 0.017528217636934594,
            "sum": 265.00316
          },
          "packets_tx": {
            "min": 0,
            "max": 3.199998,
            "min_perc": 0,
            "max_perc": 0.5591795690015908,
            "avg": 0.08222320277777777,
            "avg_perc": 0.015628157848361808,
            "sum": 296.00353
          },
          "octets_rx": {
            "min": 0,
            "max": 757.599152,
            "min_perc": 0,
            "max_perc": 98.59154989089441,
            "avg": 20.120893205555554,
            "avg_perc": 3.5463880319384504,
            "sum": 72435.21554
          },
          "octets_tx": {
            "min": 0,
            "max": 396.499556,
            "min_perc": 0,
            "max_perc": 38.676608014737276,
            "avg": 10.272531488888887,
            "avg_perc": 1.6982333703540307,
            "sum": 36981.11336
          }
        }
      }
    },
    {
      "monitor_name": "disk",
      "value": {}
    }
  ],
  "command_data": []
}

Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@root360-StefanHeitmueller