nl-calc

#!/usr/bin/env python3

# Copyright (c) 2018, Kontron Europe GmbH
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from __future__ import print_function

import argparse
import copy
import dateutil.parser
import json
import numpy
import sys

def update_histogram_timestamp(timestamp, histogram):
    tx_timestamp = numpy.datetime64(timestamp)

    if not histogram['start-timestamp']:
        histogram['start-timestamp'] = timestamp

    if not histogram['end-timestamp']:
        histogram['end-timestamp'] = timestamp

    if numpy.datetime64(timestamp) > \
            numpy.datetime64(histogram['end-timestamp']):
        histogram['end-timestamp'] = timestamp

def update_histogram_general(value, histogram):
    histogram['count'] += 1
    if value > histogram['max'] or histogram['max'] == 0:
        histogram['max'] = value
    if value < histogram['min'] or histogram['min'] == 0:
        histogram['min'] = value

def update_histogram_modulo(timestamp, value, histogram):

    update_histogram_general(value, histogram)

    if value < 0:
        histogram['time_error'] += 1
    elif value > len(histogram['histogram']):
        histogram['outliers'] += 1
    else:
        value  = value % 1000
        histogram['histogram'][value] += 1

    update_histogram_timestamp(timestamp, histogram)

def update_histogram(timestamp, value, histogram):

    update_histogram_general(value, histogram)

    if value < 0:
        histogram['time_error'] += 1
    elif value < len(histogram['histogram']):
        try:
            histogram['histogram'][value] += 1
        except IndexError as e:
            print ((e, value))
    else:
        histogram['outliers'] += 1

    update_histogram_timestamp(timestamp, histogram)

def update_histogram_jitter(timestamp, value, offset, histogram):

    update_histogram_general(value, histogram)

    value += offset
    if value < 0 or value >= len(histogram['histogram']):
        histogram['outliers'] += 1
    else:
        try:
            histogram['histogram'][value] += 1
        except IndexError as e:
            print ((e, value))

    update_histogram_timestamp(timestamp, histogram)


def calc_latency(pkt, ts):
    result = {}
    interval_start = numpy.datetime64(ts['interval-start'])
    # t0
    #tx_user_target = numpy.datetime64(ts['tx-wakeup'])
    # t1
    tx_user = numpy.datetime64(ts['tx-program'])
    # t4
    rx_hw = numpy.datetime64(ts['rx-hardware'])

    # rt-application latency: (t1 - t0) % interval

    diff_rt_app = tx_user - interval_start
    diff_interval_start_hw_rx = rx_hw - interval_start

    result['type'] = 'latency-calc'
    result['object'] = {
        'latency-program': int(diff_rt_app)/1000 % pkt['interval-usec'],
        #'latency-scheduled-times': int(diff_interval_start_hw_rx)/1000 % pkt['interval-usec'],
        'latency-scheduled-times': int(diff_interval_start_hw_rx)/1000,
        'sequence-number': pkt['sequence-number'],
        'tx-program': ts['tx-program'],
        'stream-id': pkt['stream-id'],
    }
    return result


mean_latency = 0
count_pkt = 0
jitter_min = 0
jitter_max = 0

def calc_jitter(pkt, ts):
    global mean_latency
    global count_pkt
    global jitter_min
    global jitter_max

    result = {}
    interval_start = numpy.datetime64(ts['interval-start'])
    rx_hw = numpy.datetime64(ts['rx-hardware'])

    diff_interval_start_hw_rx = rx_hw - interval_start

    val = int(diff_interval_start_hw_rx) % (pkt['interval-usec'] * 1000)
    mean_latency = (count_pkt * mean_latency + val) / (count_pkt + 1)
    count_pkt += 1
    #print(mean_latency)

    jitter = mean_latency - val
    if jitter_min > jitter:
        jitter_min = jitter
    if jitter_max < jitter:
        jitter_max = jitter
    return jitter


def dump_json_str(val):
    print(json.dumps(val), file=sys.stdout)
    sys.stdout.flush()


def main(args=None):
    parser = argparse.ArgumentParser(
        description='latency')
    parser.add_argument('-c', '--count', type=int, dest='count',
                        help='Count until histogram output', default=0)
    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'),
                        help='Input file (default is STDIN)', default=sys.stdin)
    args = parser.parse_args(args)

    output = None
    histogram_program_latency_empty = {
        'type': 'histogram-program-latency',
        'object': {
            'stream-id': 0,
            'count': 0,
            'min': 0,
            'max': 0,
            'outliers': 0,
            'time_error': 0,
            'histogram': [0] * 50,
            'start-timestamp': None,
            'end-timestamp': None,
        }
    }

    histogram_scheduled_times_empty = {
        'type': 'histogram-scheduled-times',
        'object': {
            'stream-id': 0,
            'count': 0,
            'min': 0,
            'max': 0,
            'outliers': 0,
            'time_error': 0,
            'histogram': [0] * 1000,
            'start-timestamp': None,
            'end-timestamp': None,
        }
    }

    histogram_jitter_empty= {
        'type': 'histogram-jitter',
        'object': {
            'stream-id': 0,
            'count': 0,
            'min': 0,
            'max': 0,
            'outliers': 0,
            'time_error': 0,
            'offset': 1000,
            'histogram': [0] * 2000,
            'start-timestamp': None,
            'end-timestamp': None,
        }
    }

    hist_program_latency = copy.deepcopy(histogram_program_latency_empty)
    hist_scheduled_times = copy.deepcopy(histogram_scheduled_times_empty)
    hist_jitter = copy.deepcopy(histogram_jitter_empty)

    count = 0
    try:
        for line in args.infile:
            line = line.strip()
            if not line:
                continue

            try:
                j = json.loads(line)

                if j['type'] == 'rx-error':
                    print(line, file=sys.stdout)
                elif j['type'] == 'rx-packet':
                    count += 1
                    ts = j['object']['timestamps']
                    ts = dict(zip(ts['names'], ts['values']))
                    result = calc_latency(j['object'], ts)

                    timestamp = result['object']['tx-program']

                    value = result['object']['latency-program']
                    update_histogram(timestamp, int(value),
                            hist_program_latency['object'])

                    value = result['object']['latency-scheduled-times']
                    update_histogram_modulo(timestamp, int(value),
                            hist_scheduled_times['object'])

                    jitter = calc_jitter(j['object'], ts)
                    update_histogram_jitter(timestamp, int(jitter), hist_jitter['object']['offset'],
                            hist_jitter['object'])

                if args.count != 0:
                    if count == args.count:
                        dump_json_str(hist_program_latency)
                        dump_json_str(hist_scheduled_times)
                        dump_json_str(hist_jitter)
                        count = 0
                        hist_program_latency= copy.deepcopy(
                                histogram_program_latency_empty)
                        hist_scheduled_times = copy.deepcopy(
                                histogram_scheduled_times_empty)
                        hist_jitter = copy.deepcopy(
                                histogram_jitter_empty)

                sys.stdout.flush()

            except ValueError as e:
                print(e, file=sys.stderr)
                pass
    except KeyboardInterrupt as e:
        pass

    dump_json_str(hist_program_latency)
    dump_json_str(hist_scheduled_times)
    dump_json_str(hist_jitter)


if __name__ == '__main__':
    main()