From 38a8edfed785c728fbf86f440d9ec2b39fa5ca27 Mon Sep 17 00:00:00 2001 From: Finnegan1 Date: Fri, 9 Aug 2024 18:09:36 +0200 Subject: [PATCH 1/2] improve dora implementation --- app.py | 2 +- components.py | 12 +- dora.py | 1338 +++++++++++++++--------------------------- dora_result_calcs.py | 140 +++++ 4 files changed, 637 insertions(+), 855 deletions(-) create mode 100644 dora_result_calcs.py diff --git a/app.py b/app.py index ebafaf68..b41d7a3a 100755 --- a/app.py +++ b/app.py @@ -438,7 +438,7 @@ async def initialise_app(): description='API definition', title='Delivery-Service by Gardener CICD', definitions=api_definitions, - ) + ) return app diff --git a/components.py b/components.py index 6f8d3ced..f1594e2d 100644 --- a/components.py +++ b/components.py @@ -76,6 +76,16 @@ async def check_if_component_exists( return False +def ensure_utc(ts: datetime.datetime) -> datetime.datetime: + if ts.tzinfo is None: + ts = ts.replace(tzinfo=datetime.timezone.utc) + else: + if ts.tzinfo != datetime.timezone.utc: + ts = ts.astimezone(datetime.timezone.utc) + + return ts + + def get_creation_date(component: ocm.Component) -> datetime.datetime: ''' Trys to extract creation date from creationTime attribute and if not set from label with name @@ -84,7 +94,7 @@ def get_creation_date(component: ocm.Component) -> datetime.datetime: ''' if (creationTime := component.creationTime): - return dateutil.parser.isoparse(creationTime) + return ensure_utc(dateutil.parser.isoparse(creationTime)) creation_label: ocm.Label | None = component.find_label('cloud.gardener/ocm/creation-date') diff --git a/dora.py b/dora.py index 140c851b..49e81f6b 100644 --- a/dora.py +++ b/dora.py @@ -1,185 +1,38 @@ +import asyncio import collections import collections.abc -import concurrent.futures import dataclasses import datetime -import enum import functools -import http -import logging import statistics import typing import urllib.parse import aiohttp.web import cachetools.keys +import cnudie.iter_async import dateutil.parser +import dora_result_calcs +import ocm import github3 import ci.util -import cnudie.iter -import cnudie.iter_async -import cnudie.retrieve_async +import cnudie.retrieve import cnudie.util -import ocm +import semver import version as versionutil import caching import components import consts +import middleware +import middleware.auth import util -logger = logging.getLogger(__name__) -changes_by_dependencies_cache = dict() - - -@dataclasses.dataclass(frozen=True) -class CodeChange: - ''' - Represents a code change with its commit data and deployment date - ''' - commit_sha: str - commit_date: datetime.datetime - deployment_date: datetime.datetime - - -@dataclasses.dataclass(frozen=True) -class ComponentDependencyChangeWithCommits: - ''' - Holds a Dependency Change for a specific Component as well as the commits included within the - Dependency Change - ''' - component: ocm.Component - dependency_component_vector: components.ComponentVector - commits: list[github3.github.repo.commit.ShortCommit] - - -@dataclasses.dataclass(frozen=True) -class ComponentWithDependencyChanges: - ''' - Holds a component descriptor as well as a list of dependency updates, which - where introduced in this component Version - ''' - component_descriptor: ocm.ComponentDescriptor - dependency_changes: list[components.ComponentVector] - - -class CalculationType(enum.StrEnum): - MEDIAN = 'median' - AVERAGE = 'average' - - -class DeploymentFrequencyBuckets(enum.StrEnum): - ''' - Typical Buckets to which a deplyoment Frequency can be assigned - ''' - daily = 'daily' - weekly = 'weekly' - monthly = 'monthly' - yearly = 'yearly' - - -@dataclasses.dataclass(frozen=True) -class DoraDeploymentsResponse: - ''' - Helper datacalss for creating JSON response for the DoraMetrics Route - ''' - target_deployment_version: str - component_version: str - deployment_date: datetime.datetime - median_change_lead_time: float - changes: list[CodeChange] - - -@dataclasses.dataclass(frozen=True) -class DoraMonthlyResponse: - ''' - Helper datacalss for creating JSON response for the DoraMetrics Route - ''' - year: int - month: int - median_change_lead_time: float - changes: list[CodeChange] - - -@dataclasses.dataclass(frozen=True) -class DoraDependencyResponse: - ''' - Helper datacalss for creating JSON response for the DoraMetrics Route - ''' - change_lead_time_median: float - change_lead_time_average: float - deployment_frequency: float - changes_monthly: list[DoraMonthlyResponse] - deployments: list[DoraDeploymentsResponse] - all_changes: list[CodeChange] - repo_url: str - - -@dataclasses.dataclass(frozen=True) -class DoraResponse: - ''' - Helper datacalss for creating JSON response for the DoraMetrics Route - ''' - change_lead_time_median: float - change_lead_time_average: float - dependencies: dict[str, DoraDependencyResponse] - - -async def versions_descriptors_newer_than( - component_name: str, - date: datetime.datetime, - component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, - version_lookup: cnudie.retrieve_async.VersionLookupByComponent, - only_releases: bool = True, - invalid_semver_ok: bool = False, - sorting_direction: typing.Literal['asc', 'desc'] = 'desc' -): - ''' - This function retrieves the component descriptors for the versions - of a specific Component, which are newer then the given date. - - asc-sorting means old to new => [0.102.0 ... 0.321.2] - - desc-sorting means new to old => [0.321.2 ... 0.102.0] - ''' - - def _filter_component_newer_than_date( - descriptor: ocm.ComponentDescriptor, - date: datetime.datetime, - ) -> bool: - creation_date: datetime.datetime = components.get_creation_date(descriptor.component) - return creation_date > date - - versions = await all_versions_sorted( - component=component_name, - sorting_direction='desc', - invalid_semver_ok=invalid_semver_ok, - only_releases=only_releases, - version_lookup=version_lookup, - ) - - descriptors: list[ocm.ComponentDescriptor] = [] - - for version in versions: - descriptor = await component_descriptor_lookup((component_name, version)) - try: - if not _filter_component_newer_than_date(descriptor, date): - break - except KeyError: - continue - descriptors.append(descriptor) - - if sorting_direction == 'asc': - descriptors.reverse() - - return descriptors - - def _cache_key_gen_all_versions_sorted( - component: cnudie.util.ComponentName, - version_lookup: cnudie.retrieve_async.VersionLookupByComponent, + component: cnudie.retrieve.ComponentName, + version_lookup: cnudie.retrieve.VersionLookupByComponent, only_releases: bool = True, invalid_semver_ok: bool = False, sorting_direction: typing.Literal['asc', 'desc'] = 'desc', @@ -197,18 +50,21 @@ def _cache_key_gen_all_versions_sorted( key_func=_cache_key_gen_all_versions_sorted, ) async def all_versions_sorted( - component: cnudie.util.ComponentName, - version_lookup: cnudie.retrieve_async.VersionLookupByComponent, + component: cnudie.retrieve.ComponentName, + version_lookup: cnudie.retrieve.VersionLookupByComponent, only_releases: bool = True, invalid_semver_ok: bool = False, sorting_direction: typing.Literal['asc', 'desc'] = 'desc' ) -> list[str]: ''' - This is a convenience function for looking up all versions of a specific - component. + Retrieve all versions of a specific component, sorted according to specified parameters. + + Returns: + A list of version strings sorted according to the specified parameters. - asc-sorting means old to new => [0.102.0 ... 0.321.2] - desc-sorting means new to old => [0.321.2 ... 0.102.0] + Notes: + - 'asc' sorting means old to new => [0.102.0, ..., 0.321.2] + - 'desc' sorting means new to old => [0.321.2, ..., 0.102.0] ''' component_name = cnudie.util.to_component_name(component) @@ -242,35 +98,135 @@ def filter_version(version: str, invalid_semver_ok: bool, only_releases:bool): return versions -async def get_next_older_descriptor( - component_id: ocm.ComponentIdentity, - component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, - component_version_lookup: cnudie.retrieve_async.VersionLookupByComponent, -) -> ocm.ComponentDescriptor | None: - all_versions = await all_versions_sorted( - component=component_id, - version_lookup=component_version_lookup, - sorting_direction='desc', +async def filter_versions_newer_than( + component: cnudie.retrieve.ComponentName, + all_versions: list[semver.VersionInfo], + date: datetime.datetime, + component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, +) -> list[semver.VersionInfo]: + ''' + Filter list of versions of a component for versions that are newer than a specified date. + + Returns: + A list of version information objects representing versions newer than the specified date. + ''' + all_versions = sorted( + all_versions, + key=lambda v: versionutil.parse_to_semver(v), + reverse=True, ) - if (version_index := all_versions.index(component_id.version)) != len(all_versions) - 1: - old_target_version = all_versions[version_index + 1] - else: - return None + component_versions: list[semver.VersionInfo] = [] - return await component_descriptor_lookup( - ocm.ComponentIdentity( - name=component_id.name, - version=old_target_version, - ), + for version in all_versions: + descriptor: ocm.ComponentDescriptor = await component_descriptor_lookup( + ocm.ComponentIdentity( + name=cnudie.util.to_component_name(component), + version=version, + ) + ) + creation_date = components.get_creation_date(descriptor.component) + + date = date.astimezone(datetime.timezone.utc) + creation_date = creation_date.astimezone(datetime.timezone.utc) + + if creation_date > date: + component_versions.append(version) + else: + break + + return component_versions + + +def _cache_key_gen_latest_componentversions_in_tree( + component: ocm.Component, + component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, +): + return cachetools.keys.hashkey( + cnudie.util.to_component_name(component), + component.identity() ) -def next_older_month(date: datetime.datetime) -> datetime.datetime: - month = 12 if date.month == 1 else date.month - 1 - year = date.year - 1 if date.month == 1 else date.year - older_month_date = datetime.datetime(year, month, 1, tzinfo=datetime.UTC) - return older_month_date +@caching.async_cached( + cache=caching.TTLFilesystemCache(ttl=60*60*24, max_total_size_mib=128), #1 day TODO + key_func=_cache_key_gen_latest_componentversions_in_tree, +) +async def latest_referenced_component_versions( + component: ocm.Component, + component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, +) -> dict[str, ocm.Component]: + ''' + Retrieve the latest versions of all components referenced in the component tree. + + Args: + component: The root component from which to traverse and collect referenced components. + component_descriptor_lookup: A function to lookup component descriptors by identity. + + Returns: + A dictionary mapping component names to their highest versioned Component object. + + Notes: + - Only the highest version of each component is retained. + + ''' + def default_factory(): + return None + components_by_name = collections.defaultdict(default_factory) + + referenced_components = [ + c.component async for c in cnudie.iter_async.iter( + component=component, + lookup=component_descriptor_lookup, + node_filter=cnudie.iter.Filter.components, + ) + ] + + def version_key(c): + if c is not None: + return versionutil.parse_to_semver(c.version) + else: + return semver.VersionInfo(0, 0, 0) + + for referenced_component in referenced_components: + components_by_name[referenced_component.name] = max( + components_by_name[referenced_component.name], + referenced_component, + key=version_key, + ) + + return dict(components_by_name) + + +@dataclasses.dataclass(frozen=True) +class ComponentVersionUpdate: + ''' + Data class representing version updates of a target component and its referenced component. + + Attributes: + target_component: The target component name. + target_component_version_old: The previous version of the target component. + target_component_version_new: The new version of the target component. + referenced_component: The name of the referenced component. + referenced_component_version_older_release: The version of the referenced component in the older targer component. + referenced_component_version_newer_release: The version of the referenced component in the newer target component. + ''' + target_component: cnudie.retrieve.ComponentName + target_component_version_old: str + target_component_version_new: str + referenced_component: cnudie.retrieve.ComponentName + referenced_component_version_older_release: str + referenced_component_version_newer_release: str + + def to_dict(self) -> dict[str, typing.Any]: + return { + 'target_component': cnudie.util.to_component_name(self.target_component), + 'target_component_versions_old': str(self.target_component_version_old), + 'target_component_versions_new': str(self.target_component_version_new), + 'referenced_component': cnudie.util.to_component_name(self.referenced_component), + 'referenced_component_version_older': str(self.referenced_component_version_older_release), + 'referenced_component_version_newer': str(self.referenced_component_version_newer_release), + } def can_process(dependency_update: components.ComponentVector): @@ -292,52 +248,53 @@ def can_process(dependency_update: components.ComponentVector): return True -def _cache_key_gen_component_vector_and_lookup( - left_commit: str, - right_commit: str, - github_repo, +def _cache_key_gen_commits_between( + older_commit: str, + newer_commit: str, + github_repo: github3.repos.Repository, ): return cachetools.keys.hashkey( - left_commit, - right_commit, + older_commit, + newer_commit, ) @caching.cached( cache=caching.LFUFilesystemCache(max_total_size_mib=256), - key_func=_cache_key_gen_component_vector_and_lookup, + key_func=_cache_key_gen_commits_between, ) -def commits_for_component_change( - left_commit: str, - right_commit: str, +def commits_between( + older_commit: str, + newer_commit: str, github_repo: github3.repos.Repository, ) -> tuple[github3.github.repo.commit.ShortCommit]: ''' - returns commits between passed-on commits. results are read from github-api and cached. - passed-on commits must exist in repository referenced by passed-in github_repo. + Retrieve commits between two specified commits from a GitHub repository. ''' - return tuple(github_repo.compare_commits( - left_commit, - right_commit, - ).commits()) + commits: tuple[github3.github.repo.commit.ShortCommit] = tuple( + github_repo.compare_commits( + older_commit, + newer_commit, + ).commits() + ) + return commits -def _cache_key_changes_by_dependencies( - target_descriptors_with_updates: tuple[ComponentWithDependencyChanges], -): - return cachetools.keys.hashkey(''.join([( - f'{target_descriptor_with_updates.component_descriptor.component.name}' - f'{target_descriptor_with_updates.component_descriptor.component.version}' - ) for target_descriptor_with_updates in target_descriptors_with_updates])) - -def categorize_by_changed_component( - target_descriptors_with_updates: tuple[ComponentWithDependencyChanges], +async def create_deployment_objects( + component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, + component_version_lookup: cnudie.retrieve.VersionLookupByComponent, github_api_lookup, -) -> dict[str, list[ComponentDependencyChangeWithCommits]]: - dependencies: dict[str, list[ComponentDependencyChangeWithCommits]] = ( - collections.defaultdict(list[ComponentDependencyChangeWithCommits]) - ) + target_component_name: str, + time_span_days: int, + filter_component_name: str, + target_updates: list[ComponentVersionUpdate], +) -> list[dora_result_calcs.ReturnDeploymentObject]: + ''' + Create objects containing the information about the target component updates in which the referenced component was updated. + ''' + + deployment_objects: list[dora_result_calcs.ReturnDeploymentObject] = [] _github_api = functools.cache(github_api_lookup) @@ -345,419 +302,118 @@ def categorize_by_changed_component( def _github_repo(repo_url: urllib.parse.ParseResult): github = _github_api(repo_url) org, repo = repo_url.path.strip('/').split('/') - return github.repository(org, repo) - def resolve_changes( - target_descriptor_with_updates: ComponentWithDependencyChanges, + async def create_deployment_object_for_update( + target_update: ComponentVersionUpdate, ): - for dependency_update in target_descriptor_with_updates.dependency_changes: - target_component = target_descriptor_with_updates.component_descriptor.component - dependency_component_name = dependency_update.end.name - - left_component = dependency_update.start - right_component = dependency_update.end - - left_src = cnudie.util.main_source( - left_component, - absent_ok=True, - ) - right_src = cnudie.util.main_source( - right_component, - absent_ok=True, - ) - - if not left_src or not right_src: - continue - - left_access = left_src.access - right_access = right_src.access - - if not left_access.type is ocm.AccessType.GITHUB: - continue - if not right_access.type is ocm.AccessType.GITHUB: - continue - - left_repo_url = ci.util.urlparse(left_access.repoUrl) - right_repo_url = ci.util.urlparse(right_access.repoUrl) - - if not left_repo_url == right_repo_url: - continue # ensure there was no repository-change between component-versions - - left_commit = left_access.commit or left_access.ref - right_commit = right_access.commit or right_access.ref - - github_repo = _github_repo( - repo_url=left_repo_url, # already checked for equality; choose either + old_ref_component: ocm.Component = (await component_descriptor_lookup( + ocm.ComponentIdentity( + name=cnudie.util.to_component_name(target_update.referenced_component), + version=target_update.referenced_component_version_older_release, ) + )).component - dependencies[dependency_component_name].append( - ComponentDependencyChangeWithCommits( - component=target_component, - dependency_component_vector=dependency_update, - commits=commits_for_component_change( - left_commit=left_commit, - right_commit=right_commit, - github_repo=github_repo, - ), - ) - ) - - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as tpe: - futures = { - tpe.submit(resolve_changes, target_descriptor_with_updates) - for target_descriptor_with_updates in target_descriptors_with_updates - } - concurrent.futures.wait(futures) - - key = _cache_key_changes_by_dependencies(target_descriptors_with_updates) - changes_by_dependencies_cache[key] = dependencies - - return dependencies - - -def _cache_key_gen_dora( - component_dependency_changes_with_commits: list[ - ComponentDependencyChangeWithCommits - ], - time_span_days: int | None = None, - calculation_type: CalculationType | None = None -): - component_versions = tuple( - component_dependency_change_with_commits.component.version - for component_dependency_change_with_commits in component_dependency_changes_with_commits - ) - hashkey_elements = ( - component_dependency_changes_with_commits[0].component.name, - component_dependency_changes_with_commits[0].dependency_component_vector.start.name, - component_dependency_changes_with_commits[0].dependency_component_vector.end.name, - component_dependency_changes_with_commits[0].dependency_component_vector.start.version, - component_dependency_changes_with_commits[0].dependency_component_vector.end.version, - component_versions, - ) - if time_span_days: hashkey_elements += (time_span_days, datetime.date.today()) - if calculation_type: hashkey_elements += (calculation_type,) - - return cachetools.keys.hashkey(*hashkey_elements) - - -@caching.cached( - cache=caching.LFUFilesystemCache(max_total_size_mib=128), - key_func=_cache_key_gen_dora, -) -def calculate_change_lead_time( - component_dependency_changes_with_commits: list[ - ComponentDependencyChangeWithCommits - ], - time_span_days: int, - calculation_type: CalculationType, -) -> datetime.timedelta: - time_differences: list[datetime.timedelta] = [] - - for component_dependency_change_with_commits in component_dependency_changes_with_commits: - deployment_date = components.get_creation_date( - component_dependency_change_with_commits.component - ) - for commit in component_dependency_change_with_commits.commits: - if ( - ( - commit_date := dateutil.parser.isoparse(commit.commit.author['date']) - ) > ( - datetime.datetime.now(datetime.timezone.utc) - - datetime.timedelta(days=time_span_days) + new_ref_component: ocm.Component = (await component_descriptor_lookup( + ocm.ComponentIdentity( + name=cnudie.util.to_component_name(target_update.referenced_component), + version=target_update.referenced_component_version_newer_release, ) - ): - time_differences.append(deployment_date - commit_date) - - if not time_differences: - time_differences.append(datetime.timedelta(seconds=-1)) - if calculation_type is CalculationType.MEDIAN: - result_in_seconds: float = statistics.median( - [time_difference.total_seconds() - for time_difference in time_differences] - ) - else: - result_in_seconds: float = statistics.mean( - [time_difference.total_seconds() - for time_difference in time_differences] - ) - return datetime.timedelta(seconds=result_in_seconds) - - -@caching.cached( - cache=caching.LFUFilesystemCache(max_total_size_mib=128), - key_func=_cache_key_gen_dora, -) -def dora_changes_monthly( - component_dependency_changes_with_commits: list[ - ComponentDependencyChangeWithCommits - ], - time_span_days: int, -) -> list[DoraMonthlyResponse]: + )).component - code_changes_by_month: dict[ - tuple[int, int], - list[tuple[datetime.datetime, CodeChange]], - ] = ( - collections.defaultdict(list[tuple[datetime.datetime, CodeChange]]) - ) + if not old_ref_component or not new_ref_component: + print ("ERROR, bad input data") - for component_dependency_change_with_commits in component_dependency_changes_with_commits: - for commit in component_dependency_change_with_commits.commits: - if ( - ( - commit_date := dateutil.parser.isoparse(commit.commit.author['date']) - ) > ( - datetime.datetime.now(datetime.timezone.utc) - - datetime.timedelta(days=time_span_days) + if not can_process( + components.ComponentVector( + start=old_ref_component, + end=new_ref_component, ) - ): - commit_sha: str = commit.sha - key = (commit_date.year, commit_date.month) - code_changes_by_month[key].append( - ( - components.get_creation_date( - component_dependency_change_with_commits.component - ), - CodeChange( - commit_date=commit_date, - commit_sha=commit_sha, - deployment_date=components.get_creation_date( - component_dependency_change_with_commits.component - ), - ), - ), - ) - - by_month_list: list[DoraMonthlyResponse] = [] - - for (year, month), code_changes in code_changes_by_month.items(): - - median_change_lead_time = datetime.timedelta(seconds=statistics.median( - [ - (deploy_date - commits_and_date.commit_date).total_seconds() - for deploy_date, commits_and_date in code_changes - ] - )) + ): + print("can't process") + return - by_month_list.append(DoraMonthlyResponse( - changes=[commits_and_date for _, commits_and_date in code_changes], - month=month, - year=year, - median_change_lead_time=median_change_lead_time.days, - )) + old_access = cnudie.util.main_source(old_ref_component).access + new_access = cnudie.util.main_source(new_ref_component).access - # create "empty" months which lie within the time_span_days - entry_date = ( - datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=time_span_days) - ) + old_repo_url = ci.util.urlparse(old_access.repoUrl) + new_repo_url = ci.util.urlparse(new_access.repoUrl) - while entry_date < datetime.datetime.now(datetime.timezone.utc): - if (entry_date.year, entry_date.month) not in code_changes_by_month: - by_month_list.append(DoraMonthlyResponse( - changes=[], - month=entry_date.month, - year=entry_date.year, - median_change_lead_time=-1, - )) - entry_date += datetime.timedelta(days=30) + if not old_repo_url == new_repo_url: + print("repo urls are not equal") + return # ensure there was no repository-change between component-versions - return by_month_list + old_commit = old_access.commit or old_access.ref + new_commit = new_access.commit or new_access.ref - -@caching.cached( - cache=caching.LFUFilesystemCache(max_total_size_mib=128), - key_func=_cache_key_gen_dora, -) -def dora_deployments( - component_dependency_changes_with_commits: list[ - ComponentDependencyChangeWithCommits - ], -) -> list[DoraDeploymentsResponse]: - deployments: list[DoraDeploymentsResponse] = [] - - for component_dependency_change_with_commits in component_dependency_changes_with_commits: - - median_change_lead_time = datetime.timedelta( - seconds=statistics.median([ - (components.get_creation_date( - component_dependency_change_with_commits.component - ) - dateutil.parser.isoparse( - commit.commit.author['date'] - )).total_seconds() - for commit in component_dependency_change_with_commits.commits - ]) if component_dependency_change_with_commits.commits else 0, + github_repo = _github_repo( + repo_url=old_repo_url, # already checked for equality; choose either ) - deployment_date = components.get_creation_date( - component_dependency_change_with_commits.component + commits = commits_between( + older_commit=old_commit, + newer_commit=new_commit, + github_repo=github_repo, ) - deployments.append( - DoraDeploymentsResponse( - deployment_date=deployment_date, - component_version=( - component_dependency_change_with_commits.dependency_component_vector.end.version + deployment_date = components.get_creation_date( + (await component_descriptor_lookup( + ocm.ComponentIdentity( + name=cnudie.util.to_component_name(target_component_name), + version=target_update.target_component_version_new, ), - target_deployment_version=component_dependency_change_with_commits.component.version, - changes=[ - CodeChange( - commit_date=dateutil.parser.isoparse(commit.commit.author['date']), - commit_sha=commit.sha, - deployment_date=deployment_date, - ) - for commit in component_dependency_change_with_commits.commits - ], - median_change_lead_time=median_change_lead_time.days, - ) + )).component ) - return deployments - - -def all_change_lead_time_durations( - component_dependency_changes_with_commits: list[ - ComponentDependencyChangeWithCommits - ], - time_span_days: int, -) -> list[int]: - commit_durations = [] - for component_dependency_change_with_commits in component_dependency_changes_with_commits: - commit_durations.extend( - [ - ( - components.get_creation_date( - component_dependency_change_with_commits.component - ) - - dateutil.parser.isoparse(commit.commit.author['date']) - ).total_seconds() - for commit in component_dependency_change_with_commits.commits + for commit in commits: + commit_objects: list[dora_result_calcs.ReturnCommitObject] = [] + for commit in commits: if ( - dateutil.parser.isoparse(commit.commit.author['date']) > ( - datetime.datetime.now(datetime.timezone.utc) - - datetime.timedelta(days=time_span_days) + commit_date := components.ensure_utc(dateutil.parser.isoparse(commit.commit.author['date'])) + ) > ( + datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(days=time_span_days) + ) + ): + commit_objects.append( + dora_result_calcs.ReturnCommitObject( + commitDate=commit_date, + commitSha=commit.sha, + deploymentDate=deployment_date, + leadTime=(deployment_date - commit_date), + url=commit.html_url, + ), ) - ) - ] - ) - - return commit_durations - - -def all_changes( - component_dependency_changes_with_commits: list[ - ComponentDependencyChangeWithCommits - ], - time_span_days: int, -) -> list[CodeChange]: - - all_changes = [] - for component_dependency_change_with_commits in component_dependency_changes_with_commits: - all_changes.extend( - [ - CodeChange( - commit_sha=commit.sha, - commit_date=dateutil.parser.isoparse(commit.commit.author['date']), - deployment_date=components.get_creation_date( - component_dependency_change_with_commits.component, - ), - ) for commit in component_dependency_change_with_commits.commits - if dateutil.parser.isoparse(commit.commit.author['date']) > - datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=time_span_days) - ] - ) - - return all_changes - - -def create_response_object( - target_updates_by_dependency: dict[ - str, - list[ComponentDependencyChangeWithCommits], - ], - time_span_days: int, -): - dependencies_response: dict[ - str, - DoraDependencyResponse, - ] = {} - - all_change_lead_time_durations_seconds = [] - - for dependency_name, component_dependency_changes_with_commits \ - in target_updates_by_dependency.items(): - - median = calculate_change_lead_time( - component_dependency_changes_with_commits, - time_span_days, - CalculationType.MEDIAN, - ) - average = calculate_change_lead_time( - component_dependency_changes_with_commits, - time_span_days, - CalculationType.AVERAGE, - ) - changes_monthly = dora_changes_monthly( - component_dependency_changes_with_commits, - time_span_days, - ) - deployments = dora_deployments( - component_dependency_changes_with_commits, - ) - changes = all_changes( - component_dependency_changes_with_commits, - time_span_days, - ) - repo_url = cnudie.util.main_source( - component_dependency_changes_with_commits[0].dependency_component_vector.start - ).access.repoUrl - - dependencies_response[dependency_name] = DoraDependencyResponse( - change_lead_time_median=median.days, - change_lead_time_average=average.days, - deployment_frequency=round(time_span_days / len(deployments), 2), - changes_monthly=changes_monthly, - deployments=deployments, - all_changes=changes, - repo_url=repo_url, - ) - all_change_lead_time_durations_seconds.extend( - all_change_lead_time_durations( - component_dependency_changes_with_commits, - time_span_days, + deployment_objects.append( + dora_result_calcs.ReturnDeploymentObject( + targetComponentVersionNew=target_update.target_component_version_new, + targetComponentVersionOld=target_update.target_component_version_old, + deployedComponentVersion=target_update.referenced_component_version_newer_release, + oldComponentVersion=target_update.referenced_component_version_older_release, + deploymentDate=deployment_date, + commits=commit_objects ) ) - if all_change_lead_time_durations_seconds != []: - change_lead_time_median = datetime.timedelta( - seconds=statistics.median( - all_change_lead_time_durations_seconds - ) - ).days - change_lead_time_average = datetime.timedelta( - seconds=statistics.mean( - all_change_lead_time_durations_seconds - ) - ).days - else: - change_lead_time_median = -1 - change_lead_time_average = -1 - - return DoraResponse( - change_lead_time_median=change_lead_time_median, - change_lead_time_average=change_lead_time_average, - dependencies=dependencies_response, - ) + tasks = [ + create_deployment_object_for_update(target_version_change_with_ref_change) + for target_version_change_with_ref_change in target_updates + ] + await asyncio.gather(*tasks) + return deployment_objects +@middleware.auth.noauth class DoraMetrics(aiohttp.web.View): async def get(self): ''' --- + description: + Retrieve DORA metrics for a target component over a specified time span. tags: - - Dora + - DORA produces: - application/json parameters: @@ -765,45 +421,151 @@ async def get(self): name: target_component_name type: string required: true + description: The name of the target component. - in: query name: time_span_days type: integer required: false default: 90 + description: The number of days for the time span (default is 90). - in: query - name: filter_component_names - schema: - type: array - items: - type: string - required: false - responses: + name: filter_component_name + type: string + required: true + description: The name of the component to calculate the Dora Metrics for. + responses: + responses: "200": - description: Successful operation. - schema: - type: object - required: - - change_lead_time_median - - change_lead_time_average - - dependencies - properties: - change_lead_time_median: - type: number - change_lead_time_average: - type: number - dependencies: + description: Successful operation. Returns DORA metrics for the specified components. + content: + application/json: + schema: type: object + properties: + targetComponentName: + type: string + description: The name of the target component. + timePeriod: + type: number + format: float + description: The time period over which metrics were calculated. + componentName: + type: string + description: The name of the component for which DORA metrics are calculated. + medianDeploymentFrequency: + type: number + format: float + description: Median deployment frequency. + medianLeadTime: + type: number + format: float + description: Median lead time in seconds. + deploymentsPerMonth: + type: object + additionalProperties: + type: integer + description: Deployments per month. + deploymentsPerWeek: + type: object + additionalProperties: + type: integer + description: Deployments per week. + deploymentsPerDay: + type: object + additionalProperties: + type: integer + description: Deployments per day. + leadTimePerMonth: + type: object + additionalProperties: + type: number + format: float + description: Median lead time per month in seconds. + leadTimePerWeek: + type: object + additionalProperties: + type: number + format: float + description: Median lead time per week in seconds. + leadTimePerDay: + type: object + additionalProperties: + type: number + format: float + description: Median lead time per day in seconds. + deployments: + type: array + items: + type: object + properties: + targetComponentVersionNew: + type: string + description: New version of the target component. + targetComponentVersionOld: + type: string + description: Previous version of the target component. + deployedComponentVersion: + type: string + description: New version of the deployed component. + oldComponentVersion: + type: string + description: Previous version of the deployed component. + deploymentDate: + type: string + format: date-time + description: Date and time of deployment. + commits: + type: array + items: + type: object + properties: + commitSha: + type: string + description: SHA of the commit. + commitDate: + type: string + format: date-time + description: Date of the commit. + deploymentDate: + type: string + format: date-time + description: Date of deployment. + leadTime: + type: number + format: float + description: Lead time in seconds. + url: + type: string + format: uri + description: URL to the commit. + description: List of deployments with associated commits. "202": description: Dora metric calculation pending, client should retry. ''' + params = self.request.rel_url.query - target_component_name = util.param(params, 'target_component_name', required=True) - time_span_days = int(util.param(params, 'time_span_days', default=90)) - filter_component_names = params.getall('filter_component_names', default=[]) + target_component_name: str = util.param( + params, + 'target_component_name', + required=True + ) + + time_span_days: int = int(util.param( + params, + 'time_span_days', + default=90, + )) + + filter_component_name: str = util.param( + params, + 'filter_component_name', + required=True, + ) - component_descriptor_lookup = self.request.app[consts.APP_COMPONENT_DESCRIPTOR_LOOKUP] - version_lookup = self.request.app[consts.APP_VERSION_LOOKUP] + component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById = self.request.app[consts.APP_COMPONENT_DESCRIPTOR_LOOKUP] + version_lookup: cnudie.retrieve.VersionLookupByComponent = self.request.app[consts.APP_VERSION_LOOKUP] + github_api_lookup = self.request.app[consts.APP_GITHUB_API_LOOKUP] await components.check_if_component_exists( component_name=target_component_name, @@ -811,266 +573,136 @@ async def get(self): raise_http_error=True, ) - for filter_component_name in filter_component_names: - await components.check_if_component_exists( - component_name=filter_component_name, - version_lookup=version_lookup, - raise_http_error=True, - ) - - # get all component descriptors of component versions of target component within time span - target_descriptors_in_time_span = await versions_descriptors_newer_than( - component_name=target_component_name, - date=datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=time_span_days), - component_descriptor_lookup=component_descriptor_lookup, + await components.check_if_component_exists( + component_name=filter_component_name, version_lookup=version_lookup, - sorting_direction='asc', + raise_http_error=True, ) - # Add the next older version, which is not within the time span anymore (if one exists) - # at the beginning of the descriptor list to be able to detect changes which were - # first introduced within the time span of the target component version. + all_target_component_versions = await all_versions_sorted( + component=target_component_name, + version_lookup=version_lookup, + ) - if (next_older_descriptor := await get_next_older_descriptor( - component_id=ocm.ComponentIdentity( - target_component_name, - target_descriptors_in_time_span[0].component.version, - ), + target_component_versions = await filter_versions_newer_than( + component=target_component_name, + all_versions=all_target_component_versions, + date=datetime.datetime.now() - datetime.timedelta(days=time_span_days), component_descriptor_lookup=component_descriptor_lookup, - component_version_lookup=version_lookup, - )): - target_descriptors_in_time_span.insert(0, next_older_descriptor) - - # calculate the changes which where introduced for every component version - target_descriptors_with_updates: list[ComponentWithDependencyChanges] = [] - for index in range(1, len(target_descriptors_in_time_span)): - component_diff = await _diff_components( - component_vector=components.ComponentVector( - start=target_descriptors_in_time_span[index-1].component, - end=target_descriptors_in_time_span[index].component, - ), - component_descriptor_lookup=component_descriptor_lookup, - ) + ) - if component_diff: - dependency_changes = dependency_changes_between_versions( - component_diff=component_diff, - dependency_name_filter=filter_component_names, - only_rising_changes=True, - ) - else: - dependency_changes = [] + target_component_verisons_amount = len(target_component_versions) - target_descriptors_with_updates.append( - ComponentWithDependencyChanges( - component_descriptor=target_descriptors_in_time_span[index], - dependency_changes=dependency_changes, - ) - ) + # add the last version out of the date range to the list, else it would not be possible to check if + # there where any version changes within the last release within the date range + if target_component_verisons_amount != len(all_target_component_versions): + target_component_versions.append(all_target_component_versions[len(target_component_versions)]) + # TODO how to handle if first release of target component is within the date range - target_descriptors_with_updates = tuple(target_descriptors_with_updates) + target_version_updates: list[ComponentVersionUpdate] = [] - key = _cache_key_changes_by_dependencies(target_descriptors_with_updates) + for id in range(0, target_component_verisons_amount - 1): - # categorize changes by changed dependency - # and add commits to the dependency changes - if (updates_by_dependency := changes_by_dependencies_cache.get(key)) is None: - if key not in changes_by_dependencies_cache: - changes_by_dependencies_cache[key] = None - tpe = concurrent.futures.ThreadPoolExecutor(max_workers=1) - tpe.submit( - categorize_by_changed_component, - target_descriptors_with_updates, - self.request.app[consts.APP_GITHUB_API_LOOKUP], + target_version_new = target_component_versions[id] + target_version_old = target_component_versions[id + 1] + + old_target_component: ocm.Component = ( + await component_descriptor_lookup( + ocm.ComponentIdentity( + name=cnudie.util.to_component_name( + target_component_name + ), + version=versionutil.parse_to_semver( + target_version_old + ), + ) ) + ).component - return aiohttp.web.Response( - status=http.HTTPStatus.ACCEPTED, + old_target_component_tree = await latest_referenced_component_versions( + component=old_target_component, + component_descriptor_lookup=component_descriptor_lookup, ) - return aiohttp.web.json_response( - data=create_response_object( - target_updates_by_dependency=updates_by_dependency, - time_span_days=time_span_days, - ), - dumps=util.dict_to_json_factory, - ) + referenced_component_version_older_release = old_target_component_tree[ + filter_component_name + ].version + new_target_component = (await component_descriptor_lookup( + ocm.ComponentIdentity( + name=cnudie.util.to_component_name(target_component_name), + version=versionutil.parse_to_semver(target_version_new), + ) + )).component -def _cache_key_diff_components( - component_vector: components.ComponentVector, - component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, -): - return cachetools.keys.hashkey( - component_vector.start.name, - component_vector.end.name, - component_vector.start.version, - component_vector.end.version, - ) + new_target_component_tree = await latest_referenced_component_versions( + component=new_target_component, + component_descriptor_lookup=component_descriptor_lookup, + ) + referenced_component_version_newer_release = new_target_component_tree[ + filter_component_name + ].version + + target_version_updates.append( + ComponentVersionUpdate( + target_component=target_component_name, + target_component_version_old=target_version_old, + target_component_version_new=target_version_new, + referenced_component=filter_component_name, + referenced_component_version_older_release=referenced_component_version_older_release, + referenced_component_version_newer_release=referenced_component_version_newer_release, + ) + ) -@caching.async_cached( - cache=caching.LFUFilesystemCache(max_total_size_mib=256), - key_func=_cache_key_diff_components, -) -async def _diff_components( - component_vector: components.ComponentVector, - component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, -) -> cnudie.util.ComponentDiff | None: - ''' - calculates component-diff between components from passed-in component-vector + # filter out target version updates where the referenced component version did not change + target_update_with_ref_updates = [ + target_version_update + for target_version_update in target_version_updates + if target_version_update.referenced_component_version_older_release + < target_version_update.referenced_component_version_newer_release + ] - this function is mostly identical to cnudie.util.diff_components. It differs, however, - in that it will merge multiple component-versions (of the same component) into just one - component-version, choosing greatest/smallest versions. - ''' - old_components = [ - c.component async for c in cnudie.iter_async.iter( - component=component_vector.start, - lookup=component_descriptor_lookup, - node_filter=cnudie.iter.Filter.components, + deployment_objects = await create_deployment_objects( + component_descriptor_lookup=component_descriptor_lookup, + component_version_lookup=version_lookup, + github_api_lookup=github_api_lookup, + target_component_name=target_component_name, + time_span_days=time_span_days, + filter_component_name=filter_component_name, + target_updates=target_update_with_ref_updates, ) - ] - new_components = [ - c.component async for c in cnudie.iter_async.iter( - component=component_vector.end, - lookup=component_descriptor_lookup, - node_filter=cnudie.iter.Filter.components, + deployments_per = dora_result_calcs.calc_deployments_per( + deployment_objects=deployment_objects, ) - ] - - def only_greatest_versions(components: list[ocm.Component]): - components_by_name: collections.defaultdict[ - str, list[ocm.Component] - ] = collections.defaultdict(list[ocm.Component]) - - for c in components: - components_by_name[c.name].append(c) - - greatest_component_versions = [] - for component_name, component_list in components_by_name.items(): - if len(component_list) == 1: - greatest_component_versions.append(component_list[0]) - continue - current_biggest_version = component_list[0] - for c in component_list[1:]: - if( - versionutil.parse_to_semver(c.version) > - versionutil.parse_to_semver(current_biggest_version.version) - ): - current_biggest_version = c - greatest_component_versions.append(current_biggest_version) - - return greatest_component_versions - - old_greatest_component_versions = only_greatest_versions(old_components) - new_greatest_component_versions = only_greatest_versions(new_components) - - old_greatest_component_identities = { - c.identity() for c in old_greatest_component_versions - } - new_greatest_component_identities = { - c.identity() for c in new_greatest_component_versions - } - - old_only_greatest_component_identities = ( - old_greatest_component_identities - new_greatest_component_identities - ) - new_only_greatest_component_identities = ( - new_greatest_component_identities - old_greatest_component_identities - ) - - old_only_greatest_component_versions = [ - c for c in old_greatest_component_versions - if c.identity() in old_only_greatest_component_identities - ] - new_only_greatest_component_versions = [ - c for c in new_greatest_component_versions - if c.identity() in new_only_greatest_component_identities - ] - if old_only_greatest_component_identities == new_only_greatest_component_identities: - return None # no diff + median_deployment_frequency = statistics.mean(deployments_per['deploymentsPerMonth'].values()) - def find_changed_component( - old_only_component_version: ocm.Component, - new_only_component_versions: list[ocm.Component], - ): - for new_only_component_version in new_only_component_versions: - if new_only_component_version.name == old_only_component_version.name: - return (old_only_component_version, new_only_component_version) - return (old_only_component_version, None) # no pair component found - - components_with_changed_versions = [] - for old_only_greatest_component_version in old_only_greatest_component_versions: - changed_component = find_changed_component( - old_only_greatest_component_version, - new_only_greatest_component_versions, + lead_time_per = dora_result_calcs.calc_lead_time_per( + deployment_objects=deployment_objects, ) - if changed_component[1] is not None: - components_with_changed_versions.append(changed_component) - - old_component_names = {i.name for i in old_greatest_component_identities} - new_component_names = {i.name for i in new_greatest_component_identities} - names_version_changed = {c[0].name for c in components_with_changed_versions} - - both_names = old_component_names & new_component_names - old_component_names -= both_names - new_component_names -= both_names - - return cnudie.util.ComponentDiff( - cidentities_only_left=set(), - cidentities_only_right=set(), - cpairs_version_changed=components_with_changed_versions, - names_only_left=old_component_names, - names_only_right=new_component_names, - names_version_changed=names_version_changed, - ) - - -def dependency_changes_between_versions( - component_diff: cnudie.util.ComponentDiff, - dependency_name_filter: collections.abc.Iterable[str] | None = None, - only_rising_changes: bool = False, -) -> list[components.ComponentVector]: - ''' - This function retrieves the changes which where made between two versions of a Component. - There is the possibilitie to filter for the changes of just one component. - - @param dependency_name_filter: If given a dependency_name_filter (Component Names), - only the changes of these specific components are returned. - @param only_rising_changes: If True, only the changes are returned, where the version - of the new component is higher than the version of the old component. - - @returns: List of the changes between the two versions - ''' - if not component_diff: - raise ValueError(component_diff) - - changes: list[components.ComponentVector] = [] - for left_component, right_component in component_diff.cpairs_version_changed: - if ( - dependency_name_filter - and left_component.name not in dependency_name_filter - ): - continue - - left_version = versionutil.parse_to_semver(left_component.version) - right_version = versionutil.parse_to_semver(right_component.version) - - if only_rising_changes: - if left_version >= right_version: - continue - elif left_version == right_version: - continue + median_lead_time = statistics.median( + lead_time_per['medianLeadTimePerMonth'].values() + ) - changes.append( - components.ComponentVector( - left_component, - right_component, - ) + return_object = dora_result_calcs.ReturnObject( + targetComponentName=target_component_name, + timePeriod=time_span_days, + componentName=filter_component_name, + deploymentsPerMonth=deployments_per['deploymentsPerMonth'], + deploymentsPerWeek=deployments_per['deploymentsPerWeek'], + deploymentsPerDay=deployments_per['deploymentsPerDay'], + medianDeploymentFrequency=median_deployment_frequency, + leadTimePerMonth=lead_time_per['medianLeadTimePerMonth'], + leadTimePerWeek=lead_time_per['medianLeadTimePerWeek'], + leadTimePerDay=lead_time_per['medianLeadTimePerDay'], + medianLeadTime=median_lead_time, + deployments=deployment_objects ) - return changes + return aiohttp.web.json_response( + data=return_object.to_dict(), + dumps=util.dict_to_json_factory, + ) diff --git a/dora_result_calcs.py b/dora_result_calcs.py new file mode 100644 index 00000000..1c129811 --- /dev/null +++ b/dora_result_calcs.py @@ -0,0 +1,140 @@ +import collections +import dataclasses +import datetime +import statistics + + +@dataclasses.dataclass(frozen=True) +class ReturnCommitObject: + commitSha: str + commitDate: datetime.datetime + deploymentDate: datetime.datetime + leadTime: datetime.timedelta + url: str + + def to_dict(self): + return { + 'commitSha': self.commitSha, + 'commitDate': self.commitDate.isoformat(), + 'deploymentDate': self.deploymentDate.isoformat(), + 'leadTime': self.leadTime.total_seconds(), + 'url': self.url, + } + + +@dataclasses.dataclass(frozen=True) +class ReturnDeploymentObject: + targetComponentVersionNew: str + targetComponentVersionOld: str + deployedComponentVersion: str + oldComponentVersion: str + deploymentDate: datetime.datetime + commits: list[ReturnCommitObject] + + def to_dict(self): + return { + 'targetComponentVersionNew': self.targetComponentVersionNew, + 'targetComponentVersionOld': self.targetComponentVersionOld, + 'deployedComponentVersion': self.deployedComponentVersion, + 'oldComponentVersion': self.oldComponentVersion, + 'deploymentDate': self.deploymentDate.isoformat(), + 'commits': [commit.to_dict() for commit in self.commits], + } + + +@dataclasses.dataclass(frozen=True) +class ReturnObject: + targetComponentName: str + timePeriod: float + componentName: str + medianDeploymentFrequency: float + medianLeadTime: float + deploymentsPerMonth: dict + deploymentsPerWeek: dict + deploymentsPerDay: dict + leadTimePerMonth: dict + leadTimePerWeek: dict + leadTimePerDay: dict + deployments: list[ReturnDeploymentObject] + + def to_dict(self): + return { + 'targetComponentName': self.targetComponentName, + 'timePeriod': self.timePeriod, + 'componentName': self.componentName, + 'medianDeploymentFrequency': self.medianDeploymentFrequency, + 'medianLeadTime': self.medianLeadTime, + 'deploymentsPerMonth': self.deploymentsPerMonth, + 'deploymentsPerWeek': self.deploymentsPerWeek, + 'deploymentsPerDay': self.deploymentsPerDay, + 'leadTimePerMonth': {k: v for k, v in self.leadTimePerMonth.items()}, + 'leadTimePerWeek': {k: v for k, v in self.leadTimePerWeek.items()}, + 'leadTimePerDay': {k: v for k, v in self.leadTimePerDay.items()}, + 'deployents': [deployment.to_dict() for deployment in self.deployments], + } + + +def calc_deployments_per(deployment_objects: list[ReturnDeploymentObject]): + deployments_per_month = collections.defaultdict(int) + deployments_per_week = collections.defaultdict(int) + deployments_per_day = collections.defaultdict(int) + + for deployment in deployment_objects: + + # First day of the month + first_day_month = deployment.deploymentDate.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + month_key = first_day_month.replace(hour=0, minute=0, second=0, microsecond=0).isoformat() + deployments_per_month[month_key] += 1 + + # First day of the week + first_day_week = deployment.deploymentDate - datetime.timedelta( + days=deployment.deploymentDate.weekday() + ) + week_key = first_day_week.replace(hour=0, minute=0, second=0, microsecond=0).isoformat() + deployments_per_week[week_key] += 1 + + # Exact day + day_key = deployment.deploymentDate.replace(hour=0, minute=0, second=0, microsecond=0).isoformat() + deployments_per_day[day_key] += 1 + + return { + 'deploymentsPerMonth': dict(deployments_per_month), + 'deploymentsPerWeek': dict(deployments_per_week), + 'deploymentsPerDay': dict(deployments_per_day) + } + + +def calc_lead_time_per(deployment_objects: list[ReturnDeploymentObject]): + lead_times_per_month = collections.defaultdict(list) + lead_times_per_week = collections.defaultdict(list) + lead_times_per_day = collections.defaultdict(list) + + for deployment in deployment_objects: + + # First day of the month + first_day_month = deployment.deploymentDate.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + month_key = first_day_month.isoformat() + + # First day of the week + first_day_week = deployment.deploymentDate - datetime.timedelta( + days=deployment.deploymentDate.weekday() + ) + week_key = first_day_week.replace(hour=0, minute=0, second=0, microsecond=0).isoformat() + + # Exact day + day_key = deployment.deploymentDate.replace(hour=0, minute=0, second=0, microsecond=0).isoformat() + + for commit in deployment.commits: + lead_times_per_month[month_key].append(commit.leadTime.total_seconds()) + lead_times_per_week[week_key].append(commit.leadTime.total_seconds()) + lead_times_per_day[day_key].append(commit.leadTime.total_seconds()) + + median_lead_time_per_month = {k: statistics.median(v) for k, v in lead_times_per_month.items()} + median_lead_time_per_week = {k: statistics.median(v) for k, v in lead_times_per_week.items()} + median_lead_time_per_day = {k: statistics.median(v) for k, v in lead_times_per_day.items()} + + return { + 'medianLeadTimePerMonth': median_lead_time_per_month, + 'medianLeadTimePerWeek': median_lead_time_per_week, + 'medianLeadTimePerDay': median_lead_time_per_day, + } From c5f8d8d002fe1b9cd717687b7467a4619678d664 Mon Sep 17 00:00:00 2001 From: Finnegan1 Date: Mon, 2 Dec 2024 01:18:28 +0100 Subject: [PATCH 2/2] first of the requested changes --- app.py | 2 +- components.py | 12 +- dora.py | 292 ++++++++++++------------------------------- dora_result_calcs.py | 74 +++++++---- swagger/swagger.yaml | 101 +++++++++++++++ util.py | 8 ++ 6 files changed, 245 insertions(+), 244 deletions(-) diff --git a/app.py b/app.py index b41d7a3a..ebafaf68 100755 --- a/app.py +++ b/app.py @@ -438,7 +438,7 @@ async def initialise_app(): description='API definition', title='Delivery-Service by Gardener CICD', definitions=api_definitions, - ) + ) return app diff --git a/components.py b/components.py index f1594e2d..51c6b973 100644 --- a/components.py +++ b/components.py @@ -76,16 +76,6 @@ async def check_if_component_exists( return False -def ensure_utc(ts: datetime.datetime) -> datetime.datetime: - if ts.tzinfo is None: - ts = ts.replace(tzinfo=datetime.timezone.utc) - else: - if ts.tzinfo != datetime.timezone.utc: - ts = ts.astimezone(datetime.timezone.utc) - - return ts - - def get_creation_date(component: ocm.Component) -> datetime.datetime: ''' Trys to extract creation date from creationTime attribute and if not set from label with name @@ -94,7 +84,7 @@ def get_creation_date(component: ocm.Component) -> datetime.datetime: ''' if (creationTime := component.creationTime): - return ensure_utc(dateutil.parser.isoparse(creationTime)) + return util.as_timezone(dateutil.parser.isoparse(creationTime)) creation_label: ocm.Label | None = component.find_label('cloud.gardener/ocm/creation-date') diff --git a/dora.py b/dora.py index 49e81f6b..0251f5a6 100644 --- a/dora.py +++ b/dora.py @@ -10,29 +10,27 @@ import aiohttp.web import cachetools.keys -import cnudie.iter_async import dateutil.parser -import dora_result_calcs -import ocm import github3 +import version as versionutil import ci.util +import cnudie.iter_async import cnudie.retrieve +import cnudie.retrieve_async import cnudie.util -import semver -import version as versionutil +import ocm +import dora_result_calcs import caching import components import consts -import middleware -import middleware.auth import util def _cache_key_gen_all_versions_sorted( component: cnudie.retrieve.ComponentName, - version_lookup: cnudie.retrieve.VersionLookupByComponent, + version_lookup: cnudie.retrieve_async.VersionLookupByComponent, only_releases: bool = True, invalid_semver_ok: bool = False, sorting_direction: typing.Literal['asc', 'desc'] = 'desc', @@ -51,7 +49,7 @@ def _cache_key_gen_all_versions_sorted( ) async def all_versions_sorted( component: cnudie.retrieve.ComponentName, - version_lookup: cnudie.retrieve.VersionLookupByComponent, + version_lookup: cnudie.retrieve_async.VersionLookupByComponent, only_releases: bool = True, invalid_semver_ok: bool = False, sorting_direction: typing.Literal['asc', 'desc'] = 'desc' @@ -99,41 +97,45 @@ def filter_version(version: str, invalid_semver_ok: bool, only_releases:bool): async def filter_versions_newer_than( - component: cnudie.retrieve.ComponentName, - all_versions: list[semver.VersionInfo], - date: datetime.datetime, - component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, -) -> list[semver.VersionInfo]: + component: cnudie.retrieve.ComponentName, + all_versions: list[str], + date: datetime.datetime, + component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, +) -> list[str]: ''' Filter list of versions of a component for versions that are newer than a specified date. Returns: A list of version information objects representing versions newer than the specified date. ''' - all_versions = sorted( - all_versions, - key=lambda v: versionutil.parse_to_semver(v), - reverse=True, - ) - - component_versions: list[semver.VersionInfo] = [] + print(all_versions) + print(type(all_versions)) - for version in all_versions: - descriptor: ocm.ComponentDescriptor = await component_descriptor_lookup( - ocm.ComponentIdentity( - name=cnudie.util.to_component_name(component), - version=version, + async def iter_versions_after( + all_versions: list[str], + date: datetime.datetime, + component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, + ) -> collections.abc.AsyncGenerator[str, None, None]: + for version in all_versions: + descriptor: ocm.ComponentDescriptor = await component_descriptor_lookup( + ocm.ComponentIdentity( + name=cnudie.util.to_component_name(component), + version=version, + ) ) - ) - creation_date = components.get_creation_date(descriptor.component) + creation_date = components.get_creation_date(descriptor.component) - date = date.astimezone(datetime.timezone.utc) - creation_date = creation_date.astimezone(datetime.timezone.utc) + date = date.astimezone(datetime.timezone.utc) + creation_date = creation_date.astimezone(datetime.timezone.utc) - if creation_date > date: - component_versions.append(version) - else: - break + if creation_date > date: + yield version + + component_versions: list[str] = list(iter_versions_after( + all_versions=all_versions, + date=date, + component_descriptor_lookup=component_descriptor_lookup, + )) return component_versions @@ -143,7 +145,6 @@ def _cache_key_gen_latest_componentversions_in_tree( component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, ): return cachetools.keys.hashkey( - cnudie.util.to_component_name(component), component.identity() ) @@ -154,7 +155,7 @@ def _cache_key_gen_latest_componentversions_in_tree( ) async def latest_referenced_component_versions( component: ocm.Component, - component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, + component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, ) -> dict[str, ocm.Component]: ''' Retrieve the latest versions of all components referenced in the component tree. @@ -168,11 +169,8 @@ async def latest_referenced_component_versions( Notes: - Only the highest version of each component is retained. - ''' - def default_factory(): - return None - components_by_name = collections.defaultdict(default_factory) + components_by_name = {} referenced_components = [ c.component async for c in cnudie.iter_async.iter( @@ -182,20 +180,17 @@ def default_factory(): ) ] - def version_key(c): - if c is not None: - return versionutil.parse_to_semver(c.version) - else: - return semver.VersionInfo(0, 0, 0) - for referenced_component in referenced_components: - components_by_name[referenced_component.name] = max( - components_by_name[referenced_component.name], - referenced_component, - key=version_key, - ) + if referenced_component.name not in components_by_name: + components_by_name[referenced_component.name] = referenced_component + else: + components_by_name[referenced_component.name] = max( + components_by_name[referenced_component.name], + referenced_component, + key=lambda component: versionutil.parse_to_semver(component.version) + ) - return dict(components_by_name) + return components_by_name @dataclasses.dataclass(frozen=True) @@ -282,19 +277,14 @@ def commits_between( async def create_deployment_objects( - component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, - component_version_lookup: cnudie.retrieve.VersionLookupByComponent, + component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, github_api_lookup, target_component_name: str, time_span_days: int, - filter_component_name: str, target_updates: list[ComponentVersionUpdate], -) -> list[dora_result_calcs.ReturnDeploymentObject]: - ''' - Create objects containing the information about the target component updates in which the referenced component was updated. - ''' +) -> list[dora_result_calcs.DoraDeployment]: - deployment_objects: list[dora_result_calcs.ReturnDeploymentObject] = [] + deployment_objects: list[dora_result_calcs.DoraDeployment] = [] _github_api = functools.cache(github_api_lookup) @@ -321,16 +311,12 @@ async def create_deployment_object_for_update( ) )).component - if not old_ref_component or not new_ref_component: - print ("ERROR, bad input data") - if not can_process( components.ComponentVector( start=old_ref_component, end=new_ref_component, ) ): - print("can't process") return old_access = cnudie.util.main_source(old_ref_component).access @@ -340,14 +326,13 @@ async def create_deployment_object_for_update( new_repo_url = ci.util.urlparse(new_access.repoUrl) if not old_repo_url == new_repo_url: - print("repo urls are not equal") - return # ensure there was no repository-change between component-versions + return old_commit = old_access.commit or old_access.ref new_commit = new_access.commit or new_access.ref github_repo = _github_repo( - repo_url=old_repo_url, # already checked for equality; choose either + repo_url=old_repo_url, ) commits = commits_between( @@ -366,18 +351,18 @@ async def create_deployment_object_for_update( ) for commit in commits: - commit_objects: list[dora_result_calcs.ReturnCommitObject] = [] + commit_objects: list[dora_result_calcs.DoraCommit] = [] for commit in commits: if ( ( - commit_date := components.ensure_utc(dateutil.parser.isoparse(commit.commit.author['date'])) + commit_date := util.as_timezone(dateutil.parser.isoparse(commit.commit.author['date'])) ) > ( datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=time_span_days) ) ): commit_objects.append( - dora_result_calcs.ReturnCommitObject( + dora_result_calcs.DoraCommit( commitDate=commit_date, commitSha=commit.sha, deploymentDate=deployment_date, @@ -387,13 +372,13 @@ async def create_deployment_object_for_update( ) deployment_objects.append( - dora_result_calcs.ReturnDeploymentObject( + dora_result_calcs.DoraDeployment( targetComponentVersionNew=target_update.target_component_version_new, targetComponentVersionOld=target_update.target_component_version_old, deployedComponentVersion=target_update.referenced_component_version_newer_release, oldComponentVersion=target_update.referenced_component_version_older_release, deploymentDate=deployment_date, - commits=commit_objects + commits=tuple(commit_objects) ) ) @@ -405,7 +390,7 @@ async def create_deployment_object_for_update( await asyncio.gather(*tasks) return deployment_objects -@middleware.auth.noauth + class DoraMetrics(aiohttp.web.View): async def get(self): ''' @@ -421,124 +406,22 @@ async def get(self): name: target_component_name type: string required: true - description: The name of the target component. - in: query name: time_span_days type: integer required: false - default: 90 - description: The number of days for the time span (default is 90). - in: query name: filter_component_name type: string required: true description: The name of the component to calculate the Dora Metrics for. - responses: responses: "200": description: Successful operation. Returns DORA metrics for the specified components. content: application/json: schema: - type: object - properties: - targetComponentName: - type: string - description: The name of the target component. - timePeriod: - type: number - format: float - description: The time period over which metrics were calculated. - componentName: - type: string - description: The name of the component for which DORA metrics are calculated. - medianDeploymentFrequency: - type: number - format: float - description: Median deployment frequency. - medianLeadTime: - type: number - format: float - description: Median lead time in seconds. - deploymentsPerMonth: - type: object - additionalProperties: - type: integer - description: Deployments per month. - deploymentsPerWeek: - type: object - additionalProperties: - type: integer - description: Deployments per week. - deploymentsPerDay: - type: object - additionalProperties: - type: integer - description: Deployments per day. - leadTimePerMonth: - type: object - additionalProperties: - type: number - format: float - description: Median lead time per month in seconds. - leadTimePerWeek: - type: object - additionalProperties: - type: number - format: float - description: Median lead time per week in seconds. - leadTimePerDay: - type: object - additionalProperties: - type: number - format: float - description: Median lead time per day in seconds. - deployments: - type: array - items: - type: object - properties: - targetComponentVersionNew: - type: string - description: New version of the target component. - targetComponentVersionOld: - type: string - description: Previous version of the target component. - deployedComponentVersion: - type: string - description: New version of the deployed component. - oldComponentVersion: - type: string - description: Previous version of the deployed component. - deploymentDate: - type: string - format: date-time - description: Date and time of deployment. - commits: - type: array - items: - type: object - properties: - commitSha: - type: string - description: SHA of the commit. - commitDate: - type: string - format: date-time - description: Date of the commit. - deploymentDate: - type: string - format: date-time - description: Date of deployment. - leadTime: - type: number - format: float - description: Lead time in seconds. - url: - type: string - format: uri - description: URL to the commit. - description: List of deployments with associated commits. + $ref: '#/definitions/DoraSummary' "202": description: Dora metric calculation pending, client should retry. ''' @@ -547,8 +430,8 @@ async def get(self): target_component_name: str = util.param( params, - 'target_component_name', - required=True + 'target_component_name', + required=True ) time_span_days: int = int(util.param( @@ -563,8 +446,8 @@ async def get(self): required=True, ) - component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById = self.request.app[consts.APP_COMPONENT_DESCRIPTOR_LOOKUP] - version_lookup: cnudie.retrieve.VersionLookupByComponent = self.request.app[consts.APP_VERSION_LOOKUP] + component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById = self.request.app[consts.APP_COMPONENT_DESCRIPTOR_LOOKUP] + version_lookup: cnudie.retrieve_async.VersionLookupByComponent = self.request.app[consts.APP_VERSION_LOOKUP] github_api_lookup = self.request.app[consts.APP_GITHUB_API_LOOKUP] await components.check_if_component_exists( @@ -599,7 +482,7 @@ async def get(self): target_component_versions.append(all_target_component_versions[len(target_component_versions)]) # TODO how to handle if first release of target component is within the date range - target_version_updates: list[ComponentVersionUpdate] = [] + target_update_with_ref_updates: list[ComponentVersionUpdate] = [] for id in range(0, target_component_verisons_amount - 1): @@ -644,32 +527,23 @@ async def get(self): filter_component_name ].version - target_version_updates.append( - ComponentVersionUpdate( - target_component=target_component_name, - target_component_version_old=target_version_old, - target_component_version_new=target_version_new, - referenced_component=filter_component_name, - referenced_component_version_older_release=referenced_component_version_older_release, - referenced_component_version_newer_release=referenced_component_version_newer_release, - ) + target_version_update = ComponentVersionUpdate( + target_component=target_component_name, + target_component_version_old=target_version_old, + target_component_version_new=target_version_new, + referenced_component=filter_component_name, + referenced_component_version_older_release=referenced_component_version_older_release, + referenced_component_version_newer_release=referenced_component_version_newer_release, ) - # filter out target version updates where the referenced component version did not change - target_update_with_ref_updates = [ - target_version_update - for target_version_update in target_version_updates - if target_version_update.referenced_component_version_older_release - < target_version_update.referenced_component_version_newer_release - ] + if target_version_update.referenced_component_version_older_release < target_version_update.referenced_component_version_newer_release: + target_update_with_ref_updates.append(target_version_update) deployment_objects = await create_deployment_objects( component_descriptor_lookup=component_descriptor_lookup, - component_version_lookup=version_lookup, github_api_lookup=github_api_lookup, target_component_name=target_component_name, time_span_days=time_span_days, - filter_component_name=filter_component_name, target_updates=target_update_with_ref_updates, ) @@ -677,32 +551,32 @@ async def get(self): deployment_objects=deployment_objects, ) - median_deployment_frequency = statistics.mean(deployments_per['deploymentsPerMonth'].values()) + median_deployment_frequency = statistics.mean(deployments_per.deploymentsPerMonth.values()) lead_time_per = dora_result_calcs.calc_lead_time_per( deployment_objects=deployment_objects, ) median_lead_time = statistics.median( - lead_time_per['medianLeadTimePerMonth'].values() + lead_time_per.medianLeadTimePerMonth.values() ) - return_object = dora_result_calcs.ReturnObject( + doraSummary = dora_result_calcs.DoraSummary( targetComponentName=target_component_name, timePeriod=time_span_days, componentName=filter_component_name, - deploymentsPerMonth=deployments_per['deploymentsPerMonth'], - deploymentsPerWeek=deployments_per['deploymentsPerWeek'], - deploymentsPerDay=deployments_per['deploymentsPerDay'], + deploymentsPerMonth=deployments_per.deploymentsPerMonth, + deploymentsPerWeek=deployments_per.deploymentsPerWeek, + deploymentsPerDay=deployments_per.deploymentsPerDay, medianDeploymentFrequency=median_deployment_frequency, - leadTimePerMonth=lead_time_per['medianLeadTimePerMonth'], - leadTimePerWeek=lead_time_per['medianLeadTimePerWeek'], - leadTimePerDay=lead_time_per['medianLeadTimePerDay'], + leadTimePerMonth=lead_time_per.medianLeadTimePerMonth, + leadTimePerWeek=lead_time_per.medianLeadTimePerWeek, + leadTimePerDay=lead_time_per.medianLeadTimePerDay, medianLeadTime=median_lead_time, deployments=deployment_objects ) return aiohttp.web.json_response( - data=return_object.to_dict(), + data=doraSummary.to_dict(), dumps=util.dict_to_json_factory, ) diff --git a/dora_result_calcs.py b/dora_result_calcs.py index 1c129811..88fd12b3 100644 --- a/dora_result_calcs.py +++ b/dora_result_calcs.py @@ -2,10 +2,11 @@ import dataclasses import datetime import statistics +import typing @dataclasses.dataclass(frozen=True) -class ReturnCommitObject: +class DoraCommit: commitSha: str commitDate: datetime.datetime deploymentDate: datetime.datetime @@ -23,13 +24,13 @@ def to_dict(self): @dataclasses.dataclass(frozen=True) -class ReturnDeploymentObject: +class DoraDeployment: targetComponentVersionNew: str targetComponentVersionOld: str deployedComponentVersion: str oldComponentVersion: str deploymentDate: datetime.datetime - commits: list[ReturnCommitObject] + commits: tuple[DoraCommit] def to_dict(self): return { @@ -43,7 +44,7 @@ def to_dict(self): @dataclasses.dataclass(frozen=True) -class ReturnObject: +class DoraSummary: targetComponentName: str timePeriod: float componentName: str @@ -55,7 +56,7 @@ class ReturnObject: leadTimePerMonth: dict leadTimePerWeek: dict leadTimePerDay: dict - deployments: list[ReturnDeploymentObject] + deployments: list[DoraDeployment] def to_dict(self): return { @@ -73,8 +74,21 @@ def to_dict(self): 'deployents': [deployment.to_dict() for deployment in self.deployments], } +@dataclasses.dataclass(frozen=True) +class DeploymentFrequencies: + deploymentsPerMonth: dict[typing.Any, int] + deploymentsPerWeek: dict[typing.Any, int] + deploymentsPerDay: dict[typing.Any, int] -def calc_deployments_per(deployment_objects: list[ReturnDeploymentObject]): + def to_dict(self): + return { + 'deploymentsPerMonth': self.deploymentsPerMonth, + 'deploymentsPerWeek': self.deploymentsPerWeek, + 'deploymentsPerDay': self.deploymentsPerDay, + } + + +def calc_deployments_per(deployment_objects: list[DoraDeployment]): deployments_per_month = collections.defaultdict(int) deployments_per_week = collections.defaultdict(int) deployments_per_day = collections.defaultdict(int) @@ -82,29 +96,43 @@ def calc_deployments_per(deployment_objects: list[ReturnDeploymentObject]): for deployment in deployment_objects: # First day of the month - first_day_month = deployment.deploymentDate.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - month_key = first_day_month.replace(hour=0, minute=0, second=0, microsecond=0).isoformat() + first_day_month = deployment.deploymentDate.date().replace(day=1) + month_key = first_day_month.isoformat() deployments_per_month[month_key] += 1 # First day of the week - first_day_week = deployment.deploymentDate - datetime.timedelta( + first_day_week = (deployment.deploymentDate - datetime.timedelta( days=deployment.deploymentDate.weekday() - ) - week_key = first_day_week.replace(hour=0, minute=0, second=0, microsecond=0).isoformat() + )).date() + week_key = first_day_week.isoformat() deployments_per_week[week_key] += 1 # Exact day - day_key = deployment.deploymentDate.replace(hour=0, minute=0, second=0, microsecond=0).isoformat() + day_key = deployment.deploymentDate.date().isoformat() deployments_per_day[day_key] += 1 - return { - 'deploymentsPerMonth': dict(deployments_per_month), - 'deploymentsPerWeek': dict(deployments_per_week), - 'deploymentsPerDay': dict(deployments_per_day) - } + return DeploymentFrequencies( + deploymentsPerMonth=dict(deployments_per_month), + deploymentsPerWeek=dict(deployments_per_week), + deploymentsPerDay=dict(deployments_per_day), + ) + + +@dataclasses.dataclass(frozen=True) +class LeadTimes: + medianLeadTimePerMonth: dict[typing.Any, float] + medianLeadTimePerWeek: dict[typing.Any, float] + medianLeadTimePerDay: dict[typing.Any, float] + + def to_dict(self): + return { + 'medianLeadTimePerMonth': self.medianLeadTimePerMonth, + 'medianLeadTimePerWeek': self.medianLeadTimePerWeek, + 'medianLeadTimePerDay': self.medianLeadTimePerDay, + } -def calc_lead_time_per(deployment_objects: list[ReturnDeploymentObject]): +def calc_lead_time_per(deployment_objects: list[DoraDeployment]): lead_times_per_month = collections.defaultdict(list) lead_times_per_week = collections.defaultdict(list) lead_times_per_day = collections.defaultdict(list) @@ -133,8 +161,8 @@ def calc_lead_time_per(deployment_objects: list[ReturnDeploymentObject]): median_lead_time_per_week = {k: statistics.median(v) for k, v in lead_times_per_week.items()} median_lead_time_per_day = {k: statistics.median(v) for k, v in lead_times_per_day.items()} - return { - 'medianLeadTimePerMonth': median_lead_time_per_month, - 'medianLeadTimePerWeek': median_lead_time_per_week, - 'medianLeadTimePerDay': median_lead_time_per_day, - } + return LeadTimes( + medianLeadTimePerMonth=dict(median_lead_time_per_month), + medianLeadTimePerWeek=dict(median_lead_time_per_week), + medianLeadTimePerDay=dict(median_lead_time_per_day), + ) diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index a4dd6c46..e005b8a8 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -437,3 +437,104 @@ definitions: $ref: '#/definitions/KubernetesMetadata' spec: $ref: '#/definitions/RuntimeArtefactSpec' + + DoraSummary: + type: object + properties: + targetComponentName: + type: string + description: The name of the target component. + timePeriod: + type: number + format: float + description: The time period over which metrics were calculated. + componentName: + type: string + description: The name of the component for which DORA metrics are calculated. + medianDeploymentFrequency: + type: number + format: float + description: Median deployment frequency. + medianLeadTime: + type: number + format: float + description: Median lead time in seconds. + deploymentsPerMonth: + type: object + additionalProperties: + type: integer + description: Deployments per month. + deploymentsPerWeek: + type: object + additionalProperties: + type: integer + description: Deployments per week. + deploymentsPerDay: + type: object + additionalProperties: + type: integer + description: Deployments per day. + leadTimePerMonth: + type: object + additionalProperties: + type: number + format: float + description: Median lead time per month in seconds. + leadTimePerWeek: + type: object + additionalProperties: + type: number + format: float + description: Median lead time per week in seconds. + leadTimePerDay: + type: object + additionalProperties: + type: number + format: float + description: Median lead time per day in seconds. + deployments: + description: List of deployments with associated commits. + type: array + items: + type: object + properties: + targetComponentVersionNew: + type: string + description: New version of the target component. + targetComponentVersionOld: + type: string + description: Previous version of the target component. + deployedComponentVersion: + type: string + description: New version of the deployed component. + oldComponentVersion: + type: string + description: Previous version of the deployed component. + deploymentDate: + type: string + format: date-time + description: Date and time of deployment. + commits: + type: array + items: + type: object + properties: + commitSha: + type: string + description: SHA of the commit. + commitDate: + type: string + format: date-time + description: Date of the commit. + deploymentDate: + type: string + format: date-time + description: Date of deployment. + leadTime: + type: number + format: float + description: Lead time in seconds. + url: + type: string + format: uri + description: URL to the commit. diff --git a/util.py b/util.py index f0254c31..20b742f8 100644 --- a/util.py +++ b/util.py @@ -200,3 +200,11 @@ def error_description( 'error_id': error_id, **kwargs, }) + + +def as_timezone(ts: datetime.datetime, target_tz: datetime.timezone = datetime.timezone.utc) -> datetime.datetime: + if ts.tzinfo is None: + return ts.replace(tzinfo=target_tz) + if ts.tzinfo != target_tz: + return ts.astimezone(target_tz) + return ts