From 7d987e33875dfda135ba0b7b6066c18bdff74601 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Wed, 15 Jan 2025 12:23:44 +0100 Subject: [PATCH 1/9] Ensure event reports are also stored with the desired decimal and column separator --- R/g.report.part2.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/R/g.report.part2.R b/R/g.report.part2.R index de092850b..1afd52665 100644 --- a/R/g.report.part2.R +++ b/R/g.report.part2.R @@ -266,7 +266,8 @@ g.report.part2 = function(metadatadir = c(), f0 = c(), f1 = c(), maxdur = 0, #----------------------------------------------------------------------- data.table::fwrite(x = dayEVENTSUMMARY_clean, file = paste0(metadatadir, "/results/part2_day", eventName, "summary.csv"), - row.names = F, na = "") + row.names = F, na = "", sep = params_output[["sep_reports"]], + dec = params_output[["dec_reports"]]) } # split SUMMARY in two files and reoder EventVariable names if they exist s_names = names(SUMMARY) @@ -281,7 +282,8 @@ g.report.part2 = function(metadatadir = c(), f0 = c(), f1 = c(), maxdur = 0, EVENTSUMMARY_clean = tidyup_df(EVENTSUMMARY) data.table::fwrite(x = EVENTSUMMARY_clean, file = paste0(metadatadir, "/results/part2_", eventName, "summary.csv"), - row.names = F, na = "") + row.names = F, na = "", sep = params_output[["sep_reports"]], + dec = params_output[["dec_reports"]]) } #----------------------------- # tidy up data.frames From 941c2a0f2f4abe44b8ba99e6c1825e9e49f95eb4 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Wed, 15 Jan 2025 12:24:15 +0100 Subject: [PATCH 2/9] Part 2 report needs to define f1 based on number of files in basic folder and not ms2.out folder --- R/GGIR.R | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/R/GGIR.R b/R/GGIR.R index 9b44b3da9..939ceeac1 100644 --- a/R/GGIR.R +++ b/R/GGIR.R @@ -352,13 +352,11 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(), # ----- # check a few basic assumptions before continuing if (length(which(do.report == 2)) > 0) { - N.files.ms2.out = length(dir(paste0(metadatadir, "/meta/ms2.out"))) - if (N.files.ms2.out > 0) { + N.files.basic = length(dir(paste0(metadatadir, "/meta/basic"))) + if (N.files.basic > 0) { if (verbose == TRUE) print_console_header("Report part 2") - # if (N.files.ms2.out < f0) f0 = 1 - # if (N.files.ms2.out < f1) f1 = N.files.ms2.out if (length(f0) == 0) f0 = 1 - if (f1 == 0) f1 = N.files.ms2.out + if (f1 == 0) f1 = N.files.basic if (length(params_247[["qwindow"]]) > 2 | (is.character(params_247[["qwindow"]]) && length(grep(pattern = "onlyfilter|filteronly", x = params_247[["qwindow"]])) == 0) | (length(params_247[["qwindow"]]) == 2 & !all(c(0, 24) %in% params_247[["qwindow"]]))) { From fe9577cc1c5169d25f481fed24cc7f9773c3994d Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 17 Jan 2025 13:51:49 +0100 Subject: [PATCH 3/9] improve date recognition in sleepdiary --- R/g.loadlog.R | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/R/g.loadlog.R b/R/g.loadlog.R index ec3bcdc07..eb4867684 100644 --- a/R/g.loadlog.R +++ b/R/g.loadlog.R @@ -204,6 +204,11 @@ g.loadlog = function(loglocation = c(), coln1 = c(), colid = c(), } napcnt = nwcnt = iccnt = 1 IDcouldNotBeMatched = TRUE + dateformat_found = FALSE + dateformats_to_consider = c("%Y-%m-%d", "%d-%m-%Y", "%m-%d-%Y", "%Y-%d-%m", + "%y-%m-%d", "%d-%m-%y", "%m-%d-%y", "%y-%d-%m", + "%Y/%m/%d", "%d/%m/%Y", "%m/%d/%Y", "%Y/%d/%m", + "%y/%m/%d", "%d/%m/%y", "%m/%d/%y", "%y/%d/%m") for (i in 1:nrow(S)) { # loop through rows in sleeplog ID = S[i,colid] if (ID %in% startdates$ID == TRUE) { # matching ID in acc data, if not ignore ID @@ -211,12 +216,14 @@ g.loadlog = function(loglocation = c(), coln1 = c(), colid = c(), startdate_acc = as.Date(startdates$startdate[which(startdates$ID == ID)], tz = desiredtz) startdate_sleeplog = S[i, datecols[1]] Sdates_correct = c() - # Detect data format in sleeplog: - for (dateformat in c("%Y-%m-%d", "%d-%m-%Y", "%m-%d-%Y", "%Y-%d-%m", - "%y-%m-%d", "%d-%m-%y", "%m-%d-%y", "%y-%d-%m", - "%Y/%m/%d", "%d/%m/%Y", "%m/%d/%Y", "%Y/%d/%m", - "%y/%m/%d", "%d/%m/%y", "%m/%d/%y", "%y/%d/%m")) { - startdate_sleeplog_tmp = as.Date(startdate_sleeplog, format = dateformat, tz = desiredtz) + if (dateformat_found == TRUE && dateformats_to_consider[1] != dateformat_correct) { + # If found then first try that before trying anything else + dateformats_to_consider = unique(c(dateformat_correct, dateformats_to_consider)) + } + # Detect date format in sleeplog: + for (dateformat in dateformats_to_consider) { + startdate_sleeplog_tmp = as.Date(startdate_sleeplog[which(startdate_sleeplog != "")], format = dateformat, tz = desiredtz) + if (is.null(startdate_sleeplog_tmp)) next Sdates = as.Date(as.character(S[i,datecols]), format = dateformat, tz = desiredtz) if (length(which(diff(which(is.na(Sdates))) > 1)) > 0) { stop(paste0("\nSleeplog for ID: ", ID, " has missing date(s)"), call. = FALSE) @@ -233,6 +240,17 @@ g.loadlog = function(loglocation = c(), coln1 = c(), colid = c(), } } } + if (is.null(Sdates_correct)) { + # skip this row because it is empty + warning(paste0("\nSkip sleeplow row for ID: ", ID, + " because it has no date(s)"), call. = FALSE) + next + } + if (deltadate > 300) { + warning(paste0("For ID ", ID, " the sleeplog start date is more than 300 days separated ", + "from the dates in the accelerometer recording, this may indicate a ", + "problem with date formats or their recognition, please check."), call. = FALSE) + } if (startdates$startAtMidnight[which(startdates$ID == ID)] == TRUE) { # If the first day in the advanced sleeplog is 28/11 # and the recording starts at midnight 27/11 00:00:00 From 5b32ee20a7a1b0ceacecc8a108f826b971411e69 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 17 Jan 2025 13:52:13 +0100 Subject: [PATCH 4/9] reload activity diary if file has changed --- R/g.conv.actlog.R | 2 +- R/g.part2.R | 16 ++-------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/R/g.conv.actlog.R b/R/g.conv.actlog.R index 982a8c89a..53b10d33d 100644 --- a/R/g.conv.actlog.R +++ b/R/g.conv.actlog.R @@ -4,7 +4,7 @@ g.conv.actlog = function(qwindow, qwindow_dateformat="%d-%m-%Y", epochSize = 5) # local functions: time2numeric = function(x) { x = unlist(x) - x = x[grep(pattern = "impute|uncertain", x = names(x), invert = TRUE)] + x = x[grep(pattern = "impute|imputa|uncertain", x = names(x), invert = TRUE)] c2t = function(x2) { tmp = as.numeric(unlist(strsplit(as.character(x2),":"))) if (length(tmp) == 2) hourinday = tmp[1] + (tmp[2]/60) diff --git a/R/g.part2.R b/R/g.part2.R index 61e41e0b9..3acff76b8 100644 --- a/R/g.part2.R +++ b/R/g.part2.R @@ -29,15 +29,8 @@ g.part2 = function(datadir = c(), metadatadir = c(), f0 = c(), f1 = c(), use_qwindow_as_diary = FALSE } tmp_activityDiary_file = paste0(metadatadir, "/activityDiary.RData") - convertActivityLog = TRUE - if (file.exists(tmp_activityDiary_file)) { - days_since_created = as.numeric(difftime(time2 = as.Date(Sys.time()), - time1 = as.Date(file.info(tmp_activityDiary_file)$ctime), units = "days")) - if (days_since_created < 90) { # file created in the last 90 days - convertActivityLog = FALSE - } - } - if (convertActivityLog) { + if (file.exists(tmp_activityDiary_file) || + file.info(params_247[["qwindow"]])$ctime >= file.info(tmp_activityDiary_file)$ctime) { if (verbose == TRUE) cat("\nConverting activity diary...") # This will be an object with numeric qwindow values for all individuals and days params_247[["qwindow"]] = g.conv.actlog(params_247[["qwindow"]], @@ -46,11 +39,6 @@ g.part2 = function(datadir = c(), metadatadir = c(), f0 = c(), f1 = c(), qwindow = params_247[["qwindow"]] save(qwindow, file = tmp_activityDiary_file) } else { - if (verbose == TRUE) { - cat(paste0("\nReloading previously converted activity diary file, ", - "if you want to re-convert the diary then delete file ", - "'activityDiary.RData' first before re-running GGIR.")) - } load(tmp_activityDiary_file) params_247[["qwindow"]] = qwindow } From 9573752be66ebb3ddbc8a76e7d47fbb8bf0e2229 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 17 Jan 2025 13:54:43 +0100 Subject: [PATCH 5/9] compare specific time instead of date when deciding whether to reload the sleeplog --- R/g.part4.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/g.part4.R b/R/g.part4.R index 32542b19b..1cc38143a 100644 --- a/R/g.part4.R +++ b/R/g.part4.R @@ -36,7 +36,7 @@ g.part4 = function(datadir = c(), metadatadir = c(), f0 = f0, f1 = f1, # only re-process sleeplog if sleeplog.RData does not exist or if sleeplog # is from a date equal to or after sleeplog.RData if (!file.exists(sleeplogRDataFile) || - as.Date(file.info(params_sleep[["loglocation"]])$ctime) >= as.Date(file.info(sleeplogRDataFile)$ctime)) { + file.info(params_sleep[["loglocation"]])$ctime >= file.info(sleeplogRDataFile)$ctime) { logs_diaries = g.loadlog(params_sleep[["loglocation"]], coln1 = params_sleep[["coln1"]], colid = params_sleep[["colid"]], From dd7e81964464aca5773e0f457df6306e545370fd Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 17 Jan 2025 14:25:48 +0100 Subject: [PATCH 6/9] correct previous commit where check on activityDiary existence was introduce --- R/g.part2.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/R/g.part2.R b/R/g.part2.R index 3acff76b8..17a67db01 100644 --- a/R/g.part2.R +++ b/R/g.part2.R @@ -29,8 +29,9 @@ g.part2 = function(datadir = c(), metadatadir = c(), f0 = c(), f1 = c(), use_qwindow_as_diary = FALSE } tmp_activityDiary_file = paste0(metadatadir, "/activityDiary.RData") - if (file.exists(tmp_activityDiary_file) || - file.info(params_247[["qwindow"]])$ctime >= file.info(tmp_activityDiary_file)$ctime) { + + if (!file.exists(tmp_activityDiary_file) || (file.exists(tmp_activityDiary_file) && + file.info(params_247[["qwindow"]])$ctime >= file.info(tmp_activityDiary_file)$ctime)) { if (verbose == TRUE) cat("\nConverting activity diary...") # This will be an object with numeric qwindow values for all individuals and days params_247[["qwindow"]] = g.conv.actlog(params_247[["qwindow"]], From cb2aab8ea06537c4a4ff552f422421957e836dce Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 17 Jan 2025 14:45:50 +0100 Subject: [PATCH 7/9] Update NEWS.md --- NEWS.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/NEWS.md b/NEWS.md index 046d5f3eb..63f32a289 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,13 @@ +# CHANGES IN GGIR VERSION 3.1-?? + +- Part 2: Correct identification of number of files for part 2 generation + +- Part 4: Further improvements to handling dates in sleeplog as a follow-up to work on #1243 + +- Part 2 and 4: Both activity diary and sleep diary are now always reloaded if GGIR's copy of it (in .RData) is older than diary itself. + +- Part 2: Make sure event diary is save to csv with intended sep and dec rather than default. + # CHANGES IN GGIR VERSION 3.1-10 - Part 4: From 8f7fdc2b748f63697a179cf8cb26a507ff7d496e Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 17 Jan 2025 14:50:37 +0100 Subject: [PATCH 8/9] fix typo in warning message --- R/g.loadlog.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/g.loadlog.R b/R/g.loadlog.R index a1b91e641..587bae2f3 100644 --- a/R/g.loadlog.R +++ b/R/g.loadlog.R @@ -244,7 +244,7 @@ g.loadlog = function(loglocation = c(), coln1 = c(), colid = c(), } if (is.null(Sdates_correct)) { # skip this row because it is empty - warning(paste0("\nSkip sleeplow row for ID: ", ID, + warning(paste0("\nSkipping sleeplog row for ID ", ID, " because it has no date(s)"), call. = FALSE) next } From 2ca211369a97374164d557c7bbbe302b1e3ac7da Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Fri, 17 Jan 2025 15:32:32 +0100 Subject: [PATCH 9/9] Update NEWS.md --- NEWS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 63f32a289..27630b848 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,12 +1,12 @@ # CHANGES IN GGIR VERSION 3.1-?? -- Part 2: Correct identification of number of files for part 2 generation +- Part 2: Fix bug in determining the number of files to be included in the part 2 report. - Part 4: Further improvements to handling dates in sleeplog as a follow-up to work on #1243 -- Part 2 and 4: Both activity diary and sleep diary are now always reloaded if GGIR's copy of it (in .RData) is older than diary itself. +- Part 2 and 4: Both activity diary and sleep diary are now always reloaded if the derived copy of it (in .RData) is older than the diary itself. -- Part 2: Make sure event diary is save to csv with intended sep and dec rather than default. +- Part 2: Make sure event diary is saved to csv with intended sep and dec arguments rather than default. # CHANGES IN GGIR VERSION 3.1-10