Small fixes related to loading diaries and storing part 2 output

NEWS.md

@@ -1,3 +1,13 @@
+# CHANGES IN GGIR VERSION 3.1-??
+
+- Part 2: Fix bug in determining the number of files to be included in the part 2 report.
+
+- Part 4: Further improvements to handling dates in sleeplog as a follow-up to work on #1243
+
+- Part 2 and 4: Both activity diary and sleep diary are now always reloaded if the derived copy of it (in .RData) is older than the diary itself.
+
+- Part 2: Make sure event diary is saved to csv with intended sep and dec arguments rather than default.
+
 # CHANGES IN GGIR VERSION 3.1-10
 
 - Part 4:

R/GGIR.R

@@ -352,13 +352,11 @@ GGIR = function(mode = 1:5, datadir = c(), outputdir = c(),
   # -----
   # check a few basic assumptions before continuing
   if (length(which(do.report == 2)) > 0) {
-    N.files.ms2.out = length(dir(paste0(metadatadir, "/meta/ms2.out")))
-    if (N.files.ms2.out > 0) {
+    N.files.basic = length(dir(paste0(metadatadir, "/meta/basic")))
+    if (N.files.basic > 0) {
       if (verbose == TRUE) print_console_header("Report part 2")
-      # if (N.files.ms2.out < f0) f0 = 1
-      # if (N.files.ms2.out < f1) f1 = N.files.ms2.out
       if (length(f0) == 0) f0 = 1
-      if (f1 == 0) f1 = N.files.ms2.out
+      if (f1 == 0) f1 = N.files.basic
       if (length(params_247[["qwindow"]]) > 2 |
           (is.character(params_247[["qwindow"]]) && length(grep(pattern = "onlyfilter|filteronly", x = params_247[["qwindow"]])) == 0) |
           (length(params_247[["qwindow"]]) == 2 & !all(c(0, 24) %in% params_247[["qwindow"]]))) {

R/g.conv.actlog.R

@@ -4,7 +4,7 @@ g.conv.actlog = function(qwindow, qwindow_dateformat="%d-%m-%Y", epochSize = 5)
   # local functions:
   time2numeric = function(x) {
     x = unlist(x)
-    x = x[grep(pattern = "impute|uncertain", x = names(x), invert = TRUE)]
+    x = x[grep(pattern = "impute|imputa|uncertain", x = names(x), invert = TRUE)]
     c2t = function(x2) {
       tmp = as.numeric(unlist(strsplit(as.character(x2),":")))
       if (length(tmp) == 2) hourinday = tmp[1] + (tmp[2]/60)

R/g.loadlog.R

@@ -205,24 +205,25 @@ g.loadlog = function(loglocation = c(), coln1 = c(), colid = c(),
       napcnt = nwcnt = iccnt = 1
       IDcouldNotBeMatched = TRUE
       dateformat_found = FALSE
+      dateformats_to_consider = c("%Y-%m-%d", "%d-%m-%Y", "%m-%d-%Y", "%Y-%d-%m",
+                                  "%y-%m-%d", "%d-%m-%y", "%m-%d-%y", "%y-%d-%m",
+                                  "%Y/%m/%d", "%d/%m/%Y", "%m/%d/%Y", "%Y/%d/%m",
+                                  "%y/%m/%d", "%d/%m/%y", "%m/%d/%y", "%y/%d/%m")
       for (i in 1:nrow(S)) { # loop through rows in sleeplog
         ID = S[i,colid]
         if (ID %in% startdates$ID == TRUE) { # matching ID in acc data, if not ignore ID
           IDcouldNotBeMatched = FALSE
           startdate_acc = as.Date(startdates$startdate[which(startdates$ID == ID)], tz = desiredtz)
           startdate_sleeplog = as.character(S[i, datecols[1:pmin(length(datecols), 5)]])
           Sdates_correct = c()
-          dateformats_to_consider = c("%Y-%m-%d", "%d-%m-%Y", "%m-%d-%Y", "%Y-%d-%m",
-                                      "%y-%m-%d", "%d-%m-%y", "%m-%d-%y", "%y-%d-%m",
-                                      "%Y/%m/%d", "%d/%m/%Y", "%m/%d/%Y", "%Y/%d/%m",
-                                      "%y/%m/%d", "%d/%m/%y", "%m/%d/%y", "%y/%d/%m")
           if (dateformat_found == TRUE && dateformats_to_consider[1] != dateformat_correct) {
             # If found then first try that before trying anything else
             dateformats_to_consider = unique(c(dateformat_correct, dateformats_to_consider))
           }
-          # Detect data format in sleeplog:
+          # Detect date format in sleeplog:
           for (dateformat in dateformats_to_consider) {
-            startdate_sleeplog_tmp = as.Date(startdate_sleeplog, format = dateformat, tz = desiredtz)
+            startdate_sleeplog_tmp = as.Date(startdate_sleeplog[which(startdate_sleeplog != "")], format = dateformat, tz = desiredtz)
+            if (is.null(startdate_sleeplog_tmp)) next
             Sdates = as.Date(as.character(S[i,datecols]), format = dateformat, tz = desiredtz)
             if (length(which(diff(which(is.na(Sdates))) > 1)) > 0) {
               stop(paste0("\nSleeplog for ID: ", ID, " has missing date(s)"), call. = FALSE)
@@ -241,6 +242,12 @@ g.loadlog = function(loglocation = c(), coln1 = c(), colid = c(),
               }
             }
           }
+          if (is.null(Sdates_correct)) {
+            # skip this row because it is empty
+            warning(paste0("\nSkipping sleeplog row for ID ", ID,
+                           " because it has no date(s)"), call. = FALSE)
+            next
+          }
           if (deltadate > 300) {
             warning(paste0("For ID ", ID, " the sleeplog start date is more than 300 days separated ",
                            "from the dates in the accelerometer recording, this may indicate a ",

R/g.part2.R

@@ -29,15 +29,9 @@ g.part2 = function(datadir = c(), metadatadir = c(), f0 = c(), f1 = c(),
       use_qwindow_as_diary = FALSE 
     }
     tmp_activityDiary_file = paste0(metadatadir, "/activityDiary.RData")
-    convertActivityLog = TRUE
-    if (file.exists(tmp_activityDiary_file)) {
-      days_since_created = as.numeric(difftime(time2 = as.Date(Sys.time()),
-                                              time1 =  as.Date(file.info(tmp_activityDiary_file)$ctime), units = "days"))
-      if (days_since_created < 90) { # file created in the last 90 days
-        convertActivityLog = FALSE
-      }
-    }
-    if (convertActivityLog) {
+
+    if (!file.exists(tmp_activityDiary_file) || (file.exists(tmp_activityDiary_file) && 
+        file.info(params_247[["qwindow"]])$ctime >= file.info(tmp_activityDiary_file)$ctime)) {
       if (verbose == TRUE) cat("\nConverting activity diary...")
       # This will be an object with numeric qwindow values for all individuals and days
       params_247[["qwindow"]] = g.conv.actlog(params_247[["qwindow"]],
@@ -46,11 +40,6 @@ g.part2 = function(datadir = c(), metadatadir = c(), f0 = c(), f1 = c(),
       qwindow = params_247[["qwindow"]]
       save(qwindow, file = tmp_activityDiary_file)
     } else {
-      if (verbose == TRUE) {
-        cat(paste0("\nReloading previously converted activity diary file, ",
-                   "if you want to re-convert the diary then delete file ",
-                   "'activityDiary.RData' first before re-running GGIR."))
-      }
       load(tmp_activityDiary_file)
       params_247[["qwindow"]] = qwindow
     }

R/g.part4.R

@@ -36,7 +36,7 @@ g.part4 = function(datadir = c(), metadatadir = c(), f0 = f0, f1 = f1,
     # only re-process sleeplog if sleeplog.RData does not exist or if sleeplog
     # is from a date equal to or after sleeplog.RData
     if (!file.exists(sleeplogRDataFile) || 
-        as.Date(file.info(params_sleep[["loglocation"]])$ctime) >= as.Date(file.info(sleeplogRDataFile)$ctime)) {
+        file.info(params_sleep[["loglocation"]])$ctime >= file.info(sleeplogRDataFile)$ctime) {
       logs_diaries = g.loadlog(params_sleep[["loglocation"]], 
                                coln1 = params_sleep[["coln1"]],
                                colid = params_sleep[["colid"]],

R/g.report.part2.R

@@ -266,7 +266,8 @@ g.report.part2 = function(metadatadir = c(), f0 = c(), f1 = c(), maxdur = 0,
       #-----------------------------------------------------------------------
       data.table::fwrite(x = dayEVENTSUMMARY_clean,
                          file = paste0(metadatadir, "/results/part2_day", eventName, "summary.csv"),
-                         row.names = F, na = "")
+                         row.names = F, na = "", sep = params_output[["sep_reports"]],
+                         dec = params_output[["dec_reports"]])
     }
     # split SUMMARY in two files and reoder EventVariable names if they exist
     s_names = names(SUMMARY)
@@ -281,7 +282,8 @@ g.report.part2 = function(metadatadir = c(), f0 = c(), f1 = c(), maxdur = 0,
       EVENTSUMMARY_clean = tidyup_df(EVENTSUMMARY)
       data.table::fwrite(x = EVENTSUMMARY_clean,
                          file =  paste0(metadatadir, "/results/part2_", eventName, "summary.csv"),
-                         row.names = F, na = "")
+                         row.names = F, na = "", sep = params_output[["sep_reports"]],
+                         dec = params_output[["dec_reports"]])
     }
     #-----------------------------
     # tidy up data.frames