{"action":"created","comment":{"url":"https://api.github.com/repos/PecanProject/pecan/pulls/comments/137009127","pull_request_review_id":60623031,"id":137009127,"diff_hunk":"@@ -0,0 +1,298 @@\n+##' Align meteorology datasets for debiasing\n+# -----------------------------------\n+# Description\n+# -----------------------------------\n+##'\n+##' @title align.met\n+##' @family debias - Debias & Align Meteorology Datasets into continuous time series\n+##' @author Christy Rollinson\n+##' @description This script aligns meteorology datasets in at temporal resolution for debiasing & \n+##'              temporal downscaling.  \n+##'              Note: The output here is stored in memory!  \n+##'              Note: can probably at borrow from or adapt align_data.R in Benchmarking module, but \n+##'              it's too much of a black box at the moment.\n+# -----------------------------------\n+# Parameters\n+# -----------------------------------\n+##' @param train.path - path to the dataset to be used to downscale the data\n+##' @param source.path - data to be bias-corrected aligned with training data (from align.met)\n+##' @param yrs.train - (optional) specify a specific years to be loaded for the training data; \n+##'                    prevents needing to load the entire dataset.  If NULL, all available years \n+##'                    will be loaded. If not null, should be a vector of numbers (so you can skip\n+##'                    problematic years)\n+##' @param n.ens  - number of ensemble members to generate and save\n+##' @param pair.mems - logical stating whether ensemble members should be paired\n+##' @param seed - specify seed so that random draws can be reproduced\n+##' @param verbose\n+##' @export\n+# -----------------------------------\n+# Workflow\n+# -----------------------------------\n+# 1. Read in & format the training data\n+#    1.1. Determine if training data is a single series or ensemble\n+#       - note: assumes training data is at the temporal resolution you want to work with\n+#    1.2. If not already, coerce training data into the size of the desired output ensemble\n+# 2. Read in & format the source data to match the temporal resolution of training data\n+#    - Note: for now this is only going to work with a single time series & not an ensemble of source data\n+#    - Note: end dimensions should match that of the training data\n+# 3. export data (stored in memory) for input into the debiasing or temporal downscaling workflow\n+# \n+# Returns a list called met.out with 2 levels that are matched in temporal resolution & number of ensembles\n+# List Layers\n+#  1. dat.train\n+#  2. dat.source\n+# Sublist Layers: time, met variables\n+# -----------------------------------\n+# Notes\n+# -----------------------------------\n+# 1. This assumes that both the trian and source data are in *at least* daily resolution\n+# -----------------------------------\n+\n+#----------------------------------------------------------------------\n+# Begin Function\n+#----------------------------------------------------------------------\n+align.met <- function(train.path, source.path, yrs.train=NULL, n.ens=NULL, pair.mems = TRUE, seed=Sys.Date(), verbose = FALSE) {\n+  # Load required libraries\n+  library(ncdf4)\n+  library(lubridate)\n+  \n+  met.out <- list() # where the aligned data will be stored\n+\n+  # ---------------\n+  # 1. Read in & organize training data\n+  # ---------------\n+  met.out[[\"dat.train\"]] <- list()\n+  # 1.a. Determine if we have an ensemble in the training path or if it's a single time series\n+  if(length(dir(train.path, \".nc\"))>0){ # we have a single time series\n+    n.trn = 1 # Ignore how many input ensemble members we asked for, we only actually have 1 here\n+    \n+    files.train <- dir(train.path, \".nc\")\n+    \n+    yrs.file <- strsplit(files.train, \"[.]\")\n+    yrs.file <- matrix(unlist(yrs.file), ncol=length(yrs.file[[1]]), byrow=T)\n+    yrs.file <- as.numeric(yrs.file[,ncol(yrs.file)-1]) # Assumes year is always last thing before the file extension\n+    \n+    if(!is.null(yrs.train)){\n+      files.train <- files.train[which(yrs.file %in% yrs.train)]\n+      yrs.file <- yrs.file[which(yrs.file %in% yrs.train)]\n+    }\n+    \n+    # Loop through the .nc files putting everything into a list\n+    print(\"Processing Training Data\")\n+    pb <- txtProgressBar(min=0, max=length(files.train), style=3)\n+    for(i in 1:length(files.train)){\n+      yr.now <- yrs.file[i]\n+      \n+      ncT <- nc_open(file.path(train.path, files.train[i]))\n+      \n+      # Set up the time data frame to help index\n+      nday <- ifelse(leap_year(yr.now), 366, 365)\n+      ntime <- length(ncT$dim$time$vals)\n+      step.day <- nday/ntime\n+      step.hr  <- step.day*24\n+      stamps.hr <- seq(step.hr/2, by=step.hr, length.out=1/step.day) # Time stamps centered on period\n+      \n+      # Create a data frame with all the important time info\n+      # center the hour step\n+      df.time <- data.frame(Year=yr.now, DOY=rep(1:nday, each=1/step.day), Hour=rep(stamps.hr, ntime))\n+      df.time$Date <- strptime(paste(df.time$Year, df.time$DOY, df.time$Hour, sep=\"-\"), format=(\"%Y-%j-%H\"), tz=\"UTC\")","path":"modules/data.atmosphere/R/align_met.R","position":98,"original_position":98,"commit_id":"079da9418415a4f942feffe12349a138b908f1e0","original_commit_id":"079da9418415a4f942feffe12349a138b908f1e0","user":{"login":"crollinson","id":7927379,"avatar_url":"https://avatars1.githubusercontent.com/u/7927379?v=4","gravatar_id":"","url":"https://api.github.com/users/crollinson","html_url":"https://github.com/crollinson","followers_url":"https://api.github.com/users/crollinson/followers","following_url":"https://api.github.com/users/crollinson/following{/other_user}","gists_url":"https://api.github.com/users/crollinson/gists{/gist_id}","starred_url":"https://api.github.com/users/crollinson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/crollinson/subscriptions","organizations_url":"https://api.github.com/users/crollinson/orgs","repos_url":"https://api.github.com/users/crollinson/repos","events_url":"https://api.github.com/users/crollinson/events{/privacy}","received_events_url":"https://api.github.com/users/crollinson/received_events","type":"User","site_admin":false},"body":"there may very well be a better way to do it, but to get this working, I'll ask that someone else figure it out while I move on to debugging the temporal downscaling.","created_at":"2017-09-05T14:46:52Z","updated_at":"2017-09-05T14:46:52Z","html_url":"https://github.com/PecanProject/pecan/pull/1615#discussion_r137009127","pull_request_url":"https://api.github.com/repos/PecanProject/pecan/pulls/1615","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/PecanProject/pecan/pulls/comments/137009127"},"html":{"href":"https://github.com/PecanProject/pecan/pull/1615#discussion_r137009127"},"pull_request":{"href":"https://api.github.com/repos/PecanProject/pecan/pulls/1615"}},"in_reply_to_id":136325059},"pull_request":{"url":"https://api.github.com/repos/PecanProject/pecan/pulls/1615","id":138530173,"html_url":"https://github.com/PecanProject/pecan/pull/1615","diff_url":"https://github.com/PecanProject/pecan/pull/1615.diff","patch_url":"https://github.com/PecanProject/pecan/pull/1615.patch","issue_url":"https://api.github.com/repos/PecanProject/pecan/issues/1615","number":1615,"state":"open","locked":false,"title":"Debias & Splice Met products","user":{"login":"crollinson","id":7927379,"avatar_url":"https://avatars1.githubusercontent.com/u/7927379?v=4","gravatar_id":"","url":"https://api.github.com/users/crollinson","html_url":"https://github.com/crollinson","followers_url":"https://api.github.com/users/crollinson/followers","following_url":"https://api.github.com/users/crollinson/following{/other_user}","gists_url":"https://api.github.com/users/crollinson/gists{/gist_id}","starred_url":"https://api.github.com/users/crollinson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/crollinson/subscriptions","organizations_url":"https://api.github.com/users/crollinson/orgs","repos_url":"https://api.github.com/users/crollinson/repos","events_url":"https://api.github.com/users/crollinson/events{/privacy}","received_events_url":"https://api.github.com/users/crollinson/received_events","type":"User","site_admin":false},"body":"<!--- Provide a general summary of your changes in the Title above -->\r\nPart of adding and updating new functionality in pecan to spatially and temporally downscale met data products.  In the PalEON met workflow, a debiasing step precedes the temporal downscaling that @jsimkins2 worked on. (Debugging and functionality improvements to that will come soon.). \r\n\r\nThe PalEON workflow for generating long met ensembles that combine multiple products can be found here: https://github.com/PalEON-Project/modeling_met_ensemble (Yes, I know it's messy & the documentation is not that great.  I'll update it once the PEcAn end of things is ironed out.)\r\n\r\n## Description\r\nThis pull requests adds two main functions for de-biasing (can also be seen as spatial downscaling) of met products: \r\n1. align.met -- takes two data products and matches their temporal resolution so that it can be then fed into downscaling code\r\n2. debias.met.regression -- uses a training dataset such as LDAS or Ameriflux to sequentially adjust the mean climatology and seasonal cycle of multiple met variables, leveraging covariance among parameters where appropriate.\r\n\r\n## Motivation and Context\r\nThese functions were added to allow coarse resolution data products to be adjusted to better match local conditions and/or allow multiple products to be combined to generate extended met drivers.  These were originally generated to allow the PalEON project to combine CMIP5 p1000 and historical runs, with modern reanalysis products such as CRUNCEP, NARR, and NLDAS.\r\n\r\n## Types of changes\r\n<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->\r\n- [ ] Bug fix (non-breaking change which fixes an issue) <!-- please add issue number -->\r\n- [ X] New feature (non-breaking change which adds functionality)\r\n- [ ] Breaking change (fix or feature that would cause existing functionality to change)\r\n\r\n## Checklist:\r\n<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->\r\n<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->\r\n- [ ] My change requires a change to the documentation.\r\n- [ ] I have updated the CHANGELOG.md.\r\n- [ ] I have updated the documentation accordingly.\r\n- [X] I have read the **CONTRIBUTING** document.\r\n- [ ] I have added tests to cover my changes.\r\n- [ ] All new and existing tests passed.\r\n\r\n<!--this template is from https://www.talater.com/open-source-templates/#/page/99--> ","created_at":"2017-08-30T21:15:51Z","updated_at":"2017-09-05T14:46:52Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3eec79c2db18be9ed46794e6482c9f3050e09f05","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/PecanProject/pecan/pulls/1615/commits","review_comments_url":"https://api.github.com/repos/PecanProject/pecan/pulls/1615/comments","review_comment_url":"https://api.github.com/repos/PecanProject/pecan/pulls/comments{/number}","comments_url":"https://api.github.com/repos/PecanProject/pecan/issues/1615/comments","statuses_url":"https://api.github.com/repos/PecanProject/pecan/statuses/079da9418415a4f942feffe12349a138b908f1e0","head":{"label":"crollinson:met_debias","ref":"met_debias","sha":"079da9418415a4f942feffe12349a138b908f1e0","user":{"login":"crollinson","id":7927379,"avatar_url":"https://avatars1.githubusercontent.com/u/7927379?v=4","gravatar_id":"","url":"https://api.github.com/users/crollinson","html_url":"https://github.com/crollinson","followers_url":"https://api.github.com/users/crollinson/followers","following_url":"https://api.github.com/users/crollinson/following{/other_user}","gists_url":"https://api.github.com/users/crollinson/gists{/gist_id}","starred_url":"https://api.github.com/users/crollinson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/crollinson/subscriptions","organizations_url":"https://api.github.com/users/crollinson/orgs","repos_url":"https://api.github.com/users/crollinson/repos","events_url":"https://api.github.com/users/crollinson/events{/privacy}","received_events_url":"https://api.github.com/users/crollinson/received_events","type":"User","site_admin":false},"repo":{"id":23364811,"name":"pecan","full_name":"crollinson/pecan","owner":{"login":"crollinson","id":7927379,"avatar_url":"https://avatars1.githubusercontent.com/u/7927379?v=4","gravatar_id":"","url":"https://api.github.com/users/crollinson","html_url":"https://github.com/crollinson","followers_url":"https://api.github.com/users/crollinson/followers","following_url":"https://api.github.com/users/crollinson/following{/other_user}","gists_url":"https://api.github.com/users/crollinson/gists{/gist_id}","starred_url":"https://api.github.com/users/crollinson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/crollinson/subscriptions","organizations_url":"https://api.github.com/users/crollinson/orgs","repos_url":"https://api.github.com/users/crollinson/repos","events_url":"https://api.github.com/users/crollinson/events{/privacy}","received_events_url":"https://api.github.com/users/crollinson/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/crollinson/pecan","description":"The Predictive Ecosystem Analyzer (PEcAn) is an integrated ecological bioinformatics toolbox.","fork":true,"url":"https://api.github.com/repos/crollinson/pecan","forks_url":"https://api.github.com/repos/crollinson/pecan/forks","keys_url":"https://api.github.com/repos/crollinson/pecan/keys{/key_id}","collaborators_url":"https://api.github.com/repos/crollinson/pecan/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/crollinson/pecan/teams","hooks_url":"https://api.github.com/repos/crollinson/pecan/hooks","issue_events_url":"https://api.github.com/repos/crollinson/pecan/issues/events{/number}","events_url":"https://api.github.com/repos/crollinson/pecan/events","assignees_url":"https://api.github.com/repos/crollinson/pecan/assignees{/user}","branches_url":"https://api.github.com/repos/crollinson/pecan/branches{/branch}","tags_url":"https://api.github.com/repos/crollinson/pecan/tags","blobs_url":"https://api.github.com/repos/crollinson/pecan/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/crollinson/pecan/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/crollinson/pecan/git/refs{/sha}","trees_url":"https://api.github.com/repos/crollinson/pecan/git/trees{/sha}","statuses_url":"https://api.github.com/repos/crollinson/pecan/statuses/{sha}","languages_url":"https://api.github.com/repos/crollinson/pecan/languages","stargazers_url":"https://api.github.com/repos/crollinson/pecan/stargazers","contributors_url":"https://api.github.com/repos/crollinson/pecan/contributors","subscribers_url":"https://api.github.com/repos/crollinson/pecan/subscribers","subscription_url":"https://api.github.com/repos/crollinson/pecan/subscription","commits_url":"https://api.github.com/repos/crollinson/pecan/commits{/sha}","git_commits_url":"https://api.github.com/repos/crollinson/pecan/git/commits{/sha}","comments_url":"https://api.github.com/repos/crollinson/pecan/comments{/number}","issue_comment_url":"https://api.github.com/repos/crollinson/pecan/issues/comments{/number}","contents_url":"https://api.github.com/repos/crollinson/pecan/contents/{+path}","compare_url":"https://api.github.com/repos/crollinson/pecan/compare/{base}...{head}","merges_url":"https://api.github.com/repos/crollinson/pecan/merges","archive_url":"https://api.github.com/repos/crollinson/pecan/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/crollinson/pecan/downloads","issues_url":"https://api.github.com/repos/crollinson/pecan/issues{/number}","pulls_url":"https://api.github.com/repos/crollinson/pecan/pulls{/number}","milestones_url":"https://api.github.com/repos/crollinson/pecan/milestones{/number}","notifications_url":"https://api.github.com/repos/crollinson/pecan/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/crollinson/pecan/labels{/name}","releases_url":"https://api.github.com/repos/crollinson/pecan/releases{/id}","deployments_url":"https://api.github.com/repos/crollinson/pecan/deployments","created_at":"2014-08-26T19:55:37Z","updated_at":"2017-02-08T21:34:36Z","pushed_at":"2017-08-30T21:04:37Z","git_url":"git://github.com/crollinson/pecan.git","ssh_url":"git@github.com:crollinson/pecan.git","clone_url":"https://github.com/crollinson/pecan.git","svn_url":"https://github.com/crollinson/pecan","homepage":"www.pecanproject.org","size":130628,"stargazers_count":0,"watchers_count":0,"language":"R","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"PecanProject:develop","ref":"develop","sha":"1c818092fce595778b50022111b3cd4b15dc86e2","user":{"login":"PecanProject","id":2879854,"avatar_url":"https://avatars0.githubusercontent.com/u/2879854?v=4","gravatar_id":"","url":"https://api.github.com/users/PecanProject","html_url":"https://github.com/PecanProject","followers_url":"https://api.github.com/users/PecanProject/followers","following_url":"https://api.github.com/users/PecanProject/following{/other_user}","gists_url":"https://api.github.com/users/PecanProject/gists{/gist_id}","starred_url":"https://api.github.com/users/PecanProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/PecanProject/subscriptions","organizations_url":"https://api.github.com/users/PecanProject/orgs","repos_url":"https://api.github.com/users/PecanProject/repos","events_url":"https://api.github.com/users/PecanProject/events{/privacy}","received_events_url":"https://api.github.com/users/PecanProject/received_events","type":"Organization","site_admin":false},"repo":{"id":6857384,"name":"pecan","full_name":"PecanProject/pecan","owner":{"login":"PecanProject","id":2879854,"avatar_url":"https://avatars0.githubusercontent.com/u/2879854?v=4","gravatar_id":"","url":"https://api.github.com/users/PecanProject","html_url":"https://github.com/PecanProject","followers_url":"https://api.github.com/users/PecanProject/followers","following_url":"https://api.github.com/users/PecanProject/following{/other_user}","gists_url":"https://api.github.com/users/PecanProject/gists{/gist_id}","starred_url":"https://api.github.com/users/PecanProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/PecanProject/subscriptions","organizations_url":"https://api.github.com/users/PecanProject/orgs","repos_url":"https://api.github.com/users/PecanProject/repos","events_url":"https://api.github.com/users/PecanProject/events{/privacy}","received_events_url":"https://api.github.com/users/PecanProject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/PecanProject/pecan","description":"The Predictive Ecosystem Analyzer (PEcAn) is an integrated ecological bioinformatics toolbox.","fork":false,"url":"https://api.github.com/repos/PecanProject/pecan","forks_url":"https://api.github.com/repos/PecanProject/pecan/forks","keys_url":"https://api.github.com/repos/PecanProject/pecan/keys{/key_id}","collaborators_url":"https://api.github.com/repos/PecanProject/pecan/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/PecanProject/pecan/teams","hooks_url":"https://api.github.com/repos/PecanProject/pecan/hooks","issue_events_url":"https://api.github.com/repos/PecanProject/pecan/issues/events{/number}","events_url":"https://api.github.com/repos/PecanProject/pecan/events","assignees_url":"https://api.github.com/repos/PecanProject/pecan/assignees{/user}","branches_url":"https://api.github.com/repos/PecanProject/pecan/branches{/branch}","tags_url":"https://api.github.com/repos/PecanProject/pecan/tags","blobs_url":"https://api.github.com/repos/PecanProject/pecan/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/PecanProject/pecan/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/PecanProject/pecan/git/refs{/sha}","trees_url":"https://api.github.com/repos/PecanProject/pecan/git/trees{/sha}","statuses_url":"https://api.github.com/repos/PecanProject/pecan/statuses/{sha}","languages_url":"https://api.github.com/repos/PecanProject/pecan/languages","stargazers_url":"https://api.github.com/repos/PecanProject/pecan/stargazers","contributors_url":"https://api.github.com/repos/PecanProject/pecan/contributors","subscribers_url":"https://api.github.com/repos/PecanProject/pecan/subscribers","subscription_url":"https://api.github.com/repos/PecanProject/pecan/subscription","commits_url":"https://api.github.com/repos/PecanProject/pecan/commits{/sha}","git_commits_url":"https://api.github.com/repos/PecanProject/pecan/git/commits{/sha}","comments_url":"https://api.github.com/repos/PecanProject/pecan/comments{/number}","issue_comment_url":"https://api.github.com/repos/PecanProject/pecan/issues/comments{/number}","contents_url":"https://api.github.com/repos/PecanProject/pecan/contents/{+path}","compare_url":"https://api.github.com/repos/PecanProject/pecan/compare/{base}...{head}","merges_url":"https://api.github.com/repos/PecanProject/pecan/merges","archive_url":"https://api.github.com/repos/PecanProject/pecan/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/PecanProject/pecan/downloads","issues_url":"https://api.github.com/repos/PecanProject/pecan/issues{/number}","pulls_url":"https://api.github.com/repos/PecanProject/pecan/pulls{/number}","milestones_url":"https://api.github.com/repos/PecanProject/pecan/milestones{/number}","notifications_url":"https://api.github.com/repos/PecanProject/pecan/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/PecanProject/pecan/labels{/name}","releases_url":"https://api.github.com/repos/PecanProject/pecan/releases{/id}","deployments_url":"https://api.github.com/repos/PecanProject/pecan/deployments","created_at":"2012-11-25T23:48:26Z","updated_at":"2017-07-27T08:23:52Z","pushed_at":"2017-09-05T14:21:58Z","git_url":"git://github.com/PecanProject/pecan.git","ssh_url":"git@github.com:PecanProject/pecan.git","clone_url":"https://github.com/PecanProject/pecan.git","svn_url":"https://github.com/PecanProject/pecan","homepage":"www.pecanproject.org","size":131080,"stargazers_count":65,"watchers_count":65,"language":"R","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":100,"mirror_url":null,"open_issues_count":286,"forks":100,"open_issues":286,"watchers":65,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/PecanProject/pecan/pulls/1615"},"html":{"href":"https://github.com/PecanProject/pecan/pull/1615"},"issue":{"href":"https://api.github.com/repos/PecanProject/pecan/issues/1615"},"comments":{"href":"https://api.github.com/repos/PecanProject/pecan/issues/1615/comments"},"review_comments":{"href":"https://api.github.com/repos/PecanProject/pecan/pulls/1615/comments"},"review_comment":{"href":"https://api.github.com/repos/PecanProject/pecan/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/PecanProject/pecan/pulls/1615/commits"},"statuses":{"href":"https://api.github.com/repos/PecanProject/pecan/statuses/079da9418415a4f942feffe12349a138b908f1e0"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"crollinson"}}