Merge pull request #261 from ljwoodley/use_run_etl

Update cron files to use run_etl.R

cron/billable_candidates

@@ -1,2 +1,2 @@
 # Send billable_candidates report each monday morning
-3 07 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript report/billable_candidates.R
+3 07 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R report/billable_candidates.R

cron/cleanup_bad_email_addresses

@@ -1,2 +1,2 @@
 # Clean bad email addresses every tuesday at 10 am
-0 10 * * 2 root /usr/bin/docker run --rm --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/cleanup_bad_email_addresses.R
+0 10 * * 2 root /usr/bin/docker run --rm --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/cleanup_bad_email_addresses.R

cron/cleanup_bad_project_pi_email_addresses

@@ -1,2 +1,2 @@
 # Clean bad PI email addresses every Tuesday at 10:03 am
-3 10 * * 2 root /usr/bin/docker run --rm --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/cleanup_bad_project_pi_email_addresses.R
+3 10 * * 2 root /usr/bin/docker run --rm --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/cleanup_bad_project_pi_email_addresses.R

cron/cleanup_project_ownership_table

@@ -1,2 +1,2 @@
 # This script will run daily at 12:47 AM
-47 0 * * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/cleanup_project_ownership_table.R
+47 0 * * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/cleanup_project_ownership_table.R

cron/create_and_send_new_invoice_line_items

@@ -1,2 +1,2 @@
 # send invoice line items on the 5th of the month at 8:05 a.m.
-5 8 5 * * root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/create_and_send_new_invoice_line_items.R
+5 8 5 * * root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/create_and_send_new_invoice_line_items.R

cron/delete_abandoned_projects

@@ -1,2 +1,2 @@
 # delete abandoned projects on the 28th of the month at 14:03.
-3 14 28 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/delete_abandoned_projects.R
+3 14 28 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/delete_abandoned_projects.R

cron/remind_owners_to_review_ownership

@@ -1,2 +1,2 @@
 # remind_owners_to_review_ownership.R quarterly on the first of every month at 10:05 a.m.
-13 8 10 2,5,8,11 * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript report/remind_owners_to_review_ownership.R
+13 8 10 2,5,8,11 * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R report/remind_owners_to_review_ownership.R

cron/request_correction_of_bad_ownership_data

@@ -1,2 +1,2 @@
 # Report bad ownership on the first of every month at 10:05 a.m.
-5 10 1 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript report/request_correction_of_bad_ownership_data.R
+5 10 1 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R report/request_correction_of_bad_ownership_data.R

cron/sequester_orphans

@@ -1,2 +1,2 @@
-# sequester orphans on the 1st, 23rd, and 28th of the month at 8:02 a.m. 
-2 8 1,23,28 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/sequester_orphans.R
+# sequester orphans on the 1st, 23rd, and 28th of the month at 8:02 a.m.
+2 8 1,23,28 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/sequester_orphans.R

cron/sequester_unpaid_projects

@@ -1,5 +1,5 @@
-# sequester unpaid projects every Tuesday at 8:04 a.m. 
+# sequester unpaid projects every Tuesday at 8:04 a.m.
 # Note: Date tests internal to the script will quietly exit the code
 #       on Days 8-31. This will force the sequestration events to happen
 #       only on the *first* Tuesday of the month.
-4 8 * * 2 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/sequester_unpaid_projects.R
+4 8 * * 2 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/sequester_unpaid_projects.R

cron/update_free_support_time_remaining

@@ -1,2 +1,2 @@
 # Update update_free_support_time_remaining weekdays daily every 2 hours from 7am - 7pm
-7 7-19/2 * * 1-5 root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/update_free_support_time_remaining.R
+7 7-19/2 * * 1-5 root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/update_free_support_time_remaining.R

cron/update_probono_service_request_records

@@ -1,2 +1,2 @@
 # Update probono records weekdays daily every 2 hours from 7am - 7pm
-3 7-19/2 * * 1-5 root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/update_probono_service_request_records.R
+3 7-19/2 * * 1-5 root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/update_probono_service_request_records.R

cron/update_project_billable_attribute

@@ -1,2 +1,2 @@
 # Update project billable attribute every day at 1:05 AM
-5 1 * * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/update_project_billable_attribute.R
+5 1 * * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/update_project_billable_attribute.R

cron/warn_owners_of_impending_bill

@@ -1,2 +1,2 @@
 # Warn project owners of impending bills every 1st and 23rd of the month at 10:03 a.m.
-3 10 1,23 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript report/warn_owners_of_impending_bill.R
+3 10 1,23 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R report/warn_owners_of_impending_bill.R

cron/write_uf_fiscal_orgs_to_org_hierarchies

@@ -1,2 +1,2 @@
 # Copy data from VIVO to org_hierarchies every Monday at 3:11 AM
-11 3 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/write_uf_fiscal_orgs_to_org_hierarchies.R
+11 3 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/write_uf_fiscal_orgs_to_org_hierarchies.R

cron/write_uf_fiscal_orgs_to_person_org

@@ -1,2 +1,2 @@
 # Copy data from VIVO to org_hierarchies every Monday at 3:13 AM
-13 3 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/write_uf_fiscal_orgs_to_person_org.R
+13 3 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/write_uf_fiscal_orgs_to_person_org.R

etl/run_etl.R

@@ -0,0 +1,44 @@
+library(redcapcustodian)
+library(dotenv)
+library(callr)
+library(argparse)
+
+set_script_run_time()
+
+parser <- ArgumentParser()
+parser$add_argument("script_name", help="Script to be run")
+parser$add_argument("optional_args", nargs='*', help="Zero or more optional arguments of any type")
+
+if (!interactive()) {
+  args <- parser$parse_args()
+} else {
+  args <- parser$parse_args(
+    c(
+      "study_template/etl/test_failure_alert.R",
+      "test",
+      "another test"
+    )
+  )
+}
+
+script_name <- args$script_name
+optional_args <- args$optional_args
+
+if(!fs::file_exists(script_name)) {
+  stop(sprintf("Specified file, %s, does not exist", script_name))
+}
+
+tryCatch({
+  if (length(optional_args) == 0) {
+    rscript(script = script_name, stderr = "log.txt")
+  } else {
+    rscript(script = script_name, cmdargs = optional_args, stderr = "log.txt")
+  }
+}, error = function(e) {
+  email_body <- "See the attached log for error details."
+  script_path <- paste(basename(getwd()), script_name, sep = "/")
+  email_subject <- paste0("Failed | ", script_path, " | ", format(get_script_run_time(), "%Y-%m-%d"))
+  file_name = "log.txt"
+
+  send_email(email_body = email_body, email_subject = email_subject, file_name = file_name)
+})