From 530d7da39e9c3380005c408caf7fcf605be242f5 Mon Sep 17 00:00:00 2001 From: Laurence James-Woodley Date: Tue, 14 Jan 2025 18:35:03 -0500 Subject: [PATCH 1/3] add run_etl --- etl/run_etl.R | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 etl/run_etl.R diff --git a/etl/run_etl.R b/etl/run_etl.R new file mode 100644 index 0000000..553b3f2 --- /dev/null +++ b/etl/run_etl.R @@ -0,0 +1,44 @@ +library(redcapcustodian) +library(dotenv) +library(callr) +library(argparse) + +set_script_run_time() + +parser <- ArgumentParser() +parser$add_argument("script_name", help="Script to be run") +parser$add_argument("optional_args", nargs='*', help="Zero or more optional arguments of any type") + +if (!interactive()) { + args <- parser$parse_args() +} else { + args <- parser$parse_args( + c( + "study_template/etl/test_failure_alert.R", + "test", + "another test" + ) + ) +} + +script_name <- args$script_name +optional_args <- args$optional_args + +if(!fs::file_exists(script_name)) { + stop(sprintf("Specified file, %s, does not exist", script_name)) +} + +tryCatch({ + if (length(optional_args) == 0) { + rscript(script = script_name, stderr = "log.txt") + } else { + rscript(script = script_name, cmdargs = optional_args, stderr = "log.txt") + } +}, error = function(e) { + email_body <- "See the attached log for error details." + script_path <- paste(basename(getwd()), script_name, sep = "/") + email_subject <- paste0("Failed | ", script_path, " | ", format(get_script_run_time(), "%Y-%m-%d")) + file_name = "log.txt" + + send_email(email_body = email_body, email_subject = email_subject, file_name = file_name) +}) From 6485e452cd4e6179d44273d6dbcbc6ffb9986518 Mon Sep 17 00:00:00 2001 From: Laurence James-Woodley Date: Tue, 14 Jan 2025 18:46:25 -0500 Subject: [PATCH 2/3] use run_etl --- cron/billable_candidates | 2 +- cron/cleanup_bad_email_addresses | 2 +- cron/cleanup_bad_project_pi_email_addresses | 2 +- cron/cleanup_project_ownership_table | 2 +- cron/create_and_send_new_invoice_line_items | 2 +- cron/delete_abandoned_projects | 2 +- cron/request_correction_of_bad_ownership_data | 2 +- cron/sequester_orphans | 4 ++-- cron/sequester_unpaid_projects | 4 ++-- cron/update_free_support_time_remaining | 2 +- cron/update_probono_service_request_records | 2 +- cron/update_project_billable_attribute | 2 +- cron/warn_owners_of_impending_bill | 2 +- cron/write_uf_fiscal_orgs_to_org_hierarchies | 2 +- cron/write_uf_fiscal_orgs_to_person_org | 2 +- 15 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cron/billable_candidates b/cron/billable_candidates index 4f05c5d..015126a 100644 --- a/cron/billable_candidates +++ b/cron/billable_candidates @@ -1,2 +1,2 @@ # Send billable_candidates report each monday morning -3 07 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript report/billable_candidates.R +3 07 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R report/billable_candidates.R diff --git a/cron/cleanup_bad_email_addresses b/cron/cleanup_bad_email_addresses index 28a4f8e..ce69ed6 100644 --- a/cron/cleanup_bad_email_addresses +++ b/cron/cleanup_bad_email_addresses @@ -1,2 +1,2 @@ # Clean bad email addresses every tuesday at 10 am -0 10 * * 2 root /usr/bin/docker run --rm --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/cleanup_bad_email_addresses.R +0 10 * * 2 root /usr/bin/docker run --rm --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/cleanup_bad_email_addresses.R diff --git a/cron/cleanup_bad_project_pi_email_addresses b/cron/cleanup_bad_project_pi_email_addresses index cfb74a5..511260c 100644 --- a/cron/cleanup_bad_project_pi_email_addresses +++ b/cron/cleanup_bad_project_pi_email_addresses @@ -1,2 +1,2 @@ # Clean bad PI email addresses every Tuesday at 10:03 am -3 10 * * 2 root /usr/bin/docker run --rm --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/cleanup_bad_project_pi_email_addresses.R +3 10 * * 2 root /usr/bin/docker run --rm --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/cleanup_bad_project_pi_email_addresses.R diff --git a/cron/cleanup_project_ownership_table b/cron/cleanup_project_ownership_table index 7374963..e96a674 100644 --- a/cron/cleanup_project_ownership_table +++ b/cron/cleanup_project_ownership_table @@ -1,2 +1,2 @@ # This script will run daily at 12:47 AM -47 0 * * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/cleanup_project_ownership_table.R +47 0 * * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/cleanup_project_ownership_table.R diff --git a/cron/create_and_send_new_invoice_line_items b/cron/create_and_send_new_invoice_line_items index 813a2f5..14bf00e 100644 --- a/cron/create_and_send_new_invoice_line_items +++ b/cron/create_and_send_new_invoice_line_items @@ -1,2 +1,2 @@ # send invoice line items on the 5th of the month at 8:05 a.m. -5 8 5 * * root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/create_and_send_new_invoice_line_items.R +5 8 5 * * root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/create_and_send_new_invoice_line_items.R diff --git a/cron/delete_abandoned_projects b/cron/delete_abandoned_projects index daedcb9..522f073 100644 --- a/cron/delete_abandoned_projects +++ b/cron/delete_abandoned_projects @@ -1,2 +1,2 @@ # delete abandoned projects on the 28th of the month at 14:03. -3 14 28 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/delete_abandoned_projects.R +3 14 28 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/delete_abandoned_projects.R diff --git a/cron/request_correction_of_bad_ownership_data b/cron/request_correction_of_bad_ownership_data index f54cb75..d6e1bdc 100644 --- a/cron/request_correction_of_bad_ownership_data +++ b/cron/request_correction_of_bad_ownership_data @@ -1,2 +1,2 @@ # Report bad ownership on the first of every month at 10:05 a.m. -5 10 1 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript report/request_correction_of_bad_ownership_data.R +5 10 1 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R report/request_correction_of_bad_ownership_data.R diff --git a/cron/sequester_orphans b/cron/sequester_orphans index acb7cd0..1f44b06 100644 --- a/cron/sequester_orphans +++ b/cron/sequester_orphans @@ -1,2 +1,2 @@ -# sequester orphans on the 1st, 23rd, and 28th of the month at 8:02 a.m. -2 8 1,23,28 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/sequester_orphans.R +# sequester orphans on the 1st, 23rd, and 28th of the month at 8:02 a.m. +2 8 1,23,28 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/sequester_orphans.R diff --git a/cron/sequester_unpaid_projects b/cron/sequester_unpaid_projects index cb2d4f9..273650e 100644 --- a/cron/sequester_unpaid_projects +++ b/cron/sequester_unpaid_projects @@ -1,5 +1,5 @@ -# sequester unpaid projects every Tuesday at 8:04 a.m. +# sequester unpaid projects every Tuesday at 8:04 a.m. # Note: Date tests internal to the script will quietly exit the code # on Days 8-31. This will force the sequestration events to happen # only on the *first* Tuesday of the month. -4 8 * * 2 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/sequester_unpaid_projects.R +4 8 * * 2 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/sequester_unpaid_projects.R diff --git a/cron/update_free_support_time_remaining b/cron/update_free_support_time_remaining index b1b3926..f21ffdf 100644 --- a/cron/update_free_support_time_remaining +++ b/cron/update_free_support_time_remaining @@ -1,2 +1,2 @@ # Update update_free_support_time_remaining weekdays daily every 2 hours from 7am - 7pm -7 7-19/2 * * 1-5 root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/update_free_support_time_remaining.R +7 7-19/2 * * 1-5 root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/update_free_support_time_remaining.R diff --git a/cron/update_probono_service_request_records b/cron/update_probono_service_request_records index ad899f3..0d6c490 100644 --- a/cron/update_probono_service_request_records +++ b/cron/update_probono_service_request_records @@ -1,2 +1,2 @@ # Update probono records weekdays daily every 2 hours from 7am - 7pm -3 7-19/2 * * 1-5 root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/update_probono_service_request_records.R +3 7-19/2 * * 1-5 root /usr/bin/docker run -v /rcc/rcc.billing:/root --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/update_probono_service_request_records.R diff --git a/cron/update_project_billable_attribute b/cron/update_project_billable_attribute index 5ed72e7..7d8086b 100644 --- a/cron/update_project_billable_attribute +++ b/cron/update_project_billable_attribute @@ -1,2 +1,2 @@ # Update project billable attribute every day at 1:05 AM -5 1 * * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/update_project_billable_attribute.R +5 1 * * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/update_project_billable_attribute.R diff --git a/cron/warn_owners_of_impending_bill b/cron/warn_owners_of_impending_bill index 111c3b9..52dbeb3 100644 --- a/cron/warn_owners_of_impending_bill +++ b/cron/warn_owners_of_impending_bill @@ -1,2 +1,2 @@ # Warn project owners of impending bills every 1st and 23rd of the month at 10:03 a.m. -3 10 1,23 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript report/warn_owners_of_impending_bill.R +3 10 1,23 * * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R report/warn_owners_of_impending_bill.R diff --git a/cron/write_uf_fiscal_orgs_to_org_hierarchies b/cron/write_uf_fiscal_orgs_to_org_hierarchies index fdcafcc..f8bc3b4 100644 --- a/cron/write_uf_fiscal_orgs_to_org_hierarchies +++ b/cron/write_uf_fiscal_orgs_to_org_hierarchies @@ -1,2 +1,2 @@ # Copy data from VIVO to org_hierarchies every Monday at 3:11 AM -11 3 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/write_uf_fiscal_orgs_to_org_hierarchies.R +11 3 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/write_uf_fiscal_orgs_to_org_hierarchies.R diff --git a/cron/write_uf_fiscal_orgs_to_person_org b/cron/write_uf_fiscal_orgs_to_person_org index e3c7678..9758a99 100644 --- a/cron/write_uf_fiscal_orgs_to_person_org +++ b/cron/write_uf_fiscal_orgs_to_person_org @@ -1,2 +1,2 @@ # Copy data from VIVO to org_hierarchies every Monday at 3:13 AM -13 3 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/write_uf_fiscal_orgs_to_person_org.R +13 3 * * 1 root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R etl/write_uf_fiscal_orgs_to_person_org.R From 5745d5113875f45854842eea913ac5936ad0c9e6 Mon Sep 17 00:00:00 2001 From: Laurence James-Woodley Date: Tue, 14 Jan 2025 18:48:43 -0500 Subject: [PATCH 3/3] use run_etl --- cron/remind_owners_to_review_ownership | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/remind_owners_to_review_ownership b/cron/remind_owners_to_review_ownership index d26e5c4..4745b50 100644 --- a/cron/remind_owners_to_review_ownership +++ b/cron/remind_owners_to_review_ownership @@ -1,2 +1,2 @@ # remind_owners_to_review_ownership.R quarterly on the first of every month at 10:05 a.m. -13 8 10 2,5,8,11 * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript report/remind_owners_to_review_ownership.R +13 8 10 2,5,8,11 * root /usr/bin/docker run --rm --env-file /rcc/default.env --env-file /rcc/rcc.billing/prod.env rcc.billing Rscript etl/run_etl.R report/remind_owners_to_review_ownership.R