![[Jenkins]](/jenkins/static/57c8a776/images/svgs/logo.svg){kind=logo}
Started by an SCM change Obtained jenkins_tests/clubb_release_diff_test/Jenkinsfile from git https://github.com/larson-group/clubb.git [Pipeline] Start of Pipeline [Pipeline] node Running on Jenkins in /home/jenkins/workspace/clubb_release_diff_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git Cloning the remote Git repository Cloning repository https://github.com/larson-group/clubb.git > git init /home/jenkins/workspace/clubb_release_diff_test # timeout=10 Fetching upstream changes from https://github.com/larson-group/clubb.git > git --version # timeout=10 > git --version # 'git version 2.30.2' using GIT_ASKPASS to set credentials A token based key used by Jenkins to preform Github actions, created 6/21/2021 > git fetch --tags --force --progress -- https://github.com/larson-group/clubb.git +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/larson-group/clubb.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch > git rev-parse refs/remotes/origin/master^{commit} # timeout=10 Checking out Revision 6de84c3d28a538d620009cbe3d98a634782820be (refs/remotes/origin/master) > git config core.sparsecheckout # timeout=10 > git checkout -f 6de84c3d28a538d620009cbe3d98a634782820be # timeout=10 Commit message: "Major overhaul of run_bindiff_all.py" > git rev-list --no-walk 59786b03dd4afb944e5e9284a2b207220122dbb9 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] stage [Pipeline] { (Checkout Clubb and Clubb_Release) [Pipeline] sh + git clone https://github.com/larson-group/clubb.git Cloning into 'clubb'... + git clone https://github.com/larson-group/clubb_release.git Cloning into 'clubb_release'... [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Diff) [Pipeline] sh + diff --exclude=.git --exclude=version_clubb_core.txt --exclude=version_silhs.txt -r clubb clubb_release diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/input/stats/all_stats.in clubb_release/input/stats/all_stats.in 16c16 < 'thlm', 'thvm', 'rtm', 'rcm', 'rvm', 'um', 'vm', 'um_ref', 'vm_ref', 'wm_zt', --- > 'thlm', 'thvm', 'rtm', 'rcm', 'rvm', 'um', 'vm', 'um_ref','vm_ref','wm_zt', 19,20c19,20 < 'rel_humidity', 'wp3', 'thlp3', 'rtp3', 'wpthlp2', 'wp2thlp', 'wprtp2', 'wp2rtp', 'Skw_zt', < 'Skthl_zt', 'Skrt_zt', --- > 'rel_humidity', 'wp3','thlp3','rtp3', 'wpthlp2', 'wp2thlp', 'wprtp2', 'wp2rtp', 'Skw_zt', > 'Skthl_zt','Skrt_zt', 32c32 < 'rtm_forcing', 'rtm_sdmp', 'rtm_mc', 'rtm_pd', 'rvm_mc', 'rcm_mc', 'rcm_sd_mg_morr', --- > 'rtm_forcing', 'rtm_sdmp','rtm_mc', 'rtm_pd', 'rvm_mc', 'rcm_mc', 'rcm_sd_mg_morr', 34c34 < 'thlm_forcing', 'thlm_sdmp', 'thlm_mc', --- > 'thlm_forcing', 'thlm_sdmp','thlm_mc', 136c136 < 'up2', 'vp2', 'Skw_zm', 'Skthl_zm', 'Skrt_zm', --- > 'up2', 'vp2', 'Skw_zm', 'Skthl_zm','Skrt_zm', 142c142 < 'gamma_Skw_fnc', 'coef_wp4_implicit', 'C6rt_Skw_fnc', 'C6thl_Skw_fnc', 'C6_term', --- > 'gamma_Skw_fnc', 'coef_wp4_implicit', 'C6rt_Skw_fnc', 'C6thl_Skw_fnc', 179c179 < 'wpsclrprtp', 'wpsclrpthlp', 'wpedsclrp', 'stability_correction', --- > 'wpsclrprtp', 'wpsclrpthlp', 'wpedsclrp', 'stability_correction', 'Richardson_num', 181,182c181 < 'bv_freq_sqd', 'bv_freq_sqd_splat', 'bv_freq_sqd_mixed', 'bv_freq_sqd_moist', 'bv_freq_sqd_dry', < 'shear_sqd', 'invrs_tau_zm', 'invrs_tau_xp2_zm', 'invrs_tau_wp2_zm', --- > 'bv_freq_sqd', 'bv_freq_sqd_splat', 'shear_sqd', 'invrs_tau_zm', 'invrs_tau_xp2_zm', 'invrs_tau_wp2_zm', 184c183 < 'invrs_tau_sfc', 'invrs_tau_shear', 'Ri_zm', --- > 'invrs_tau_sfc', 'invrs_tau_shear', 'sqrt_Ri_zm', diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/input/stats/standard_stats.in clubb_release/input/stats/standard_stats.in 16c16 < 'thlm', 'thvm', 'rtm', 'rcm', 'rvm', 'um', 'vm', 'um_ref', 'vm_ref', 'wm_zt', --- > 'thlm', 'thvm', 'rtm', 'rcm', 'rvm', 'um', 'vm', 'um_ref','vm_ref','wm_zt', 18,24c18,23 < 'rcm_in_cloud', 'p_in_Pa', 'exner', 'rho_ds_zt', 'thv_ds_zt', 'Lscale', 'T_in_K', < 'rel_humidity', 'wp3', 'thlp3', 'rtp3', 'wpthlp2', 'wp2thlp', 'wprtp2', 'wp2rtp', 'Skw_zt', < 'Skthl_zt', 'Skrt_zt', < 'Lscale_up', 'Lscale_down', 'tau_zt', 'Kh_zt', 'wp2thvp', 'wp2rcp', 'wp2hmp', < < 'wprtpthlp', 'wpup2', 'wpvp2', 'rc_coef', 'Lscale_pert_1', 'Lscale_pert_2', < 'sigma_sqd_w_zt', 'rho', 'radht', 'radht_LW', 'radht_SW', --- > 'rcm_in_cloud', 'p_in_Pa', 'exner', 'rho_ds_zt', 'thv_ds_zt', 'Lscale', > 'Lscale_pert_1', 'Lscale_pert_2', 'T_in_K', 'rel_humidity', 'wp3','thlp3','rtp3','wpthlp2', > 'wp2thlp', 'wprtp2', 'wp2rtp', 'Skw_zt', 'Skthl_zt','Skrt_zt', 'Lscale_up', 'Lscale_down', 'tau_zt', > 'Kh_zt', 'wp2thvp', 'wp2rcp', 'wp2hmp', 'wprtpthlp', 'wpup2', 'wpvp2', 'rc_coef', > 'sigma_sqd_w_zt', 'rho', > 'radht', 'radht_LW', 'radht_SW', 29c28,29 < 'precip_rate_zt', 'rsm', 'rim', 'rgm', --- > 'precip_rate_zt', > 'rsm', 'rim', 'rgm', 32c32 < 'rtm_forcing', 'rtm_sdmp', 'rtm_mc', 'rtm_pd', 'rvm_mc', 'rcm_mc', 'rcm_sd_mg_morr', --- > 'rtm_forcing', 'rtm_sdmp','rtm_mc', 'rtm_pd', 'rvm_mc', 'rcm_mc', 'rcm_sd_mg_morr', 34c34 < 'thlm_forcing', 'thlm_sdmp', 'thlm_mc', --- > 'thlm_forcing', 'thlm_sdmp','thlm_mc', 83,84c83 < 'mu_hm_i', 'mu_Ncn_i', < 'sigma_hm_i', 'sigma_Ncn_i', --- > 'mu_hm_i', 'mu_Ncn_i', 'sigma_hm_i', 'sigma_Ncn_i', 89,90d87 < < 94d90 < 103c99 < 'NIACRS', 'NGRACS', 'NSMLTS', 'NSAGG', 'NPRCI', 'NSCNG', 'NSUBS', 'PRC', 'PRA', --- > 'NIACRS', 'NGRACS', 'NSMLTS', 'NSAGG', 'NPRCI', 'NSCNG','NSUBS', 'PRC', 'PRA', 124,126c120 < 'lh_Nrm_src_adj', 'lh_Nrm_evap_adj', 'lh_precip_frac', 'lh_mixt_frac', < < 'lh_m_vol_rad_rain', --- > 'lh_Nrm_src_adj', 'lh_Nrm_evap_adj', 'lh_precip_frac', 'lh_mixt_frac', 'lh_m_vol_rad_rain', 136c130 < 'up2', 'vp2', 'Skw_zm', 'Skthl_zm', 'Skrt_zm', --- > 'up2', 'vp2', 'Skw_zm', 'Skthl_zm','Skrt_zm', 142c136 < 'gamma_Skw_fnc', 'coef_wp4_implicit', 'C6rt_Skw_fnc', 'C6thl_Skw_fnc', 'C6_term', --- > 'gamma_Skw_fnc', 'coef_wp4_implicit', 'C6rt_Skw_fnc', 'C6thl_Skw_fnc', 145c139 < 'em', 'mean_w_up', 'mean_w_down', 'shear', 'wp3_zm', 'thlp3_zm', 'rtp3_zm', --- > 'em', 'mean_w_up', 'mean_w_down', 'shear', 'wp3_zm','thlp3_zm','rtp3_zm', 153,155c147,148 < 'wp2_pr3', 'wp2_pr_dfsn', 'wp2_dp1', 'wp2_dp2', 'wp2_sdmp', < 'wp2_cl', 'wp2_pd', 'wp2_sf', 'wp2_splat', < 'vp2_bt', 'vp2_ma', 'vp2_ta', 'vp2_tp', 'vp2_dp1', 'vp2_dp2', --- > 'wp2_pr3', 'wp2_dp1', 'wp2_dp2', 'wp2_sdmp', 'wp2_cl', 'wp2_pd', 'wp2_sf', 'wp2_splat', > 'wp2_pr_dfsn', 'vp2_bt', 'vp2_ma', 'vp2_ta', 'vp2_tp', 'vp2_dp1', 'vp2_dp2', 163,165c156,158 < 'wprtp_bt', 'wprtp_ma', 'wprtp_ta', 'wprtp_tp', 'wprtp_ac', < 'wprtp_bp', 'wprtp_pr1', 'wprtp_pr2', 'wprtp_pr3', 'wprtp_dp1', < 'wprtp_mfl', 'wprtp_cl', 'wprtp_sicl', 'wprtp_pd', 'wprtp_forcing', 'wprtp_mc', --- > 'wprtp_bt', 'wprtp_ma', 'wprtp_ta', 'wprtp_tp', 'wprtp_ac', 'wprtp_bp', > 'wprtp_pr1', 'wprtp_pr2', 'wprtp_pr3', 'wprtp_dp1', 'wprtp_mfl', > 'wprtp_cl', 'wprtp_sicl', 'wprtp_pd', 'wprtp_forcing', 'wprtp_mc', 169,174c162,167 < 'rtp2_bt', 'rtp2_ma', 'rtp2_ta', 'rtp2_tp', < 'rtp2_dp1', 'rtp2_dp2', 'rtp2_cl', 'rtp2_pd', 'rtp2_sf', 'rtp2_forcing', 'rtp2_mc', < 'thlp2_bt', 'thlp2_ma', 'thlp2_ta', 'thlp2_tp', < 'thlp2_dp1', 'thlp2_dp2', 'thlp2_cl', 'thlp2_pd', 'thlp2_sf', 'thlp2_forcing', 'thlp2_mc', < 'rtpthlp_bt', 'rtpthlp_ma', 'rtpthlp_ta', 'rtpthlp_tp1', 'rtpthlp_tp2', < 'rtpthlp_dp1', 'rtpthlp_dp2', 'rtpthlp_cl', 'rtpthlp_sf', 'rtpthlp_forcing', 'rtpthlp_mc', --- > 'rtp2_bt', 'rtp2_ma', 'rtp2_ta', 'rtp2_tp', 'rtp2_dp1', 'rtp2_dp2', > 'rtp2_cl', 'rtp2_pd', 'rtp2_sf', 'rtp2_forcing', 'rtp2_mc', > 'thlp2_bt', 'thlp2_ma', 'thlp2_ta', 'thlp2_tp', 'thlp2_dp1', 'thlp2_dp2', > 'thlp2_cl', 'thlp2_pd', 'thlp2_sf', 'thlp2_forcing', 'thlp2_mc', > 'rtpthlp_bt', 'rtpthlp_ma', 'rtpthlp_ta', 'rtpthlp_tp1', 'rtpthlp_tp2', 'rtpthlp_dp1', > 'rtpthlp_dp2', 'rtpthlp_cl', 'rtpthlp_sf', 'rtpthlp_forcing', 'rtpthlp_mc', 176,182c169,170 < 'wpthlp_mfl_min', 'wpthlp_mfl_max', 'wprtp_mfl_min', 'wprtp_mfl_max', < < < 'stability_correction', < < 'bv_freq_sqd', 'bv_freq_sqd_splat', 'bv_freq_sqd_mixed', 'bv_freq_sqd_moist', 'bv_freq_sqd_dry', < 'shear_sqd', 'invrs_tau_zm', 'invrs_tau_xp2_zm', 'invrs_tau_wp2_zm', --- > 'wpthlp_mfl_min', 'wpthlp_mfl_max', 'wprtp_mfl_min', 'wprtp_mfl_max', 'stability_correction', > 'bv_freq_sqd', 'bv_freq_sqd_splat', 'shear_sqd', 'invrs_tau_zm', 'invrs_tau_xp2_zm', 'invrs_tau_wp2_zm', 184c172 < 'invrs_tau_sfc', 'invrs_tau_shear', 'Ri_zm', --- > 'invrs_tau_sfc', 'invrs_tau_shear', 'Richardson_num', 209,212c197 < 'rtm_spur_src', 'thlm_spur_src', < < < 'rs_sd_morr_int', --- > 'rtm_spur_src', 'thlm_spur_src', 'rs_sd_morr_int', diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/input/tunable_parameters_Lscale/configurable_model_flags.in clubb_release/input/tunable_parameters_Lscale/configurable_model_flags.in 3,5c3 < ipdf_call_placement = 2 < penta_solve_method = 1 < tridiag_solve_method = 1 --- > ipdf_call_placement = 1 7,55c5,47 < < l_godunov_upwind_wpxp_ta = .false., < l_upwind_xpyp_ta = .true., < l_godunov_upwind_xpyp_ta = .false., < l_upwind_xm_ma = .true., < l_quintic_poly_interp = .false., < l_tke_aniso = .true., < l_vert_avg_closure = .false., < l_standard_term_ta = .false., < l_partial_upwind_wp3 = .false., < l_use_cloud_cover = .false., < l_rcm_supersat_adj = .true., < l_damp_wp3_Skw_squared = .true., < l_min_wp2_from_corr_wx = .true., < l_min_xp2_from_corr_wx = .true., < l_C2_cloud_frac = .false., < l_predict_upwp_vpwp = .true., < l_diag_Lscale_from_tau = .false., < l_stability_correct_tau_zm = .false., < l_damp_wp2_using_em = .true., < l_use_C7_Richardson = .true., < l_use_precip_frac = .true., < l_do_expldiff_rtm_thlm = .false., < l_use_C11_Richardson = .false., < l_use_shear_Richardson = .false., < l_prescribed_avg_deltaz = .false., < l_diffuse_rtm_and_thlm = .false., < l_stability_correct_Kh_N2_zm = .false., < l_trapezoidal_rule_zt = .false., < l_trapezoidal_rule_zm = .false., < l_call_pdf_closure_twice = .false., < l_Lscale_plume_centered = .false., < l_brunt_vaisala_freq_moist = .false., < l_use_thvm_in_bv_freq = .false., < l_lmm_stepping = .false., < l_e3sm_config = .false., < l_vary_convect_depth = .false., < l_use_tke_in_wp3_pr_turb_term = .true., < l_use_tke_in_wp2_wp3_K_dfsn = .false., < l_use_wp3_lim_with_smth_Heaviside = .false., < l_smooth_Heaviside_tau_wpxp = .false., < l_modify_limiters_for_cnvg_test = .false., < l_enable_relaxed_clipping = .false., < l_linearize_pbl_winds = .false., < l_mono_flux_lim_thlm = .true., < l_mono_flux_lim_rtm = .true., < l_mono_flux_lim_um = .true., < l_mono_flux_lim_vm = .true., < l_mono_flux_lim_spikefix = .true. --- > l_godunov_upwind_wpxp_ta = .false., > l_upwind_xpyp_ta = .true., > l_godunov_upwind_xpyp_ta = .false., > l_upwind_xm_ma = .true., > l_quintic_poly_interp = .false., > l_tke_aniso = .true., > l_vert_avg_closure = .false., > l_standard_term_ta = .false., > l_partial_upwind_wp3 = .false., > l_use_cloud_cover = .false., > l_rcm_supersat_adj = .true., > l_damp_wp3_Skw_squared = .true., > l_min_wp2_from_corr_wx = .true., > l_min_xp2_from_corr_wx = .true., > l_C2_cloud_frac = .false., > l_predict_upwp_vpwp = .true., > l_diag_Lscale_from_tau = .false., > l_stability_correct_tau_zm = .false., > l_damp_wp2_using_em = .true., > l_use_C7_Richardson = .true., > l_use_precip_frac = .true., > l_do_expldiff_rtm_thlm = .false., > l_use_C11_Richardson = .false., > l_use_shear_Richardson = .false., > l_prescribed_avg_deltaz = .false., > l_diffuse_rtm_and_thlm = .false., > l_stability_correct_Kh_N2_zm = .false., > l_trapezoidal_rule_zt = .false., > l_trapezoidal_rule_zm = .false., > l_call_pdf_closure_twice = .false., > l_Lscale_plume_centered = .false., > l_brunt_vaisala_freq_moist = .false., > l_use_thvm_in_bv_freq = .false., > l_lmm_stepping = .false., > l_e3sm_config = .false., > l_vary_convect_depth = .false., > l_use_tke_in_wp3_pr_turb_term = .false., > l_use_tke_in_wp2_wp3_K_dfsn = .false. > l_mono_flux_lim_thlm = .true., > l_mono_flux_lim_rtm = .true., > l_mono_flux_lim_um = .true., > l_mono_flux_lim_vm = .true., > l_mono_flux_lim_spikefix = .true. 72,75d63 < / < < &configurable_multi_column_nl < num_standalone_columns = 1 Only in clubb/postprocessing/pyplotgen/config: VariableGroupBaseBudgetsLumpedBuoy.py diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupBaseBudgets.py clubb_release/postprocessing/pyplotgen/config/VariableGroupBaseBudgets.py 18c18 < priority_vars=False, background_rcm=False, background_rcm_folder=None): --- > priority_vars=False): 538,539c538 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupBaseBudgetsSamStyle.py clubb_release/postprocessing/pyplotgen/config/VariableGroupBaseBudgetsSamStyle.py 18c18 < priority_vars=False, background_rcm=False, background_rcm_folder=None): --- > priority_vars=False): 376,377c376 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupBase.py clubb_release/postprocessing/pyplotgen/config/VariableGroupBase.py 22,23c22 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False): 685,686c684 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupCorrelations.py clubb_release/postprocessing/pyplotgen/config/VariableGroupCorrelations.py 16c16 < priority_vars=False, background_rcm=False, background_rcm_folder=None): --- > priority_vars=False): 161,162c161 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupIceMP.py clubb_release/postprocessing/pyplotgen/config/VariableGroupIceMP.py 14,15c14,15 < def __init__(self, case, clubb_datasets=None, sam_benchmark_dataset=None, sam_datasets=None, < coamps_benchmark_dataset=None, wrf_benchmark_dataset=None, --- > def __init__(self, case, clubb_datasets=None, sam_benchmark_dataset=None, sam_datasets=None, coamps_benchmark_dataset=None, > wrf_benchmark_dataset=None, 17,18c17 < e3sm_datasets=None, wrf_datasets=None, priority_vars=False, background_rcm=False, < background_rcm_folder=None): --- > e3sm_datasets=None, wrf_datasets=None, priority_vars=False): 151,152c150 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupKKMP.py clubb_release/postprocessing/pyplotgen/config/VariableGroupKKMP.py 13,14c13,14 < def __init__(self, case, clubb_datasets=None, sam_benchmark_dataset=None, < coamps_benchmark_dataset=None, r408_dataset=None, wrf_benchmark_dataset=None, --- > def __init__(self, case, clubb_datasets=None, sam_benchmark_dataset=None, coamps_benchmark_dataset=None, r408_dataset=None, > wrf_benchmark_dataset=None, 16,17c16 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False): 69,70c68 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupLiquidMP.py clubb_release/postprocessing/pyplotgen/config/VariableGroupLiquidMP.py 16,17c16 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None,priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None,priority_vars=False): 151,152c150 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupNondimMoments.py clubb_release/postprocessing/pyplotgen/config/VariableGroupNondimMoments.py 21,22c21 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False): 215,216c214 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupNormalizedVariations.py clubb_release/postprocessing/pyplotgen/config/VariableGroupNormalizedVariations.py 17,18c17 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False): 76,77c75 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupSamBudgets.py clubb_release/postprocessing/pyplotgen/config/VariableGroupSamBudgets.py 18c18 < priority_vars=False, background_rcm=False, background_rcm_folder=None): --- > priority_vars=False): 984,985c984 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupSamMultilineProfiles.py clubb_release/postprocessing/pyplotgen/config/VariableGroupSamMultilineProfiles.py 18,19c18 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False): 451,452c450 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupSamProfiles.py clubb_release/postprocessing/pyplotgen/config/VariableGroupSamProfiles.py 18,19c18 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False): 1204,1205c1203 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupScalars.py clubb_release/postprocessing/pyplotgen/config/VariableGroupScalars.py 16,17c16 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False): 81,82c80 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupSubcolumns.py clubb_release/postprocessing/pyplotgen/config/VariableGroupSubcolumns.py 16,17c16 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, < background_rcm=False, background_rcm_folder=False): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None): 186,187c185 < cam_datasets=cam_datasets, sam_datasets=sam_datasets, wrf_datasets=wrf_datasets, < background_rcm=background_rcm, background_rcm_folder=background_rcm_folder) --- > cam_datasets=cam_datasets, sam_datasets=sam_datasets, wrf_datasets=wrf_datasets) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupTaus.py clubb_release/postprocessing/pyplotgen/config/VariableGroupTaus.py 16,17c16 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False): 37,81d35 < 'clubb': ['bv_freq_sqd_mixed'], < 'sam': [], < 'coamps': [], < 'r408': ['bv_freq_sqd_mixed'], < 'hoc': ['bv_freq_sqd_mixed'], < 'e3sm': ['bv_freq_sqd_mixed'], < 'cam': ['bv_freq_sqd_mixed'], < 'wrf': ['bv_freq_sqd_mixed'], < }, < 'title': 'Mixed Brunt-Vaisala frequency squared', < 'axis_title': 'bv_freq_sqd_mixed [$\mathrm{1/s^2}$]', < 'sci_scale': 0, < }, < {'var_names': < { < 'clubb': ['bv_freq_sqd_moist'], < 'sam': [], < 'coamps': [], < 'r408': ['bv_freq_sqd_moist'], < 'hoc': ['bv_freq_sqd_moist'], < 'e3sm': ['bv_freq_sqd_moist'], < 'cam': ['bv_freq_sqd_moist'], < 'wrf': ['bv_freq_sqd_moist'], < }, < 'title': 'Brunt-Vaisala frequency squared in moist air', < 'axis_title': 'bv_freq_sqd_moist [$\mathrm{1/s^2}$]', < 'sci_scale': 0, < }, < {'var_names': < { < 'clubb': ['bv_freq_sqd_dry'], < 'sam': [], < 'coamps': [], < 'r408': ['bv_freq_sqd_dry'], < 'hoc': ['bv_freq_sqd_dry'], < 'e3sm': ['bv_freq_sqd_dry'], < 'cam': ['bv_freq_sqd_dry'], < 'wrf': ['bv_freq_sqd_dry'], < }, < 'title': 'Brunt-Vaisala frequency squared in dry air', < 'axis_title': 'bv_freq_sqd_dry [$\mathrm{1/s^2}$]', < 'sci_scale': 0, < }, < {'var_names': < { 171,239d124 < {'var_names': < { < 'clubb': ['C6_term'], < 'sam': ['C6_TERM'], # Dummy < 'coamps': ['C6_term'], # Dummy < 'r408': ['C6_term'], # Dummy < 'hoc': ['C6_term'], # Dummy < 'e3sm': ['C6_term'], # Dummy < 'cam': ['C6_term'], # Dummy < 'wrf': ['C6_term'], # Dummy < }, < 'sci_scale': 0, < }, < {'var_names': < { < 'clubb': ['C1_Skw_fnc'], < 'sam': ['C1_SKW_FNC'], # Dummy < 'coamps': ['C1_Skw_fnc'], # Dummy < 'r408': ['C1_Skw_fnc'], # Dummy < 'hoc': ['C1_Skw_fnc'], # Dummy < 'e3sm': ['C1_Skw_fnc'], # Dummy < 'cam': ['C1_Skw_fnc'], # Dummy < 'wrf': ['C1_Skw_fnc'], # Dummy < }, < 'title': 'C1_Skw_fnc', < 'sci_scale': 0, < }, < {'var_names': < { < 'clubb': ['C6rt_Skw_fnc'], < 'sam': ['C6RT_SKW_FNC'], # Dummy < 'coamps': ['C6rt_Skw_fnc'], # Dummy < 'r408': ['C6rt_Skw_fnc'], # Dummy < 'hoc': ['C6rt_Skw_fnc'], # Dummy < 'e3sm': ['C6rt_Skw_fnc'], # Dummy < 'cam': ['C6rt_Skw_fnc'], # Dummy < 'wrf': ['C6rt_Skw_fnc'], # Dummy < }, < 'title': 'C6rt_Skw_fnc', < 'sci_scale': 0, < }, < {'var_names': < { < 'clubb': ['C6thl_Skw_fnc'], < 'sam': ['C6THL_SKW_FNC'], # Dummy < 'coamps': ['C6thl_Skw_fnc'], # Dummy < 'r408': ['C6thl_Skw_fnc'], # Dummy < 'hoc': ['C6thl_Skw_fnc'], # Dummy < 'e3sm': ['C6thl_Skw_fnc'], # Dummy < 'cam': ['C6thl_Skw_fnc'], # Dummy < 'wrf': ['C6thl_Skw_fnc'], # Dummy < }, < 'title': 'C6thl_Skw_fnc', < 'sci_scale': 0, < }, < {'var_names': < { < 'clubb': ['C7_Skw_fnc'], < 'sam': ['C7_SKW_FNC'], # Dummy < 'coamps': ['C7_Skw_fnc'], # Dummy < 'r408': ['C7_Skw_fnc'], # Dummy < 'hoc': ['C7_Skw_fnc'], # Dummy < 'e3sm': ['C7_Skw_fnc'], # Dummy < 'cam': ['C7_Skw_fnc'], # Dummy < 'wrf': ['C7_Skw_fnc'], # Dummy < }, < 'title': 'C7_Skw_fnc', < 'sci_scale': 0, < }, 248,249c133 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/config/VariableGroupWs.py clubb_release/postprocessing/pyplotgen/config/VariableGroupWs.py 16,17c16 < e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False, < background_rcm=False, background_rcm_folder=None): --- > e3sm_datasets=None, sam_datasets=None, wrf_datasets=None, priority_vars=False): 211,212c210 < priority_vars=priority_vars, background_rcm=background_rcm, < background_rcm_folder=background_rcm_folder) --- > priority_vars=priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/pyplotgen.py clubb_release/postprocessing/pyplotgen/pyplotgen.py 48,50c48,50 < plot_budgets=False, bu_morr=False, lumped_buoy_budgets=False, background_rcm=False, diff=None, < show_alphabetic_id=False, time_height=False, animation=None, samstyle=False, disable_multithreading=False, < pdf=False, pdf_filesize_limit=None, plot_subcolumns=False, image_extension=".png"): --- > plot_budgets=False, bu_morr=False, diff=None, show_alphabetic_id=False, > time_height=False, animation=None, samstyle=False, disable_multithreading=False, pdf=False, > pdf_filesize_limit=None, plot_subcolumns=False, image_extension=".png"): 88,89d87 < :param lumped_buoy_budgets: Lump together wpxp_bp and wpxp_pr3 terms in CLUBB's budgets < :param background_rcm: Show a height-based "contour" plot of time-averaged rcm behind CLUBB profiles. 114,115d111 < self.lumped_buoy_budgets = lumped_buoy_budgets < self.background_rcm = background_rcm 422,424c418,420 < plot_subcolumns=self.plot_subcolumns, lumped_buoy_budgets=self.lumped_buoy_budgets, < background_rcm=self.background_rcm, image_extension=self.image_extension, < total_panels_to_plot=0, priority_vars=self.priority_vars) --- > plot_subcolumns=self.plot_subcolumns, > image_extension=self.image_extension, total_panels_to_plot=0, > priority_vars=self.priority_vars) 646,649d641 < parser.add_argument("--lumped-buoy-budgets", help="Lump together wpxp_bp and wpxp_pr3 terms in CLUBB's budgets.", < action="store_true") < parser.add_argument("--background-rcm", help="Show a height-based 'contour' plot of time-averaged rcm behind CLUBB profiles.", < action="store_true") 803,806c795,798 < no_legends=args.no_legends, plot_budgets=args.plot_budgets, bu_morr=args.bu_morr, < lumped_buoy_budgets=args.lumped_buoy_budgets, background_rcm=args.background_rcm, diff=args.diff, < show_alphabetic_id=args.show_alphabetic_id, time_height=args.time_height_plots, animation=args.movies, < samstyle=args.sam_style_budgets, disable_multithreading=args.disable_multithreading, pdf=args.pdf, --- > no_legends=args.no_legends, plot_budgets=args.plot_budgets, > bu_morr=args.bu_morr, diff=args.diff, show_alphabetic_id=args.show_alphabetic_id, > time_height=args.time_height_plots, animation=args.movies, samstyle=args.sam_style_budgets, > disable_multithreading=args.disable_multithreading, pdf=args.pdf, diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/README.md clubb_release/postprocessing/pyplotgen/README.md 52,53d51 < | --lumped-buoy-budgets | Lump together wpxp_bp and wpxp_pr3 terms in CLUBB's budgets | < | --background_rcm | Show a height-based "contour" plot of time-averaged rcm behind CLUBB profiles | diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/src/CaseGallerySetup.py clubb_release/postprocessing/pyplotgen/src/CaseGallerySetup.py 11d10 < from config.VariableGroupBaseBudgetsLumpedBuoy import VariableGroupBaseBudgetsLumpedBuoy 33,35c32,34 < plot_les=False, plot_budgets=False, lumped_buoy_budgets=False, background_rcm=False, plot_r408=False, < plot_hoc=False, e3sm_folders=[], cam_folders=[], time_height=False, animation=None, samstyle=False, < plot_subcolumns=False, image_extension=".png", total_panels_to_plot=0, priority_vars=False): --- > plot_les=False, plot_budgets=False, plot_r408=False, plot_hoc=False, e3sm_folders=[], cam_folders=[], > time_height=False, animation=None, samstyle=False, plot_subcolumns=False, image_extension=".png", > total_panels_to_plot=0, priority_vars=False): 49,50d47 < :param lumped_buoy_budgets: If True and if plot_budgets in true, wpxp_bp and wpxp_pr3 will be lumped into one budget term < :param background_rcm: Show a height-based "contour" plot of time-averaged rcm behind CLUBB profiles. 69,70d65 < self.lumped_buoy_budgets = lumped_buoy_budgets < self.background_rcm = background_rcm 98d92 < self.lumped_buoy_budgets = False 117,122d110 < # Preserve the CLUBB folder listed first to use for background rcm plots. < if len(clubb_folders) != 0: < self.background_rcm_folder = clubb_folders[0] < else: < self.background_rcm_folder = None < 172,177c160,161 < if not self.lumped_buoy_budgets: < budget_variables = VariableGroupBaseBudgets(self, priority_vars=self.priority_vars, < clubb_datasets={folder_name:self.clubb_datasets[input_folder]}) < else: < budget_variables = VariableGroupBaseBudgetsLumpedBuoy(self, priority_vars=self.priority_vars, < clubb_datasets={folder_name:self.clubb_datasets[input_folder]}) --- > budget_variables = VariableGroupBaseBudgets(self, priority_vars=self.priority_vars, > clubb_datasets={folder_name:self.clubb_datasets[input_folder]}) 250,251c234 < e3sm_datasets=self.e3sm_datasets, cam_datasets=self.cam_file, priority_vars=self.priority_vars, < background_rcm=self.background_rcm, background_rcm_folder=self.background_rcm_folder) --- > e3sm_datasets=self.e3sm_datasets, cam_datasets=self.cam_file, priority_vars=self.priority_vars) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/src/Panel.py clubb_release/postprocessing/pyplotgen/src/Panel.py 34,36c34,35 < def __init__(self, plots, bkgrnd_rcm_tavg, altitude_bkgrnd_rcm, start_alt_idx, end_alt_idx, < panel_type="profile", title="Unnamed panel", dependent_title="dependent variable", sci_scale = None, < centered = False, background_rcm = False): --- > def __init__(self, plots, panel_type="profile", title="Unnamed panel", dependent_title="dependent variable", > sci_scale = None, centered = False): 41,44d39 < :param bkgrnd_rcm_tavg: Time-average vertical profile of rcm that can be displayed in the background of plots. < :param altitude_bkgrnd_rcm: Heights corresponding with the background rcm profile. < :param start_alt_idx: Index of bkgrnd_rcm_tavg that corresponds with bottom of the plot. < :param end_alt_idx: Index of bkgrnd_rcm_tavg that corresponds with top of the plot. 52d46 < :param background_rcm: Show a height-based "contour" plot of time-averaged rcm behind CLUBB profiles. 56,59d49 < self.bkgrnd_rcm_tavg = bkgrnd_rcm_tavg < self.altitude_bkgrnd_rcm = altitude_bkgrnd_rcm < self.start_alt_idx = start_alt_idx < self.end_alt_idx = end_alt_idx 67d56 < self.background_rcm = background_rcm 236,239c225 < if not self.background_rcm: < ax.legend(loc='center left', bbox_to_anchor=(1, 0.5)) < else: < ax.legend(loc='upper right', bbox_to_anchor=(1, 1)) --- > ax.legend(loc='center left', bbox_to_anchor=(1, 0.5)) 253,270d238 < < # Background rcm contour plot < if self.background_rcm: < num_points = self.end_alt_idx - self.start_alt_idx + 1 < bkgrnd_rcm_tavg_contours = np.eye( num_points ) < for i in range(num_points): < for j in range(num_points): < bkgrnd_rcm_tavg_contours[i,j] = self.bkgrnd_rcm_tavg[self.start_alt_idx+i] < min_value = min( self.bkgrnd_rcm_tavg ) < max_value = max( self.bkgrnd_rcm_tavg ) < x_vector_contour = np.zeros( num_points ) < x_diff = xlim[1] - xlim[0] < x_interval = x_diff / ( num_points - 1 ) < for k in range(num_points): < x_vector_contour[k] = xlim[0] + float(k) * x_interval < plt.contourf( x_vector_contour, self.altitude_bkgrnd_rcm[self.start_alt_idx:self.end_alt_idx+1], bkgrnd_rcm_tavg_contours, < vmin=min_value, vmax=2.0*max_value, cmap=plt.set_cmap("gist_yarg") ) < plt.colorbar( label="rcm [kg/kg]", orientation="vertical" ) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/postprocessing/pyplotgen/src/VariableGroup.py clubb_release/postprocessing/pyplotgen/src/VariableGroup.py 39c39 < priority_vars=False, background_rcm=False, background_rcm_folder=None): --- > priority_vars=False): 55,56d54 < :param background_rcm: Show a height-based "contour" plot of time-averaged rcm behind CLUBB profiles. < :param background_rcm_folder: Folder of CLUBB output data from which to pull the background rcm (if background_rcm is enabled). 80,81d77 < self.background_rcm = background_rcm < self.background_rcm_folder = background_rcm_folder 98,137d93 < if self.background_rcm: < if self.clubb_datasets is not None and len(self.clubb_datasets) != 0: < # Extract rcm from the zt NetCDF file. Also extract the time and height values to which the < # rcm data points correspond. < bkgrnd_rcm = np.squeeze( self.clubb_datasets[self.background_rcm_folder]['zt'].variables['rcm'] ) < self.altitude_bkgrnd_rcm = np.squeeze( self.clubb_datasets[self.background_rcm_folder]['zt'].variables['altitude'] ) < time_bkgrnd_rcm = np.squeeze( self.clubb_datasets[self.background_rcm_folder]['zt'].variables['time'] ) < # Find the indices in the rcm data that correspond to the start time and end time requested as the < # time-averaging interval for the case, as well as the minimum height and maximum height requested < # for the plots. < start_time_seconds = 60.0 * self.start_time # self.start_time is in minutes, while time_bkgrnd_rcm is in seconds. < end_time_seconds = 60.0 * self.end_time # self.end_time is in minutes, while time_bkgrnd_rcm is in seconds. < start_time_idx, end_time_idx = DataReader.__getStartEndIndex__(time_bkgrnd_rcm, start_time_seconds, end_time_seconds) < self.start_alt_idx, self.end_alt_idx = DataReader.__getStartEndIndex__(self.altitude_bkgrnd_rcm, self.height_min_value, self.height_max_value) < # Calculate the time-averaged vertical profile of rcm for use as contours in the background of plots < # of CLUBB time-averaged vertical profiles of various model fields. < nzt = np.shape(bkgrnd_rcm)[1] < self.bkgrnd_rcm_tavg = np.zeros(nzt) < for z_indx in range(nzt): < lev_sum = 0 < count = 0 < for t_indx in range(start_time_idx, end_time_idx): < lev_sum = lev_sum + bkgrnd_rcm[t_indx,z_indx] < count = count + 1 < self.bkgrnd_rcm_tavg[z_indx] = lev_sum / float(count) < else: < # Set the relevant "output" variables to a value just to have them set. < # They will be flagged out of interacting with the code. < self.bkgrnd_rcm_tavg = 0 < self.altitude_bkgrnd_rcm = 0 < self.start_alt_idx = 0 < self.end_alt_idx = 0 < else: < # Set the relevant "output" variables to a value just to have them set. < # They will be flagged out of interacting with the code. < self.bkgrnd_rcm_tavg = 0 < self.altitude_bkgrnd_rcm = 0 < self.start_alt_idx = 0 < self.end_alt_idx = 0 < 438,440c394,395 < panel = Panel(plotset, self.bkgrnd_rcm_tavg, self.altitude_bkgrnd_rcm, self.start_alt_idx, self.end_alt_idx, < title=title, dependent_title=axis_label, panel_type=panel_type, sci_scale=sci_scale, < centered=centered, background_rcm=self.background_rcm) --- > panel = Panel(plotset, title=title, dependent_title=axis_label, panel_type=panel_type, > sci_scale=sci_scale, centered=centered) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/run_scripts/run_bindiff_all.py clubb_release/run_scripts/run_bindiff_all.py 11c11,12 < DEBUG = False --- > scriptPath = os.path.realpath(__file__)[0:-18] > #print(scriptPath) 13,19d13 < scriptPath = os.path.dirname(os.path.realpath(__file__))+'/' < if DEBUG: < print(scriptPath) < outFilePath = os.path.normpath(scriptPath+"../output/bindiffs/") < filePostFix = "_diff{}.log" < < ## CONSTANTS ## 23,32d16 < # NetCDF file postfixes we want to diff < nc_data_formats = ["_zm.nc", "_zt.nc", "_sfc.nc"] < < # File containing all the case names < case_file = "RUN_CASES" < < # Define output messages: < # 1. Sum of absolute differences below threshold for all variables in all files for a single case. < no_value_diff_out = ">>>Differences in linux diff but no differences in common variables detected for case {}.<<<" < 35,48c19,23 < # Set up and parse command line arguments < parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) < parser.add_argument("-f", "--fileout", action="store", choices=["skip", "replace", "enumerate"], default=None, < help="Output a table of diffs into a text file, with one output file per each pair of diffed netcdf files.\n"+ < "The text files will be named '<input_file>-diff_out' and will be stored in a subfolder of CLUBB's output folder.\n"+ < "The choices for this option are:"+ < "\n1. Flag omitted: No files will be created."+ < "\n2. 'enumerate': If a diff log file for a case already exists\n\t\tthe new file will be created with an increasing numerical postfix."+ < "\n3. 'skip': If a diff log file for a case already exists the creation of the log file will be skipped."+ < "\n4. 'replace': If a diff log file for a case already exists the old file will be replaced with a new log file.") < parser.add_argument("-v", "--verbose", action="store_true", help="Print to console a table of numerical differences found in common variables.") < parser.add_argument("-t", "--threshold", dest="threshold", type=float, action="store", help="(float) Define the maximum absolute difference for an individual variable to be treated as different.") < parser.add_argument("dirs", nargs=2, help="Need 2 clubb output directories containing netCDF files with the same name to diff. Usage: python run_bindiff_all.py dir_path1 dir_path2") < args = parser.parse_args() --- > parser = argparse.ArgumentParser() > parser.add_argument("-o", "--output", action="store_true", help="add this option if you want to get output for each file compared put into a file made in the cwd.") > parser.add_argument("-d", "--diffs", action="store_true", help="print the differences found in variables") > parser.add_argument("-t", "--threshold", dest="threshold", type=float, action="store", help="print the differences found in variables") > parser.add_argument("dirs", nargs=2, help="need 2 clubb output directories to diff. Usage: python run_bindiff_all.py dir_path1 dir_path2") 50,51c25 < if DEBUG: < print(args) --- > args = parser.parse_args() 53d26 < # Check if folders exist 55c28 < --- > 61c34,40 < # Define difference detection threshold --- > dir1_files = os.listdir(args.dirs[0]) > dir2_files = os.listdir(args.dirs[1]) > > #print(dir1_files, dir2_files) > > cases = get_cases(dir1_files, dir2_files) > 69,85c48,55 < # Create folder in CLUBB output folder < if args.fileout: < if DEBUG: < print("Creating output folder") < if not os.path.exists(outFilePath): < os.makedirs(outFilePath) < < linux_diff, diff_in_files, file_skipped = find_diffs_in_all_files(args.dirs[0], args.dirs[1], args.fileout, args.verbose, tot_abs_diff_thresh) < < if DEBUG: < print(linux_diff, diff_in_files, file_skipped) < print("SUMMARY:") < if linux_diff: < if not diff_in_files: < print("Differences detected by linux diff but no differences in the common variables of the netCDF files compared.") < else: < print("There were differences detected in the common variables in netCDF (*.nc) files.") --- > diff_in_files = run_diff(cases, args.dirs[0], args.dirs[1], args.output, args.diffs, tot_abs_diff_thresh) > > if( args.output and diff_in_files ): > print("\nOutput files made and placed in " + os.getcwd()) > > if( diff_in_files ): > print("\nThere were differences detected in netCDF (*.nc) files.") > sys.exit(1) 87,93c57,58 < print("Linux diff did not detect any differences in the compared files.") < if args.fileout: < if diff_in_files: < print("Output files made and placed in " + outFilePath) < if file_skipped: < print("The creation of some or all of the output files was skipped because files with the same name already existed. "+ < "Use output options 'replace' or 'enumerate' to avoid this issue.") --- > if( args.output ): > print("\nNo differences detected, so no files were made!") 95,98c60 < print("No output files were created in " + outFilePath) < if diff_in_files: < sys.exit(1) < --- > print("\nNo differences detected!") 101c63,64 < sys.exit(2) --- > sys.exit(1) > 102a66 > def get_cases(dir1_files, dir2_files): 104,133c68,70 < def get_cases(dir1, dir2, l_verbose): < # Generate and return a dict of all cases that have files present in the diff folders < # This dict will have the following structure: < # cases = {<case> : <folder_dict>, ...} < # where folder_dict is another dict with two entries and the following structure: < # folder_dict = {<dir>: [list, of, nc_files], ...} < # where dir is either of the passed arguments and < # [list, to, nc_files] contains the names of the existing netCDF files for <case> in the same order as nc_data_formats. < # The entry will be None if a specific file does not exist. < # Example: cases = {'arm': { <dir1>: ['arm_zm.nc', None, None], <dir2>: ['arm_zm.nc', None, None]}, < # 'bomex': { <dir1>: ['bomex_zm.nc', 'bomex_zt.nc', 'bomex_sfc.nc']}, <dir2>: ['bomex_zm.nc', 'bomex_zt.nc', 'bomex_sfc.nc']}, < # 'wangara': { <dir1>: [None, 'wangara_zt.nc',None]}, <dir2>: ['wangara_zm.nc', None, None]} < # < # Warnings will be printed for each missing file both on the console and in the output files. < ##################################################################################### < < # Get lists of files in each folder < dir1_files = os.listdir(dir1) < dir2_files = os.listdir(dir2) < < if DEBUG: < print(dir1_files, dir2_files) < < # Create dict structure with case names as keys and a list for each folder of file names (or None if nonexistent) as values < # This dict is returned from this function and will later be used as a basis for the comparison loop in 'find_diffs_in_all_files' < cases = {} < < # Loop through all the case names listed in the RUN_CASES file and check if comparison is possible (i.e. at least one pair of case files is found in the folders) < # First, open the RUN_CASES file < with open(os.path.join(scriptPath, case_file)) as file: --- > cases = [] > > with open(scriptPath + "RUN_CASES") as file: 135,138d71 < if DEBUG: < print(line) < # Remove white space and the leading "!" in commented out lines < # since we even want to compare commented out cases! 141,210c74,82 < if DEBUG: < print(case) < # Skip line if empty < if case == "" or case == "!": < if DEBUG: < print("continue") < continue < # "Uncomment case name" < if case[0] == "!": < if DEBUG: < print("Commented") < case = case[1:].strip() < if DEBUG: < print(case) < # Skip line if it contains multiple words < if " " in case or "$" in case: < if DEBUG: < print("continue") < continue < if DEBUG: < print("Create entry") < cases[case] = {dir1: [], dir2: []} < < # We want to compare all cases that have at least one common file in the folders, i.e. add that case to dict cases with a list of comparable files. < # Note ANY missing files in a warning (also in log file) < if DEBUG: < print("iterating") < for postfix in nc_data_formats: < if DEBUG: < print(postfix) < ncfname = case+postfix < if DEBUG: < print(ncfname) < # Check if files exist in either folder < # Check dir1 < if ncfname in dir1_files: < if DEBUG: < print("Exists in " + dir1) < cases[case][dir1].append(ncfname) < else: < if DEBUG: < print("Not in " + dir1) < cases[case][dir1].append(None) < # Check dir2 < if ncfname in dir2_files: < if DEBUG: < print("Exists in " + dir2) < cases[case][dir2].append(ncfname) < else: < if DEBUG: < print("Not in " + dir2) < cases[case][dir2].append(None) < < # Remove case from list if no FILES exist < if not any(cases[case][dir1]) or not any(cases[case][dir2]): < if DEBUG: < print("Removing case because no files exist.") < print(cases[case]) < del cases[case] < if l_verbose: < print("Warning! No files found for case {}. It will be skipped.".format(case)) < # Remove case from list if no file PAIRS exist < elif not find_comparable_files(cases[case][dir1], cases[case][dir2]): < if DEBUG: < print("Removing case because there are no pairs.") < print(cases[case]) < del cases[case] < if l_verbose: < print("Warning! There are files for case {} but no pairs for comparison were found. It will be skipped.".format(case)) < print("\nThe following cases will be compared: {}\n".format(list(cases.keys()))) --- > > #If statement goes through every case in the RUN_CASES file and checks if the case is able to be run, > # and if that case has the files in the directories to diff it. If it passes all these conditions, > # the case is added to the runnable cases. > if(line[0] != "!" and len(line) > 1 and case + "_zt.nc" in dir1_files and case + "_zt.nc" in dir2_files > and case + "_zm.nc" in dir1_files and case + "_zm.nc" in dir2_files > and case + "_sfc.nc" in dir1_files and case + "_sfc.nc" in dir2_files): > cases.append(case) > #print(cases) 213,214d84 < def find_comparable_files(l1, l2): < return any([p1 and p2 for (p1,p2) in zip(l1,l2)]) 216,234c86,88 < def find_diffs_in_all_files(dir1, dir2, save_to_file, l_verbose, thresh): < # For each case with existing netCDF files in the diff folders: < # 1. Create an output file if those are requested < # 2. Loop through the netCDF files and call `find_diffs_in_common_vars` on each pair < # 3. The cumulative boolean `diff_in_all_files` stores whether or not any differences have been found < # < # Parameters: < # dir1, dir2: Paths to folders containing netCDF files we want to compare < # save_to_file: Value of the --outfile command line argument < # l_verbose: Value of the --verbose command line argument < # thresh: Detection threshold for differences in a variable < ##################################################################################### < file_missing_msg = "Warning! The file {} does not exist in {}. No comparison possible.\n" < < # Find all cases that need to be compared (all files present in both folders) < cases = get_cases(dir1, dir2, l_verbose) < diff_in_all_files = False < linux_diff = False < file_skipped = False --- > def run_diff(cases, dir1_path, dir2_path, l_output, l_diffs, thresh): > > diff_in_files = False 236,237c90,93 < # This for loop runs through all the cases you have the files to check, < # and each netcdf format. It runs the linux diff on the binary netcdf files --- > nc_data_formats = ["_zm.nc", "_zt.nc", "_sfc.nc"] > > # This for loop runs through all the cases you have the files to check, > # and each netcdf format. It runs the linux diff on the binary netcdf files 240,267c96,99 < print("###DIFFING " + case + " netCDF (*.nc) files###") < linux_diff_in_case = False < diff_in_case = False < < # Write any output that should be written to file to string 'content' < if save_to_file: < content = "Output for 'run_bindiff_all.py' of the folders:\n" < content += " - " + dir1 < content += "\n - " + dir2 < content += "\nDiffed case: " + case + "\n" < content += "********************************************************************************************\n" < < # Looping over all files associated to <case> < for i, postfix in enumerate(nc_data_formats): < ncfname = case+postfix < if not cases[case][dir1][i]: < if l_verbose: < print(file_missing_msg.format(ncfname, dir1)) < if save_to_file: < content += file_missing_msg.format(ncfname, dir1) < if not cases[case][dir2][i]: < if l_verbose: < print(file_missing_msg.format(ncfname, dir2)) < if save_to_file: < content += file_missing_msg.format(ncfname, dir2) < if not cases[case][dir1][i] or not cases[case][dir2][i]: < print(">File {} can not be compared since it does not exist both folders.<".format(ncfname)) < continue --- > > print("DIFFING " + case + " netCDF (*.nc) files") > > for out_form in nc_data_formats: 270,283c102,103 < if( len(subprocess.getoutput("diff " + dir1 + "/" + ncfname + " " + dir2 + "/" + ncfname)) > 0 ): < linux_diff_in_case = True < print(">The linux diff detected differences in " + ncfname + "<") < if save_to_file: < content += ">The linux diff detected differences in " + ncfname + "<\n" < # Update diff_in_all_files: if either it or the output of find_diffs_in_common_vars is true, we conclude that there is a difference. The printed messages will elaborate on the specific differences. < diff_in_case, new_content = find_diffs_in_common_vars(ncfname, dir1, dir2, save_to_file, l_verbose, thresh) or diff_in_case < if l_verbose: < print('') < if save_to_file: < content += new_content < content += '\n' < else: < print(">No differences detected by the linux diff in " + ncfname + "<") --- > if( len(subprocess.getoutput("diff " + dir1_path + "/" + case + out_form + " " + dir2_path + "/" + case + out_form)) > 0 ): > diff_in_files = run_py_diff(case+out_form, dir1_path, dir2_path, l_output, l_diffs, thresh) or diff_in_files 285,313c105 < if diff_in_case: < print(">>>Differences in common variables detected for case {}<<<".format(case)) < if save_to_file: < # Create file to save diff log for <case> into < diff_file_name = outFilePath + case + filePostFix < if DEBUG: < print(diff_file_name) < # Check if we need to create a file < if not os.path.exists(diff_file_name.format('')) or save_to_file=="replace": < # If file does not exist or should be overwritten, open file with default name and write content < with open(diff_file_name.format(''), "w") as caseLogFile: < caseLogFile.write(content) < elif save_to_file == "enumerate": < # (File exists already) First find which file index is next and create the file with that index in its name < i = 1 < diff_file_name = diff_file_name.format('{:02d}') < while os.path.exists(diff_file_name.format(i)): < i = i+1 < with open(diff_file_name.format(i), "w") as caseLogFile: < caseLogFile.write(content) < elif save_to_file == "skip": < file_skipped = True < if l_verbose: < print("Warning! Despite detected differences no log file was created for case {} since the file {} already exists and the 'skip' output option was used ".format(case, diff_file_name.format(''))) < elif linux_diff_in_case: < print(no_value_diff_out.format(case)) < else: < print(">>>The linux diff could not detect any differences in the file pairs for case {}.<<<".format(case)) < print('**********************************************************************************************************') --- > return diff_in_files 315,339d106 < # Update diff_in_all_files: Set to True if diff_in_case==True (differences in case found) < linux_diff = linux_diff_in_case or linux_diff < diff_in_all_files = diff_in_case or diff_in_all_files < < return (linux_diff, diff_in_all_files, file_skipped) < < def find_diffs_in_common_vars( test_file, dir1, dir2, save_to_file, l_verbose, tot_abs_diff_thresh ): < # This is the integral function of this script! < # Compare content of one specific pair of files with the same name in each folder: < # 1. Find the variables that are present in only one of the files < # 2. Find common variables and iterate through them < # 3. Compare values between the files and detect differences < # 4. Print findings < # 5. Return TRUE if significant differences were detected and FALSE otherwise < ##################################################################################### < < # Assume no differences until proven otherwise < diff_in_common_vars = False < # Declare string containing all the output for <test_file> < new_content = "" < < if DEBUG: < print("DIFFING " + test_file + "netCDF (*.nc) files") < print(os.path.join(dir1, test_file)) < print(os.path.join(dir2, test_file)) 341,343c108 < # Create datasets from nc files < dset1 = netCDF4.Dataset(os.path.join(dir1, test_file)) < dset2 = netCDF4.Dataset(os.path.join(dir2, test_file)) --- > def run_py_diff( test_file, dir1_path, dir2_path, l_save_output, l_print_diffs, tot_abs_diff_thresh ): 345,365c110,119 < # Find variables that are only present in ONE of the files < diff1 = set(dset1.variables.keys()).difference(dset2.variables.keys()) < diff2 = set(dset2.variables.keys()).difference(dset1.variables.keys()) < # Print those variables < if diff1: < var_set_diff_out = "{} contains the following extra variables:".format(os.path.join(dir1, test_file)) < var_set_diff_out = var_set_diff_out + "\n" + "\n".join(diff1) < print(var_set_diff_out) < if save_to_file: < new_content += var_set_diff_out + "\n" < < if diff2: < var_set_diff_out = "{} contains the following extra variables:".format(os.path.join(dir2, test_file)) < var_set_diff_out = var_set_diff_out + "\n" + "\n".join(diff2) < print(var_set_diff_out) < if save_to_file: < new_content += var_set_diff_out + "\n" < if diff1 or diff2: < print('') < if save_to_file: < new_content += "\n" --- > diff_in_files = False > any_diff_above_thresh = False > > #print("DIFFING " + test_file + "netCDF (*.nc) files") > #print(dir1_path + test_file) > #print(dir2_path + test_file) > > # Create datasets from nc files > dset1 = netCDF4.Dataset(dir1_path+"/"+test_file) > dset2 = netCDF4.Dataset(dir2_path+"/"+test_file) 368,371c122,125 < table = [[ 'Var', < 'Max Abs Diff', < 'Max % Diff', < 'Total Abs Diff', --- > table = [[ 'Var', > 'Max Abs Diff', > 'Max % Diff', > 'Total Abs Diff', 374,378c128,130 < # Create set of variables that are common to both files < # These are the variables we can and want to compare the other variables are printed above < vars_in_common = set(dset1.variables.keys()).intersection(dset2.variables.keys()) < for var in sorted(vars_in_common): < # The CLUBB netcdf variables we are interested in are all --- > for var in dset1.variables: > > # The CLUBB netcdf variables we are interested in are all 382c134 < # dimensions (unlikely), then the variables of interested would be 2D. So --- > # dimensions (unlikely), then the variables of interested would be 2D. So 384c136,138 < if( dset1[var].ndim > 1 and dset2[var].ndim > 1 ): --- > if( dset1[var].ndim > 1 ): > > diff_in_files = True 389c143 < if ( np.sum(abs_diff) <= tot_abs_diff_thresh ): --- > if ( np.sum(abs_diff) < tot_abs_diff_thresh ): 392c146 < diff_in_common_vars = True --- > any_diff_above_thresh = True 401,406c155,160 < < # Append the table array with the values we want to print < table.append( [ var, < np.max(abs_diff), < np.max(percent_diff), < np.sum(abs_diff), --- > > # Append the table array with the values we want to print > table.append( [ var, > np.max(abs_diff), > np.max(percent_diff), > np.sum(abs_diff), 408,410d161 < else: < if l_verbose: < print("The variable {} is not comparable because its dimension is not fit for comparison: {}, {}".format(var, dset1[var].ndim, dset2[var].ndim)) 411a163,165 > # Print results > header = "************** Differences detected in " + test_file + "! **************" > print( header ) 413c167 < if diff_in_common_vars: --- > if( any_diff_above_thresh ): 418c172 < if l_verbose: --- > if( l_print_diffs ): 422c176 < if save_to_file: --- > if( l_save_output ): 424c178,181 < new_content += output + "\n" --- > f = open(test_file + "-diff_out", "w") > f.write(header) > f.write(output) > f.close() 426,430c183 < if not diff_in_common_vars: < print(">>No differences detected in the common fields in file " + test_file + "<<") < else: < print(">>Differences above threshold were detected in the common fields in file " + test_file + "<<") < return (diff_in_common_vars, new_content) --- > return diff_in_files diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/run_scripts/RUN_CASES clubb_release/run_scripts/RUN_CASES 2,4c2,4 < ! Declare any cases that should be run with run_scm_all.bash and run_bindiff_all.bash. < ! Any blank lines and lines starting with '!' will be ignored in run_scm_all < ! while run_bindiff_all runs all cases regardless if commented or not. --- > ! Declare any cases that should be run with run_scm_all.bash and > ! run_bindiff_all.bash. Any blank lines and lines starting with '!' will > ! be ignored. diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/run_scripts/run_scm.bash clubb_release/run_scripts/run_scm.bash 539,540c539 < mv ../output/*.nc $CUSTOM_OUTPUT_DIR < mv ../output/*_setup.txt $CUSTOM_OUTPUT_DIR --- > mv ../output/* $CUSTOM_OUTPUT_DIR diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/src/CLUBB_core/advance_clubb_core_module.F90 clubb_release/src/CLUBB_core/advance_clubb_core_module.F90 365,368c365 < ibrunt_vaisala_freq_sqd_mixed, & < ibrunt_vaisala_freq_sqd_moist, & < ibrunt_vaisala_freq_sqd_dry, & < iRi_zm --- > isqrt_Ri_zm 817,820c814,817 < brunt_vaisala_freq_sqd_mixed, & ! A mixture of dry and moist N^2 [s^-2] < brunt_vaisala_freq_sqd_dry, & ! dry N^2 [s^-2] < brunt_vaisala_freq_sqd_moist, & ! moist N^2 [s^-2] < brunt_vaisala_freq_sqd_splat, & ! [s^-2] --- > brunt_vaisala_freq_sqd_mixed, & ! A mixture of dry and moist N^2 > brunt_vaisala_freq_sqd_dry, & ! dry N^2 > brunt_vaisala_freq_sqd_moist, & ! moist N^2 > brunt_vaisala_freq_sqd_splat, & 822c819 < Ri_zm ! Richardson number [-] --- > sqrt_Ri_zm ! square root of Richardson number 989,990c986,987 < !$acc brunt_vaisala_freq_sqd_splat, & < !$acc brunt_vaisala_freq_sqd_zt, Ri_zm, Lscale_max, & --- > !$acc brunt_vaisala_freq_sqd_plus, brunt_vaisala_freq_sqd_splat, & > !$acc brunt_vaisala_freq_sqd_zt, sqrt_Ri_zm, Lscale_max, & 1555d1551 < invrs_tau_wp3_zm(i,k) = invrs_tau_zm(i,k) 1594c1590 < Ri_zm, & ! Out --- > sqrt_Ri_zm, & ! Out 1833c1829 < !$acc Ri_zm, invrs_tau_wp3_zm, invrs_tau_no_N2_zm, invrs_tau_bkgnd, & --- > !$acc sqrt_Ri_zm, invrs_tau_wp3_zm, invrs_tau_no_N2_zm, invrs_tau_bkgnd, & 1835,1836c1831 < !$acc brunt_vaisala_freq_sqd_splat, brunt_vaisala_freq_sqd_mixed, & < !$acc brunt_vaisala_freq_sqd_moist, brunt_vaisala_freq_sqd_dry ) --- > !$acc brunt_vaisala_freq_sqd_splat ) 1848c1843 < call stat_update_var(iRi_zm, Ri_zm(i,:), & ! intent(in) --- > call stat_update_var(isqrt_Ri_zm, sqrt_Ri_zm(i,:), & ! intent(in) 1861a1857,1858 > call stat_update_var(ibrunt_vaisala_freq_sqd, brunt_vaisala_freq_sqd(i,:), & ! intent(in) > stats_zm(i)) ! intent(inout) 1863,1864d1859 < call stat_update_var(ibrunt_vaisala_freq_sqd, brunt_vaisala_freq_sqd(i,:), & ! intent(in) < stats_zm(i)) 1866,1872c1861 < stats_zm(i)) ! intent(inout) < call stat_update_var(ibrunt_vaisala_freq_sqd_mixed, brunt_vaisala_freq_sqd_mixed(i,:), & ! intent(in) < stats_zm(i)) ! intent(inout) < call stat_update_var(ibrunt_vaisala_freq_sqd_moist, brunt_vaisala_freq_sqd_moist(i,:), & ! intent(in) < stats_zm(i)) ! intent(inout) < call stat_update_var(ibrunt_vaisala_freq_sqd_dry, brunt_vaisala_freq_sqd_dry(i,:), & ! intent(in) < stats_zm(i)) ! intent(inout) --- > stats_zm(i)) 2913,2914c2902,2903 < !$acc brunt_vaisala_freq_sqd_splat, & < !$acc brunt_vaisala_freq_sqd_zt, Ri_zm, Lscale_max, & --- > !$acc brunt_vaisala_freq_sqd_plus, brunt_vaisala_freq_sqd_splat, & > !$acc brunt_vaisala_freq_sqd_zt, sqrt_Ri_zm, Lscale_max, & diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/src/CLUBB_core/advance_helper_module.F90 clubb_release/src/CLUBB_core/advance_helper_module.F90 335c335 < !$acc lambda0_stability, Lscale_zm ) --- > !$acc brunt_vaisala_freq_sqd_plus, lambda0_stability, Lscale_zm ) 601c601,602 < ishear_sqd, & ! Variable(s) --- > iRichardson_num, & ! Variable(s) > ishear_sqd, & 617c618 < l_Cx_fnc_Richardson_vert_avg = .false. ! Vertically average Cx_fnc_Richardson over a --- > l_Cx_fnc_Richardson_vert_avg = .false., & ! Vertically average Cx_fnc_Richardson over a 618a620,623 > l_Richardson_vert_avg = .false., & ! Vertically average Richardson_num over a > ! distance of Lscale > l_use_shear_turb_freq_sqd = .false. ! Use turb_freq_sqd and shear_sqd in denominator of > ! Richardson_num 676a682 > Richardson_num, & 683a690 > turb_freq_sqd, & 685a693 > Richardson_num_clipped, & 706,708c714,716 < !$acc Cx_fnc_interp, & < !$acc Ri_zm, ddzt_um, ddzt_vm, shear_sqd, Lscale_zm, & < !$acc Cx_fnc_Richardson_avg, fnc_Richardson, & --- > !$acc Richardson_num, Cx_fnc_interp, & > !$acc Ri_zm, ddzt_um, ddzt_vm, shear_sqd, turb_freq_sqd, Lscale_zm, & > !$acc Richardson_num_clipped, Cx_fnc_Richardson_avg, fnc_Richardson, & 710a719,723 > if ( l_modify_limiters_for_cnvg_test .and. l_use_shear_turb_freq_sqd ) then > error stop "ERROR: l_modify_limiters_for_cnvg_test .and. l_use_shear_turb_freq_sqd "// & > "both true in compute_Cx_fnc_Richardson" > end if > 744,752c757 < if ( l_stats_samp ) then < !$acc update host(shear_sqd) < do i = 1, ngrdcol < call stat_update_var( ishear_sqd, shear_sqd(i,:), & ! intent(in) < stats_zm(i) ) ! intent(inout) < end do < end if < < if ( l_use_shear_Richardson ) then --- > if ( l_use_shear_turb_freq_sqd ) then 757,758c762,764 < Ri_zm(i,k) = max( 1.0e-7_core_rknd, brunt_vaisala_freq_sqd_mixed(i,k) ) & < / max( shear_sqd(i,k), 1.0e-7_core_rknd ) --- > turb_freq_sqd(i,k) = em(i,k) / Lscale_zm(i,k)**2 > Richardson_num(i,k) = brunt_vaisala_freq_sqd(i,k) / max( shear_sqd(i,k), turb_freq_sqd(i,k), & > Richardson_num_divisor_threshold ) 762a769,776 > if ( l_stats_samp ) then > !$acc update host(shear_sqd) > do i = 1, ngrdcol > call stat_update_var( ishear_sqd, shear_sqd(i,:), & ! intent(in) > stats_zm(i) ) ! intent(inout) > end do > end if > 763a778,807 > > if ( l_use_shear_Richardson ) then > > !$acc parallel loop gang vector collapse(2) default(present) > do k = 1, nz > do i = 1, ngrdcol > Richardson_num(i,k) = brunt_vaisala_freq_sqd_mixed(i,k) * invrs_num_div_thresh > > Ri_zm(i,k) = max( 1.0e-7_core_rknd, brunt_vaisala_freq_sqd_mixed(i,k) ) & > / max( shear_sqd(i,k), 1.0e-7_core_rknd ) > end do > end do > !$acc end parallel loop > > else > !$acc parallel loop gang vector collapse(2) default(present) > do k = 1, nz > do i = 1, ngrdcol > Richardson_num(i,k) = brunt_vaisala_freq_sqd(i,k) * invrs_num_div_thresh > Ri_zm(i,k) = Richardson_num(i,k) > end do > end do > !$acc end parallel loop > endif > > end if > > if ( l_Richardson_vert_avg ) then > > ! Clip below-min values of Richardson_num 767c811 < Ri_zm(i,k) = brunt_vaisala_freq_sqd(i,k) * invrs_num_div_thresh --- > Richardson_num_clipped(i,k) = max( Richardson_num(i,k), Richardson_num_min ) 770a815,818 > > Richardson_num = Lscale_width_vert_avg( nz, ngrdcol, gr, smth_type, & > Richardson_num, Lscale_zm, rho_ds_zm, & > Richardson_num_max ) 840a889,897 > ! Stats sampling > if ( l_stats_samp ) then > !$acc update host(Richardson_num) > do i = 1, ngrdcol > call stat_update_var( iRichardson_num, Richardson_num(i,:), & ! intent(in) > stats_zm(i) ) ! intent(inout) > end do > end if > 843,845c900,902 < !$acc Cx_fnc_interp, Ri_zm, & < !$acc ddzt_um, ddzt_vm, shear_sqd, Lscale_zm, & < !$acc Cx_fnc_Richardson_avg, fnc_Richardson, & --- > !$acc brunt_vaisala_freq_sqd_plus, Richardson_num, Cx_fnc_interp, & > !$acc Ri_zm, ddzt_um, ddzt_vm, shear_sqd, turb_freq_sqd, Lscale_zm, & > !$acc Richardson_num_clipped, Cx_fnc_Richardson_avg, fnc_Richardson, & diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/src/CLUBB_core/mixing_length.F90 clubb_release/src/CLUBB_core/mixing_length.F90 1258c1258 < Ri_zm, & ! intent out --- > sqrt_Ri_zm, & ! intent out 1369c1369 < ! (reduce threshold on limiters for Ri_zm in mixing_length.F90) --- > ! (reduce threshold on limiters for sqrt_Ri_zm in mixing_length.F90) 1379c1379 < Ri_zm, & --- > sqrt_Ri_zm, & 1444,1445c1444,1445 < Ri_zm_clipped, & < Ri_zm_smooth, & --- > sqrt_Ri_zm_clipped, & > sqrt_Ri_zm_smooth, & 1465,1466c1465,1466 < !$acc create( Ri_zm_clipped, ddzt_umvm_clipped, & < !$acc tau_zm_unclipped, tau_zt_unclipped, Ri_zm_smooth, em_clipped, & --- > !$acc create( sqrt_Ri_zm_clipped, ddzt_umvm_clipped, & > !$acc tau_zm_unclipped, tau_zt_unclipped, sqrt_Ri_zm_smooth, em_clipped, & 1586c1586 < brunt_vaisala_freq_sqd_smth = zm2zt2zm( nz,ngrdcol,gr, brunt_vaisala_freq_sqd_mixed ) --- > brunt_vaisala_freq_sqd_smth = zm2zt2zm( nz,ngrdcol,gr, brunt_vaisala_freq_sqd ) 1595c1595 < tmp_calc(i,k) = 1.e8_core_rknd * abs(brunt_vaisala_freq_sqd_mixed(i,k))**3 --- > tmp_calc(i,k) = 1.e8_core_rknd * abs(brunt_vaisala_freq_sqd(i,k))**3 1601c1601 < brunt_vaisala_freq_sqd_mixed, & --- > brunt_vaisala_freq_sqd, & 1612,1613c1612,1613 < brunt_vaisala_freq_clipped(i,k) = min( brunt_vaisala_freq_sqd_mixed(i,k), & < 1.e8_core_rknd * abs(brunt_vaisala_freq_sqd_mixed(i,k))**3) --- > brunt_vaisala_freq_clipped(i,k) = min( brunt_vaisala_freq_sqd(i,k), & > 1.e8_core_rknd * abs(brunt_vaisala_freq_sqd(i,k))**3) 1629,1630c1629,1630 < Ri_zm_clipped(i,k) = max( 0.0_core_rknd, brunt_vaisala_freq_sqd_smth(i,k) ) & < / max( ddzt_umvm(i,k), 1.0e-12_core_rknd ) --- > sqrt_Ri_zm_clipped(i,k) = sqrt( max( 0.0_core_rknd, brunt_vaisala_freq_sqd_smth(i,k) ) & > / max( ddzt_umvm(i,k), 1.0e-12_core_rknd) ) 1635c1635 < Ri_zm = zm2zt2zm( nz, ngrdcol, gr, Ri_zm_clipped ) --- > sqrt_Ri_zm = zm2zt2zm( nz, ngrdcol, gr, sqrt_Ri_zm_clipped ) 1650c1650 < Ri_zm(i,k) = brunt_vaisala_freq_clipped(i,k) / ddzt_umvm_clipped(i,k) --- > sqrt_Ri_zm(i,k) = sqrt( brunt_vaisala_freq_clipped(i,k) / ddzt_umvm_clipped(i,k) ) 1660,1661c1660,1661 < Ri_zm(i,k) = max( 1.0e-7_core_rknd, brunt_vaisala_freq_sqd_smth(i,k) ) & < / max( ddzt_umvm(i,k), 1.0e-7_core_rknd ) --- > sqrt_Ri_zm(i,k) = sqrt( max( 1.0e-7_core_rknd, brunt_vaisala_freq_sqd_smth(i,k) ) & > / max( ddzt_umvm(i,k), 1.0e-7_core_rknd) ) 1844c1844 < invrs_tau_xp2_zm(i,k) = invrs_tau_bkgnd(i,k) + invrs_tau_sfc(i,k) & --- > invrs_tau_xp2_zm(i,k) = 0.1_core_rknd * invrs_tau_bkgnd(i,k) + invrs_tau_sfc(i,k) & 1852,1861c1852,1861 < ! !$acc parallel loop gang vector collapse(2) default(present) < ! do k = 1, nz < ! do i = 1, ngrdcol < ! if ( ice_supersat_frac_zm(i,k) <= 0.01_core_rknd & < ! .and. invrs_tau_xp2_zm(i,k) >= 0.003_core_rknd ) then < ! invrs_tau_xp2_zm(i,k) = 0.003_core_rknd < ! end if < ! end do < ! end do < ! !$acc end parallel loop --- > !$acc parallel loop gang vector collapse(2) default(present) > do k = 1, nz > do i = 1, ngrdcol > if ( ice_supersat_frac_zm(i,k) <= 0.01_core_rknd & > .and. invrs_tau_xp2_zm(i,k) >= 0.003_core_rknd ) then > invrs_tau_xp2_zm(i,k) = 0.003_core_rknd > end if > end do > end do > !$acc end parallel loop 1903,1904c1903,1904 < Ri_zm_clipped = smooth_max( nz, ngrdcol, Ri_zm, zero, & < 12.0_core_rknd * min_max_smth_mag ) --- > sqrt_Ri_zm_clipped = smooth_max( nz, ngrdcol, sqrt_Ri_zm, zero, & > 12.0_core_rknd * min_max_smth_mag ) 1906,1907c1906,1907 < Ri_zm_smooth = smooth_min( nz, ngrdcol, Ri_zm_clipped, 12.0_core_rknd, & < 12.0_core_rknd * min_max_smth_mag ) --- > sqrt_Ri_zm_smooth = smooth_min( nz, ngrdcol, sqrt_Ri_zm_clipped, 12.0_core_rknd, & > 12.0_core_rknd * min_max_smth_mag ) 1916c1916 < * C_invrs_tau_wpxp_Ri * sqrt ( Ri_zm_smooth(i,k) )) --- > * C_invrs_tau_wpxp_Ri * sqrt_Ri_zm_smooth(i,k) ) 1932c1932 < * min( max( sqrt( Ri_zm(i,k) ), zero), 12.0_core_rknd ) ) --- > * min( max( sqrt_Ri_zm(i,k), zero), 12.0_core_rknd ) ) 2042,2043c2042,2043 < !$acc delete( Ri_zm_clipped, ddzt_umvm_clipped, & < !$acc tau_zm_unclipped, tau_zt_unclipped, Ri_zm_smooth, em_clipped, & --- > !$acc delete( sqrt_Ri_zm_clipped, ddzt_umvm_clipped, & > !$acc tau_zm_unclipped, tau_zt_unclipped, sqrt_Ri_zm_smooth, em_clipped, & diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/src/CLUBB_core/model_flags.F90 clubb_release/src/CLUBB_core/model_flags.F90 604c604 < l_use_tke_in_wp3_pr_turb_term = .true. --- > l_use_tke_in_wp3_pr_turb_term = .false. diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/src/CLUBB_core/stats_variables.F90 clubb_release/src/CLUBB_core/stats_variables.F90 1089,1092c1089,1090 < ibrunt_vaisala_freq_sqd_mixed = 0, & < ibrunt_vaisala_freq_sqd_moist = 0, & < ibrunt_vaisala_freq_sqd_dry = 0, & < iRi_zm = 0, & --- > isqrt_Ri_zm = 0, & > iRichardson_num = 0, & 1097c1095 < !$omp threadprivate(iC7_Skw_fnc, iC1_Skw_fnc, iRi_zm) --- > !$omp threadprivate(iC7_Skw_fnc, iC1_Skw_fnc, isqrt_Ri_zm) 1099,1100c1097 < !$omp threadprivate(ibrunt_vaisala_freq_sqd_mixed, ibrunt_vaisala_freq_sqd_moist) < !$omp threadprivate(ibrunt_vaisala_freq_sqd_dry, ishear_sqd) --- > !$omp threadprivate(iRichardson_num, ishear_sqd) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/src/CLUBB_core/stats_zm_module.F90 clubb_release/src/CLUBB_core/stats_zm_module.F90 52c52 < iRi_zm, & --- > isqrt_Ri_zm, & 323,325c323 < ibrunt_vaisala_freq_sqd_mixed, & < ibrunt_vaisala_freq_sqd_moist, & < ibrunt_vaisala_freq_sqd_dry, & --- > iRichardson_num, & 2514,2534c2512,2515 < case ( 'bv_freq_sqd_mixed' ) < ibrunt_vaisala_freq_sqd_mixed = k < call stat_assign( var_index=ibrunt_vaisala_freq_sqd_mixed, var_name="bv_freq_sqd_mixed", & < var_description="Interpolated Brunt-Vaisala freq. squared between moist and dry air", & < var_units="1/s^2", & < l_silhs=.false., grid_kind=stats_zm ) < k = k + 1 < < case ( 'bv_freq_sqd_moist' ) < ibrunt_vaisala_freq_sqd_moist = k < call stat_assign( var_index=ibrunt_vaisala_freq_sqd_moist, var_name="bv_freq_sqd_moist", & < var_description="Brunt-Vaisala freq. squared in moist air", & < var_units="1/s^2", & < l_silhs=.false., grid_kind=stats_zm ) < k = k + 1 < < case ( 'bv_freq_sqd_dry' ) < ibrunt_vaisala_freq_sqd_dry = k < call stat_assign( var_index=ibrunt_vaisala_freq_sqd_dry, var_name="bv_freq_sqd_dry", & < var_description="Brunt-Vaisala freq. squared in dry air", & < var_units="1/s^2", & --- > case ( 'sqrt_Ri_zm' ) > isqrt_Ri_zm = k > call stat_assign( var_index=isqrt_Ri_zm,var_name="sqrt_Ri_zm", & > var_description="Richardson number [-]", var_units="-", & 2538,2541c2519,2522 < case ( 'Ri_zm' ) < iRi_zm = k < call stat_assign( var_index=iRi_zm,var_name="Ri_zm", & < var_description="Richardson number [-]", var_units="-", & --- > case ( 'Richardson_num' ) > iRichardson_num = k > call stat_assign( var_index=iRichardson_num, var_name="Richardson_num", & > var_description="Richardson_num, Richardson number", var_units="-", & diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/src/error.F90 clubb_release/src/error.F90 111c111 < character(len=20), dimension(:), allocatable, private :: & --- > character(len=10), dimension(:), allocatable, private :: & 290c290 < character(len=20), dimension(max_variables) :: & --- > character(len=10), dimension(max_variables) :: & 376c376 < t_variables(1:max_variables) = " " --- > t_variables(1:max_variables) = " " 423c423 < if (t_variables(i) == " ") exit --- > if (t_variables(i) == " ") exit diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/utilities/sens_matrix/sens_matrix_dashboard.py clubb_release/utilities/sens_matrix/sens_matrix_dashboard.py 11,14c11,14 < from dash import dcc < #import dash_core_components as dcc < from dash import html < #import dash_html_components as html --- > #from dash import dcc > import dash_core_components as dcc > #from dash import html > import dash_html_components as html 71,72d70 < #print("metricsNames=", metricsNames) < #print("obsMetricValsCol=", obsMetricValsCol) 112,113c110 < # defaultBiasesCol + prescribedBiasesCol = -fwdFnc_tuned_params (see objFnc). < # This lumps the prescribed-parameter adjustment into defaultBiasesCol --- > # defaultBiasesCol + prescribedBiasesCol = -fwdFnc_tuned_params (see objFnc) 143,144d139 < normlzdWeightedLinplusSensMatrixPoly = np.diag(np.transpose(metricsWeights)[0]) @ normlzdLinplusSensMatrixPoly < 231c226 < # For use in figures such as paramsErrorBarsFig. --- > # For use in figures such as paramsBarChart. 244a240,250 > > # Set up a column vector of metric values from the default simulation > defaultMetricValsCol = setupDefaultMetricValsCol(metricsNames, defaultNcFilename) > > # Set up a column vector of metric values from the global simulation based on optimized > # parameter values. > linSolnMetricValsCol = setupDefaultMetricValsCol(metricsNames, linSolnNcFilename) > > # Store biases in default simulation, ( global_model - default ) > linSolnBiasesCol = np.subtract(linSolnMetricValsCol, defaultMetricValsCol) > 248d253 < normlzdWeightedDefaultBiasesCol = metricsWeights * normlzdDefaultBiasesCol 273,299d277 < # Set up a column vector of metric values from the global simulation based on optimized < # parameter values. < linSolnMetricValsCol = setupDefaultMetricValsCol(metricsNames, linSolnNcFilename) < < # Store biases in default simulation, ( global_model - obs ) < linSolnBiasesCol = np.subtract(linSolnMetricValsCol, obsMetricValsCol) < #linSolnBiasesCol = linSolnBiasesCol + prescribedBiasesCol < < # Check whether the minimizer actually reduces chisqd < # Initial value of chisqd, which assumes parameter perturbations are zero < normlzdLinSolnBiasesCol = linSolnBiasesCol/np.abs(normMetricValsCol) < chisqdLinSolnMin = objFnc(np.zeros_like(defaultParamValsOrigRow), \ < normlzdSensMatrixPoly, normlzdLinSolnBiasesCol, metricsWeights, \ < normlzdCurvMatrix, reglrCoef, numMetrics) < < #print("chisqdZero =", chisqdZero) < #print("chisqdMin =", chisqdMin) < print("chisqdLinSolnMinRatio =", chisqdLinSolnMin/chisqdZero) < < chisqdUnweightedLinSolnMin = objFnc(np.zeros_like(defaultParamValsOrigRow), \ < normlzdSensMatrixPoly, normlzdLinSolnBiasesCol, np.ones_like(metricsWeights), \ < normlzdCurvMatrix, reglrCoef, numMetrics) < < #print("chisqdUnweightedZero =", chisqdUnweightedZero) < #print("chisqdUnweightedMin =", chisqdUnweightedMin) < print("chisqdUnweightedLinSolnMinRatio =", chisqdUnweightedLinSolnMin/chisqdUnweightedZero) < print("-----------------------------------------------------") 344c322 < #linSolnBias = ( linSolnBiasesCol + defaultBiasesCol ) --- > linSolnBias = ( linSolnBiasesCol + defaultBiasesCol ) 346,347c324,325 < #linSolnBiasMagRatio = np.linalg.norm(linSolnBias/np.abs(normMetricValsCol))**2 / \ < # np.linalg.norm(defaultBiasesCol/np.abs(normMetricValsCol))**2 --- > linSolnBiasMagRatio = np.linalg.norm(linSolnBias/np.abs(normMetricValsCol))**2 / \ > np.linalg.norm(defaultBiasesCol/np.abs(normMetricValsCol))**2 394,395c372,373 < #weightedBiasLinSoln = metricsWeights * ( linSolnBiasesCol + defaultBiasesCol ) / np.abs(normMetricValsCol) < #weightedBiasLinSolnMagRatio = np.linalg.norm(weightedBiasLinSoln)**2 / np.linalg.norm(normlzdMDeltaB)**2 --- > weightedBiasLinSoln = metricsWeights * ( linSolnBiasesCol + defaultBiasesCol ) / np.abs(normMetricValsCol) > weightedBiasLinSolnMagRatio = np.linalg.norm(weightedBiasLinSoln)**2 / np.linalg.norm(normlzdMDeltaB)**2 531,532c509 < #y= (-linSolnBiasesCol-defaultBiasesCol)[metricsSensOrdered[i],0]/np.abs(normMetricValsCol[metricsSensOrdered[i],0]), < y= (-linSolnBiasesCol)[metricsSensOrdered[i],0]/np.abs(normMetricValsCol[metricsSensOrdered[i],0]), --- > y= (-linSolnBiasesCol-defaultBiasesCol)[metricsSensOrdered[i],0]/np.abs(normMetricValsCol[metricsSensOrdered[i],0]), 638,642d614 < < #wide_df = px.data.medals_wide() < #paramsTotContrbBarFig = px.bar(wide_df, x="nation", y=["gold", "silver", "bronze"], base=[0,0,0]) < #pdb.set_trace() < 645,646d616 < # barBase = np.zeros(numMetrics), < #barBase = -defaultBiasesCol[metricsSensOrdered]/np.abs(normMetricValsCol[metricsSensOrdered]) @ np.ones((1,len(paramsNames))), 652,659c622 < # paramsTotContrbBarFig = \ < # createBarChart( minusNonlinMatrixOrdered.T, index=paramsNames, columns=metricsNamesOrdered, < # orientation = 'v', < # title="""Linear + nonlinear contributions of parameters to actual removal of regional biases""", < # xlabel="Parameter", ylabel="Contribution to bias removal", < # width=800, height=500 ) < < sensMatrixBarFig = \ --- > paramsTotContrbBarFig = \ 661,662d623 < #createBarChart( normlzdSensMatrixOrdered.T, index=paramsNames, columns=metricsNamesOrdered, < # barBase = np.zeros_like(paramsScales), 664,672c625 < title="""Linear+nonlinear contributions of parameters to actual removal of regional biases""", < xlabel="Parameter", ylabel="Contribution to bias removal", < width=800, height=500 ) < < linplusSensMatrixBarFig = \ < createBarChart( normlzdLinplusSensMatrixPoly[metricsSensOrdered,:].T, index=paramsNames, columns=metricsNamesOrdered, < # barBase = np.zeros_like(paramsScales), < orientation = 'v', < title="""Linplus contributions of parameters to actual removal of regional biases""", --- > title="""Linear + nonlinear contributions of parameters to actual removal of regional biases""", 676,683d628 < metricsCorrArrayFig = createCorrArrayFig( normlzdLinplusSensMatrixPoly, metricsNames, < title='cos(angle) among metrics (i.e., rows of sens matrix)' ) < < minusNormlzdDefaultBiasesCol = \ < -defaultBiasesCol[metricsSensOrdered,0]/np.abs(normMetricValsCol[metricsSensOrdered,0]) < metricsBarChart = createMetricsBarChart(metricsNames[metricsSensOrdered],paramsNames, < minusNormlzdDefaultBiasesCol,minusNonlinMatrixOrdered) < 1066c1011 < paramsErrorBarsFig = px.scatter(df, x=df.index, y=df.columns, --- > paramsBarChart = px.scatter(df, x=df.index, y=df.columns, 1069c1014 < paramsErrorBarsFig.update_traces( go.Scatter( --- > paramsBarChart.update_traces( go.Scatter( 1074c1019 < #paramsErrorBarsFig.add_trace(go.Scatter(x=paramsNames, y=paramsLowValsPCBound[:,0]*paramsScales, --- > #paramsBarChart.add_trace(go.Scatter(x=paramsNames, y=paramsLowValsPCBound[:,0]*paramsScales, 1077c1022 < #paramsErrorBarsFig.add_trace(go.Scatter(x=paramsNames, y=paramsHiValsPCBound[:,0]*paramsScales, fill='tonexty', --- > #paramsBarChart.add_trace(go.Scatter(x=paramsNames, y=paramsHiValsPCBound[:,0]*paramsScales, fill='tonexty', 1080c1025 < #paramsErrorBarsFig.add_trace(go.Scatter(x=paramsNames, y=defaultParamValsOrigRow[0,:]*paramsScales, --- > #paramsBarChart.add_trace(go.Scatter(x=paramsNames, y=defaultParamValsOrigRow[0,:]*paramsScales, 1082c1027 < paramsErrorBarsFig.add_trace(go.Scatter(x=paramsAbbrv, y=paramsSolnLin[:,0]*paramsScales, --- > paramsBarChart.add_trace(go.Scatter(x=paramsAbbrv, y=paramsSolnLin[:,0]*paramsScales, 1087c1032 < paramsErrorBarsFig.add_trace(go.Scatter(x=paramsAbbrv, y=paramsSolnNonlin[:,0]*paramsScales, --- > paramsBarChart.add_trace(go.Scatter(x=paramsAbbrv, y=paramsSolnNonlin[:,0]*paramsScales, 1093c1038 < paramsErrorBarsFig.add_trace(go.Scatter(x=paramsAbbrv, y=paramsSolnElastic[:,0]*paramsScales, --- > paramsBarChart.add_trace(go.Scatter(x=paramsAbbrv, y=paramsSolnElastic[:,0]*paramsScales, 1103,1106c1048,1051 < paramsErrorBarsFig.update_yaxes(title="User-scaled parameter value") < paramsErrorBarsFig.update_xaxes(title="Parameter Name") < paramsErrorBarsFig.update_layout(hovermode="x") < paramsErrorBarsFig.update_layout( width=1000, height=500 ) --- > paramsBarChart.update_yaxes(title="User-scaled parameter value") > paramsBarChart.update_xaxes(title="Parameter Name") > paramsBarChart.update_layout(hovermode="x") > paramsBarChart.update_layout( width=1000, height=500 ) 1139,1174c1084,1152 < # Create figure that shows the sensitivity matrix and bias column, both color coded. < sensMatrixBiasFig = createMatrixPlusColFig( matrix = normlzdLinplusSensMatrixPoly, < matIndexLabel = metricsNames, < matColLabel = paramsNames, < colVector = -np.around(defaultBiasesCol/np.abs(normMetricValsCol), decimals=2), < colVectIndexLabel = metricsNames, < colVectColLabel = ['-Normalized Biases'], < plotTitle='Color-coded normalized sensitivity matrix', < reversedYAxis = 'reversed' ) < < < # Create figure that plots color-coded parameter correlation matrix plus parameter-bias correlation column. < XT_dot_X_Linplus = normlzdLinplusSensMatrixPoly.T @ normlzdLinplusSensMatrixPoly < (XT_dot_X_Linplus_corr, stdMatrixInv ) = covMatrix2corrMatrix( XT_dot_X_Linplus, returnStd=True ) < normlzdStdDefaultBiasesCol = stdMatrixInv @ normlzdLinplusSensMatrixPoly.T @ normlzdDefaultBiasesCol < paramsCorrArrayBiasFig = createMatrixPlusColFig( matrix = XT_dot_X_Linplus_corr, < matIndexLabel = paramsNames, < matColLabel = paramsNames, < colVector = -np.around(normlzdStdDefaultBiasesCol, decimals=2), < colVectIndexLabel = paramsNames, < colVectColLabel = ['-Normalized, Standardized Biases'], < plotTitle='XTdotX and bias column, converted to correlation', < reversedYAxis = 'reversed' ) < < # Create figure that plots color-coded projection matrix plus bias column. < XT_dot_X_Linplus_inv = np.linalg.inv( XT_dot_X_Linplus ) < projMatrix = normlzdLinplusSensMatrixPoly @ XT_dot_X_Linplus_inv @ normlzdLinplusSensMatrixPoly.T < print("projMatrix rows=", np.linalg.norm( projMatrix, axis=1)) < projectionMatrixFig = createMatrixPlusColFig( matrix = projMatrix, < matIndexLabel = metricsNames, < matColLabel = metricsNames, < colVector = -np.around(normlzdDefaultBiasesCol, decimals=2), < colVectIndexLabel = metricsNames, < colVectColLabel = ['-Normalized Biases'], < plotTitle='Projection matrix', < reversedYAxis = 'reversed' ) --- > roundedNormlzdSensMatrix = np.around( normlzdLinplusSensMatrixPoly, decimals=2) > df_sensmat = pd.DataFrame(roundedNormlzdSensMatrix, > index=metricsNames, > columns=paramsNames) > matMaxAbs = np.max(np.abs(roundedNormlzdSensMatrix)) > normlzdSensMatrixFig = ff.create_annotated_heatmap( > z=df_sensmat.to_numpy(), > x=df_sensmat.columns.tolist(), > y=df_sensmat.index.tolist(), > #coloraxis="coloraxis", > #colorscale=px.colors.diverging.balance, > colorscale='balance', > zmin=-matMaxAbs, zmax=matMaxAbs, > showscale=False, ygap=1, xgap=1 > ) > normlzdSensMatrixFig.update_xaxes(side="bottom") > normlzdSensMatrixFig.update_layout( > title_text='Color-coded normalized sensitivity matrix', > title_x=0.5, > #width=800, > #height=1400, > xaxis_showgrid=False, > yaxis_showgrid=False, > xaxis_zeroline=False, > yaxis_zeroline=False, > #yaxis_autorange='reversed', > template='plotly_white' > ) > df_biasArray = pd.DataFrame( -np.around(defaultBiasesCol/np.abs(normMetricValsCol), decimals=2), > index=metricsNames, > columns= ['-Normalized Biases']) > normlzdBiasArrayFig = ff.create_annotated_heatmap( > z=df_biasArray.to_numpy(), > x=df_biasArray.columns.tolist(), > y=df_biasArray.index.tolist(), > #colorscale=normlzdSensMatrixFig.colorscale, > #colorscale=px.colors.diverging.balance, > #coloraxis="coloraxis", > colorscale='balance', > zmin=-matMaxAbs, zmax=matMaxAbs, > showscale=True, ygap=1, xgap=1 > ) > normlzdBiasArrayFig.update_layout( > title_text='', > title_x=0.5, > #width=10, > #height=1400, > xaxis_showgrid=False, > yaxis_showgrid=False, > xaxis_zeroline=False, > yaxis_zeroline=False, > #yaxis_autorange='reversed', > #template='plotly_white' > ) > sensMatrixBiasFig = make_subplots( > rows=1, cols=2, > column_widths=[0.9, 0.1], > horizontal_spacing=0.2, > ) > sensMatrixBiasFig.add_trace(normlzdSensMatrixFig.data[0], row=1, col=1) > #sensMatrixBiasFig.add_trace(normlzdBiasArrayFig.data[0], 1, 1) > sensMatrixBiasFig.add_trace(normlzdBiasArrayFig.data[0], row=1, col=2) > sensMatrixBiasFig.update_layout( > title_text='Color-coded normalized sensitivity matrix', > height=700, > template='plotly_white') > sensMatrixBiasFig.update_layout(coloraxis=dict(colorscale='RdBu',cmin=-1,cmax=1), showlegend=False) > > #pdb.set_trace() 1175a1154,1200 > cosAnglesMatrix = calcMatrixAngles( normlzdLinplusSensMatrixPoly ) > roundedCosAnglesMatrix = np.around(cosAnglesMatrix, decimals=2) > df = pd.DataFrame(roundedCosAnglesMatrix, > index=metricsNames, > columns=metricsNames) > upTriMask = np.logical_not( np.tril(np.ones_like(roundedCosAnglesMatrix, dtype=bool)) ) > df_mask = df.mask(upTriMask) > #maskedRoundedCosAnglesMatrix = np.ma.masked_array(roundedCosAnglesMatrix, mask=upTriMask) > #maskedRoundedCosAnglesMatrix.filled(np.nan) > #print("maskedAngles =", maskedRoundedCosAnglesMatrix) > #print("cosAnglesMatrix =", cosAnglesMatrix) > #print("upTriMask =", upTriMask) > metricsCorrArrayFig = ff.create_annotated_heatmap( > z=df_mask.to_numpy(), > x=df_mask.columns.tolist(), > y=df_mask.columns.tolist(), > colorscale=px.colors.diverging.balance, > showscale=True, ygap=1, xgap=1 > ) > #metricsCorrArrayFig = go.Figure(data=go.Heatmap( > # z=roundedCosAnglesMatrix, > ## labels=dict(x="Metrics", y="Metrics")x=['SWCF_GLB', 'SWCF_DYCOMS', 'SWCF_HAWAII', 'SWCF_VOCAL', 'SWCF_VOCAL_near', 'SWCF_LBA', 'SWCF_WP', 'SWCF_EP', 'SWCF_NP', 'SWCF_SP', 'SWCF_CAF', 'SWCF_Nambian', 'SWCF_Nambian_near', 'LWCF_GLB', 'PRECT_GLB']) > ## labels=dict(x="hullo") > # x=metricsNames.tolist(), > # y=metricsNames.tolist() ) > ## text_auto=True ) > # ) > # metricsCorrArrayFig = px.imshow( > # img=roundedCosAnglesMatrix, > # x=metricsNames.tolist(), > # y=metricsNames.tolist(), > # color_continuous_scale=px.colors.diverging.balance > # ) > # metricsCorrArrayFig.update_traces(text=roundedCosAnglesMatrix) > metricsCorrArrayFig.update_xaxes(side="bottom") > metricsCorrArrayFig.update_layout( > title_text='cos(angle) among metrics (i.e., rows of sens matrix)', > title_x=0.5, > width=800, > height=700, > xaxis_showgrid=False, > yaxis_showgrid=False, > xaxis_zeroline=False, > yaxis_zeroline=False, > yaxis_autorange='reversed', > template='plotly_white' > ) 1179d1203 < #normlzdSensMatrixConcatBiases = np.hstack((normlzdWeightedLinplusSensMatrixPoly, -1*normlzdWeightedDefaultBiasesCol)) 1210c1234 < invrsCosFactorPlusMatrix = np.power( np.maximum( np.finfo(float).eps, 2. * ( 1. + cosAnglesMatrix ) ), -0.5 ) --- > invrsCosFactorPlusMatrix = np.power( 2. * ( 1. + cosAnglesMatrix ) , -0.5 ) 1253d1276 < 1259c1282 < dcc.Graph( id='paramsErrorBarsFig', figure=paramsErrorBarsFig ), --- > dcc.Graph( id='paramsBarChart', figure=paramsBarChart ), 1261d1283 < dcc.Graph( id='metricsBarChart', figure=metricsBarChart ), 1263,1265c1285 < # dcc.Graph( id='paramsTotContrbBarFig', figure=paramsTotContrbBarFig ), < dcc.Graph( id='sensMatrixBarFig', figure=sensMatrixBarFig ), < #dcc.Graph( id='linplusSensMatrixBarFig', figure=linplusSensMatrixBarFig ), --- > dcc.Graph( id='paramsTotContrbBarFig', figure=paramsTotContrbBarFig ), 1271d1290 < dcc.Graph( id='paramsCorrArrayBiasFig', figure=paramsCorrArrayBiasFig ), 1274d1292 < dcc.Graph( id='projectionMatrixFig', figure=projectionMatrixFig ), 1301,1314d1318 < def covMatrix2corrMatrix( covMatrix, returnStd=False ): < < # https://gist.github.com/wiso/ce2a9919ded228838703c1c7c7dad13b < < import numpy as np < < stdVector = np.sqrt( np.diag( covMatrix ) ) < stdMatrixInv = np.diag( 1.0 / stdVector ) < corrMatrix = stdMatrixInv @ covMatrix @ stdMatrixInv < if returnStd: < return ( corrMatrix, stdMatrixInv ) < else: < return corrMatrix < 1370,1371d1373 < #pdb.set_trace() < 1375c1377 < dnormlzdParamsSolnNonlin = minimize(objFnc,x0=np.zeros_like(np.transpose(defaultParamValsOrigRow[0])), \ --- > dnormlzdParamsSolnNonlin = minimize(objFnc,x0=np.zeros_like(np.transpose(defaultParamValsOrigRow)), \ 1419c1421 < dnormlzdParamsSolnLin = minimize(objFnc,x0=np.zeros_like(np.transpose(defaultParamValsOrigRow[0])), \ --- > dnormlzdParamsSolnLin = minimize(objFnc,x0=np.zeros_like(np.transpose(defaultParamValsOrigRow)), \ 1606,1739d1607 < < < < def createMatrixPlusColFig( matrix, matIndexLabel, matColLabel, < colVector, colVectIndexLabel, colVectColLabel, < plotTitle, reversedYAxis=None ): < '''Creates a figure that displays a color-coded matrix and an accompanying column vector.''' < < import numpy as np < import pandas as pd < import plotly.figure_factory as ff < import plotly.express as px < from plotly.subplots import make_subplots < import pdb < < # First create a sub-figure that displays color-coded matrix < roundedNormlzdSensMatrix = np.around( matrix, decimals=2) < df_sensmat = pd.DataFrame(roundedNormlzdSensMatrix, < index=matIndexLabel, < columns=matColLabel) < matMaxAbs = np.max(np.abs(roundedNormlzdSensMatrix)) < matSubfig = px.imshow( < df_sensmat.to_numpy(), < x=df_sensmat.columns.tolist(), < y=df_sensmat.index.tolist(), < text_auto=True < ) < matSubfig.update_xaxes(side="bottom") < matSubfig.update_layout( < title_text=plotTitle, < title_x=0.5, < #width=800, < #height=1400, < xaxis_showgrid=False, < yaxis_showgrid=False, < xaxis_zeroline=False, < yaxis_zeroline=False, < ) < < # Now create a sub-figure showing a color-coded column matrix < df_biasArray = pd.DataFrame( colVector, < index=colVectIndexLabel, < columns= colVectColLabel) < colVectSubfig = px.imshow( < df_biasArray.to_numpy(), < x=df_biasArray.columns.tolist(), < y=df_biasArray.index.tolist(), < text_auto=True < ) < colVectSubfig.update_layout( < title_text='', < title_x=0.5, < #width=10, < #height=1400, < xaxis_showgrid=False, < yaxis_showgrid=False, < xaxis_zeroline=False, < yaxis_zeroline=False, < ) < < # Now combine the matrix and column sub-figures into one figure < matrixPlusColFig = make_subplots( < rows=1, cols=2, < column_widths=[0.9, 0.1], < horizontal_spacing=0.3, < ) < matrixPlusColFig.add_trace(matSubfig.data[0], row=1, col=1) < matrixPlusColFig.add_trace(colVectSubfig.data[0], row=1, col=2) < matrixPlusColFig.update_layout( < title_text=plotTitle, < height=700, < width= 1000, < template='plotly_white') < matrixPlusColFig.update_layout(coloraxis=dict(colorscale='RdBu_r',cmin=-matMaxAbs,cmax=matMaxAbs), showlegend=False) < matrixPlusColFig.update_yaxes( autorange=reversedYAxis, row=1, col=2 ) < matrixPlusColFig.update_yaxes( autorange=reversedYAxis, row=1, col=1 ) < < #pdb.set_trace() < < return ( matrixPlusColFig ) < < < def createMetricsBarChart( metricsNames, paramsNames, biases, sensMatrix ): < < < import plotly.graph_objects as go < import numpy as np < import pdb < < biases = np.reshape(biases, (-1,1)) < barBase = np.copy(biases) # np.copy prevents biases variable from changing < rightEnd = np.copy(biases) < leftEnd = np.copy(biases) < barsData = [] < for col in range(len(paramsNames)): < #print("paramsNames[col]=", paramsNames[col]) < sensCol = sensMatrix[:,[col]] < #print("sensCol=", sensCol ) < #print("rightEnd=", rightEnd ) < for row in range( len(sensCol) ): < if ( np.sign(sensCol[row]) > 0 ): < barBase[row] = rightEnd[row] < else: < barBase[row] = leftEnd[row] < < #print("barBase=", barBase) < #print("biases during=", biases) < barsData.append( go.Bar(name=paramsNames[col], y=metricsNames, x=sensCol[:,0], < base=barBase[:,0], orientation="h" ) ) < rightEnd = rightEnd + np.maximum( np.zeros_like(sensCol), sensCol ) < leftEnd = leftEnd + np.minimum( np.zeros_like(sensCol), sensCol ) < < #print("biases after=", biases) < # Insert a black line in each bar to denote default biases that we want to remove < barsData.append( go.Bar(name='default bias', < y=metricsNames, x=np.zeros(len(metricsNames)), base=biases[:,0], < orientation="h", < marker_line_color = 'black', marker_color='black', marker_line_width = 4 < ) < ) < < metricsBarChart = go.Figure(data=barsData) < < # Change the bar mode < metricsBarChart.update_layout(barmode='stack') < metricsBarChart.update_xaxes(visible=True,zeroline=True,zerolinewidth=4,zerolinecolor='gray') # Plot y axis < metricsBarChart.update_layout( width=800, height=50*len(metricsNames) ) < metricsBarChart.update_xaxes(title="-Normalized biases") < < #pdb.set_trace() < < return metricsBarChart < < 1741d1608 < # barBase, 1748d1614 < import plotly.graph_objects as go 1754,1757c1620 < barChart = px.bar(df, x=df.index, y=df.columns, < #base=barBase, < #offset=1, < orientation=orientation, --- > barChart = px.bar(df, x=df.index, y=df.columns, orientation=orientation, 1765d1627 < #barChart.update_layout(barmode='relative') 1915,1976d1776 < def createCorrArrayFig( matrix, indexLabels, title ): < < import numpy as np < import pandas as pd < import plotly.figure_factory as ff < import plotly.express as px < import pdb < < cosAnglesMatrix = calcMatrixAngles( matrix ) < #cosAnglesMatrix = np.copy( matrix ) < roundedCosAnglesMatrix = np.around(cosAnglesMatrix, decimals=2) < < df = pd.DataFrame(roundedCosAnglesMatrix, < index=indexLabels, < columns=indexLabels) < # Display only the lower-triangular elements of the matrix < upTriMask = np.logical_not( np.tril(np.ones_like(roundedCosAnglesMatrix, dtype=bool)) ) < df_mask = df.mask(upTriMask) < #maskedRoundedCosAnglesMatrix = np.ma.masked_array(roundedCosAnglesMatrix, mask=upTriMask) < #maskedRoundedCosAnglesMatrix.filled(np.nan) < #print("maskedAngles =", maskedRoundedCosAnglesMatrix) < #print("cosAnglesMatrix =", cosAnglesMatrix) < #print("upTriMask =", upTriMask) < corrArrayFig = ff.create_annotated_heatmap( < z=df_mask.to_numpy(), < x=df_mask.columns.tolist(), < y=df_mask.columns.tolist(), < colorscale=px.colors.diverging.balance, < showscale=True, ygap=1, xgap=1 < ) < #metricsCorrArrayFig = go.Figure(data=go.Heatmap( < # z=roundedCosAnglesMatrix, < ## labels=dict(x="Metrics", y="Metrics")x=['SWCF_GLB', 'SWCF_DYCOMS', 'SWCF_HAWAII', 'SWCF_VOCAL', 'SWCF_VOCAL_near', 'SWCF_LBA', 'SWCF_WP', 'SWCF_EP', 'SWCF_NP', 'SWCF_SP', 'SWCF_CAF', 'SWCF_Nambian', 'SWCF_Nambian_near', 'LWCF_GLB', 'PRECT_GLB']) < ## labels=dict(x="hullo") < # x=metricsNames.tolist(), < # y=metricsNames.tolist() ) < ## text_auto=True ) < # ) < # metricsCorrArrayFig = px.imshow( < # img=roundedCosAnglesMatrix, < # x=metricsNames.tolist(), < # y=metricsNames.tolist(), < # color_continuous_scale=px.colors.diverging.balance < # ) < # metricsCorrArrayFig.update_traces(text=roundedCosAnglesMatrix) < corrArrayFig.update_xaxes(side="bottom") < corrArrayFig.update_layout( < title_text=title, < title_x=0.5, < width=800, < height=700, < xaxis_showgrid=False, < yaxis_showgrid=False, < xaxis_zeroline=False, < yaxis_zeroline=False, < yaxis_autorange='reversed', < template='plotly_white' < ) < < #pdb.set_trace() < < return ( corrArrayFig ) diff '--exclude=.git' '--exclude=version_clubb_core.txt' '--exclude=version_silhs.txt' -r clubb/utilities/sens_matrix/set_up_dashboard_inputs.py clubb_release/utilities/sens_matrix/set_up_dashboard_inputs.py 56,58c56 < # ['SWCF_RACC', 0.01, 0.2], \ < ['SWCF_RMSEP', 4.01, 15.], \ < # ['SWCF_RMSE', 0.01, 15.], \ --- > ['SWCF_RMSE', 4.00, 15.], \ 61,62c59,60 < ['SWCF_HAWAII', 1.00, -999], \ < ['SWCF_VOCAL', 1.00, -999], \ --- > ['SWCF_HAWAII', 2.00, -999], \ > ['SWCF_VOCAL', 2.00, -999], \ 69c67 < ## ['SWCF_PA', 1.01, -999], \ --- > # ['SWCF_PA', 1.01, -999], \ 71c69 < ['SWCF_Namibia', 4.00, -999], \ --- > ['SWCF_Namibia', 1.00, -999], \ 78c76 < ## ['LWCF_WP', 1.00, -999], \ --- > # ['LWCF_WP', 1.00, -999], \ 80,81c78,79 < ## ['LWCF_NP', 1.01, -999], \ < ## ['LWCF_SP', 1.01, -999], \ --- > # ['LWCF_NP', 1.01, -999], \ > # ['LWCF_SP', 1.01, -999], \ 85,87d82 < # ['PRECT_RACC', 0.01, 1.0], \ < # ['PRECT_RMSEP', 0.01, 1.0], \ < # ['PRECT_RMSE', 0.01, 1.0], \ 129c124 < folder_name = 'Regional_files/20230910/' # folder where regional netcdf files are stored. --- > folder_name = 'Regional_files/20230804/' # folder where regional netcdf files are stored. 153,155c148,150 < # ['clubb_c_invrs_tau_sfc', 1.0, \ < # folder_name + 'sens0707_6_Regional.nc', < # folder_name + 'sens0707_7_Regional.nc'], \ --- > ['clubb_c_invrs_tau_sfc', 1.0, \ > folder_name + 'sens0707_6_Regional.nc', > folder_name + 'sens0707_7_Regional.nc'], \ 185,187d179 < # ['cldfrc_dp2', 1.0, \ < # folder_name + 'sens0707_38_Regional.nc', < # folder_name + 'sens0707_39_Regional.nc'], \ 211,231d202 < # ['clubb_c8', 1.0, 0.7, \ < # folder_name + 'sens0707_14_Regional.nc', \ < # folder_name + 'sens0707_15_Regional.nc'], \ < # ['clubb_c_k10', 1.0, 0.3, \ < # folder_name + 'sens0707_12_Regional.nc', \ < # folder_name + 'sens0707_13_Regional.nc'], \ < # ['clubb_c_invrs_tau_n2', 1.0, 0.4, \ < # folder_name + 'sens0707_10_Regional.nc', < # folder_name + 'sens0707_11_Regional.nc'], \ < ['clubb_c_invrs_tau_sfc', 1.0, 0.05, \ < folder_name + 'sens0707_6_Regional.nc', < folder_name + 'sens0707_7_Regional.nc'], \ < # ['clubb_c_invrs_tau_wpxp_n2_thresh', 1.e3, 0.00045, \ < # folder_name + 'sens0707_8_Regional.nc', \ < # folder_name + 'sens0707_9_Regional.nc'], \ < # ['clubb_c_invrs_tau_shear', 1.0, 0.22, \ < # folder_name + 'sens0707_2_Regional.nc', \ < # folder_name + 'sens0707_3_Regional.nc'], \ < # ['clubb_c_invrs_tau_bkgnd', 1.0, 1.1, \ < # folder_name + 'sens0707_16_Regional.nc', < # folder_name + 'sens0707_17_Regional.nc'], \ 234a206,208 > # ['clubb_c_invrs_tau_bkgnd', 1.0, 1.50686559, \ > # folder_name + 'sens0707_16_Regional.nc', \ > # folder_name + 'sens0707_17_Regional.nc'], \ 264c238 < folder_name + 'sens0707_25_Regional.nc' --- > folder_name + 'sens0707_1_Regional.nc' 273,274d246 < 'SWCF_RACC': 0, \ < 'SWCF_RMSEP': 0, \ 290,292d261 < 'PRECT_RACC': 0, \ < 'PRECT_RMSEP': 0, \ < 'PRECT_RMSE': 0, \ 394,395d362 < < #print("defaultMetricValsCol=", defaultMetricValsCol) [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Declarative: Post Actions) [Pipeline] script [Pipeline] { [Pipeline] cleanWs [WS-CLEANUP] Deleting project workspace... [WS-CLEANUP] Deferred wipeout is used... [WS-CLEANUP] done [Pipeline] } [Pipeline] // script [Pipeline] script [Pipeline] { [Pipeline] emailext Sending email to: messnermet@uwm.edu [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] End of Pipeline ERROR: script returned exit code 1 Finished: FAILURE