Console Output

+ source /etc/profile.d/larson-group.sh
++ export GIT_EDITOR=vi
++ GIT_EDITOR=vi
++ export SVN_EDITOR=vi
++ SVN_EDITOR=vi
++ export OMP_STACKSIZE=1048579
++ OMP_STACKSIZE=1048579
++ export LMOD_ROOT=/opt/lmod/
++ LMOD_ROOT=/opt/lmod/
++ source /opt/lmod//lmod/lmod/init/bash
+++ '[' -z '' ']'
+++ case "$-" in
+++ __lmod_vx=x
+++ '[' -n x ']'
+++ set +x
Shell debugging temporarily silenced: export LMOD_SH_DBG_ON=1 for this output (/usr/local/spack/opt/spack/linux-pop22-cascadelake/gcc-12.2.0/lmod-8.7.37-bi3kyxcdrfgw3y7vv2k7c5rjxg75qzju/lmod/lmod/init/bash)
Shell debugging restarted
+++ unset __lmod_vx
+++ find /usr/local/spack/share/spack/lmod/linux-pop22-x86_64/Core -print -quit
++ export MODULEPATH=/usr/local/spack/share/spack/lmod/linux-pop22-x86_64/Core
++ MODULEPATH=/usr/local/spack/share/spack/lmod/linux-pop22-x86_64/Core
+ ulimit -s 8388608
+ python3 /home/pub/jenkins_BFB_test_configs/sys_admin/python_nightly_test_suite/run_test.py -c /home/pub/jenkins_BFB_test_configs/cam_BFB_coarse_res_gfortran_test_config
Build script /home/jenkins/workspace/cam_BFB_coarse_res_gfortran_test/libraries/FTorch/buildlib for component FTorch not found.
submit_jobs case.run
Submit job case.run
check for resubmit
dout_s False 
mach larson-group 
resubmit_num 0
Build script /home/jenkins/workspace/cam_BFB_coarse_res_gfortran_test/libraries/FTorch/buildlib for component FTorch not found.
submit_jobs case.run
Submit job case.run
check for resubmit
dout_s False 
mach larson-group 
resubmit_num 0
-------- Variable Diff Summary -------- 
File 1: 07b885f24884d7b8f04fa69ce3e2d14da0da8c35
File 2: 7add1e3967b3f136dc120143d43e5e84d701092d

Variables discovered (union): 1720
Variables present in both files: 1720
Variables only in file 1: 0
Variables only in file 2: 0
Variables with differences (all dimensions): 2
Variables identical (all dimensions): 1718

Variables all-zero in file 1 (actual data): 138
Variables all-zero in file 2 (actual data): 138
Variables all-zero in both files (actual data): 138

Variables with >=2 dimensions tested (PASS/FAIL basis): 627
Variables with >=2 dimensions with differences: 2
Variables with >=2 dimensions identical: 625
Variables with >=2 dimensions all-zero in both files (actual data): 105

Variables skipped for PASS/FAIL (fewer than 2 dimensions in both files): 1093
Variables with differences:
  - FDL | reason=data mismatch | file1: ndim=2, dims=('ilev', 'ncol') | file2: ndim=2, dims=('ilev', 'ncol')
  - time_written | reason=data mismatch | file1: ndim=2, dims=('time', 'chars') | file2: ndim=2, dims=('time', 'chars')

FAIL: Found differences in one or more variables with >=2 dimensions.

=============================== Configuring Test ===============================


Working directory set to: /home/jenkins/workspace/cam_BFB_coarse_res_gfortran_test
Checking specified repo: /home/jenkins/workspace/cam_BFB_coarse_res_gfortran_test

Using /home/jenkins/workspace/cam_BFB_coarse_res_gfortran_test with origin https://github.com/larson-group/cam.git

Output Save Mode: none

Using branch: clubb_silhs_devel

Making list from 'UnresolvedCommits':

CheckRange = False: only comparing baseline to final commit.

Run Output Path: /home/jenkins/cam_output/scratch/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu/run/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu.cam.rh0.1979-01-02-00000.nc

Run commands: ['rm -f /home/jenkins/cam_output/scratch/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu/run/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu.cam.rh0.1979-01-02-00000.nc &&\nrun_scripts/set_up_repo.py --caseroot /home/$USER/cam_output/caseroot/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu --reuse_build . &&\nsed -i -e \'s/^CASE=.*/CASE="UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37"/\' -e \'s/^COMPSET=.*/COMPSET="FHISTC_MTso"/\' -e \'s/ -silhs / /g\' -e \'s/ -psubcols \\$NUMSC//g\' -e \'s/ -DSILHS//g\' -e "s/^subcol_scheme = .*/subcol_scheme = \'off\'/" -e \'s/^use_subcol_microp = .*/use_subcol_microp = .false./\' -e \'s/^microp_uniform = .*/microp_uniform = .false./\' -e "s/, \'SILHS_CLUBB_PRECIP_FRAC\',//" -e "/^\'SILHS_CLUBB_ICE_SS_FRAC\'/d" run_scripts/run_cesm_uwm_coarse_res.sh &&\nrun_scripts/run_cesm_uwm_coarse_res.sh']

Diffing command: python3 run_scripts/netcdf_3D_vars_diff.py

=============================== Running Test ===============================

Parsing message from commit: 07b885f24884d7b8f04fa69ce3e2d14da0da8c35

Checking Commit: 07b885f24884d7b8f04fa69ce3e2d14da0da8c35
	Running: final commit to check, not labelled BIT_CHANGING
	Checking out 07b885f24884d7b8f04fa69ce3e2d14da0da8c35
	Running: 'rm -f /home/jenkins/cam_output/scratch/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu/run/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu.cam.rh0.1979-01-02-00000.nc &&
run_scripts/set_up_repo.py --caseroot /home/$USER/cam_output/caseroot/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu --reuse_build . &&
sed -i -e 's/^CASE=.*/CASE="UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37"/' -e 's/^COMPSET=.*/COMPSET="FHISTC_MTso"/' -e 's/ -silhs / /g' -e 's/ -psubcols \$NUMSC//g' -e 's/ -DSILHS//g' -e "s/^subcol_scheme = .*/subcol_scheme = 'off'/" -e 's/^use_subcol_microp = .*/use_subcol_microp = .false./' -e 's/^microp_uniform = .*/microp_uniform = .false./' -e "s/, 'SILHS_CLUBB_PRECIP_FRAC',//" -e "/^'SILHS_CLUBB_ICE_SS_FRAC'/d" run_scripts/run_cesm_uwm_coarse_res.sh &&
run_scripts/run_cesm_uwm_coarse_res.sh'
	Checking out 7add1e3967b3f136dc120143d43e5e84d701092d
	Running: 'rm -f /home/jenkins/cam_output/scratch/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu/run/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu.cam.rh0.1979-01-02-00000.nc &&
run_scripts/set_up_repo.py --caseroot /home/$USER/cam_output/caseroot/UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37_larson-group_gnu --reuse_build . &&
sed -i -e 's/^CASE=.*/CASE="UWM_debug_FHISTC_MTso_cam7_ne3pg3_ne3pg3_mg37"/' -e 's/^COMPSET=.*/COMPSET="FHISTC_MTso"/' -e 's/ -silhs / /g' -e 's/ -psubcols \$NUMSC//g' -e 's/ -DSILHS//g' -e "s/^subcol_scheme = .*/subcol_scheme = 'off'/" -e 's/^use_subcol_microp = .*/use_subcol_microp = .false./' -e 's/^microp_uniform = .*/microp_uniform = .false./' -e "s/, 'SILHS_CLUBB_PRECIP_FRAC',//" -e "/^'SILHS_CLUBB_ICE_SS_FRAC'/d" run_scripts/run_cesm_uwm_coarse_res.sh &&
run_scripts/run_cesm_uwm_coarse_res.sh'
	Diffing: 'python3 run_scripts/netcdf_3D_vars_diff.py 07b885f24884d7b8f04fa69ce3e2d14da0da8c35 7add1e3967b3f136dc120143d43e5e84d701092d'
	ISSUE WITH COMMIT DETECTED: 07b885f24884d7b8f04fa69ce3e2d14da0da8c35 differs from 7add1e3967b3f136dc120143d43e5e84d701092d

=============================== Test Results ===============================

Commit Range Checked = 7add1e3967b3f136dc120143d43e5e84d701092d...07b885f24884d7b8f04fa69ce3e2d14da0da8c35
-- CheckRange was set to False, no intermediate commits checked

New Unresolved Commits:
	07b885f24884d7b8f04fa69ce3e2d14da0da8c35

No fake BIT_CHANGING commits found.

Remaining Unresolved Commits (oldest commit is listed first):
	07b885f24884d7b8f04fa69ce3e2d14da0da8c35

=============================== Updating Config File ===============================


Saving into UnresolvedCommits: 07b885f24884d7b8f04fa69ce3e2d14da0da8c35


Saving into BaselineCommit: 07b885f24884d7b8f04fa69ce3e2d14da0da8c35

=============================== Cleanup ===============================

Removing all generated output
	REMOVING: 7add1e3967b3f136dc120143d43e5e84d701092d
	REMOVING: 07b885f24884d7b8f04fa69ce3e2d14da0da8c35

================== Unresolved Commit Status ==================

Remaining Unresolved Commits (oldest commit is listed first):
	07b885f24884d7b8f04fa69ce3e2d14da0da8c35

Exiting with code: 1