+ source /etc/profile.d/larson-group.sh
++ export PATH MANPATH LD_LIBRARY_PATH
++ export SVNROOT=http://carson.math.uwm.edu/repos
++ SVNROOT=http://carson.math.uwm.edu/repos
++ export GIT_EDITOR=vi
++ GIT_EDITOR=vi
++ export SVN_EDITOR=vi
++ SVN_EDITOR=vi
++ export OMP_STACKSIZE=1048579
++ OMP_STACKSIZE=1048579
++ export LMOD_ROOT=/usr/local/spack/opt/spack/linux-pop22-cascadelake/gcc-12.2.0/lmod-8.7.37-bi3kyxcdrfgw3y7vv2k7c5rjxg75qzju
++ LMOD_ROOT=/usr/local/spack/opt/spack/linux-pop22-cascadelake/gcc-12.2.0/lmod-8.7.37-bi3kyxcdrfgw3y7vv2k7c5rjxg75qzju
++ source /usr/local/spack/opt/spack/linux-pop22-cascadelake/gcc-12.2.0/lmod-8.7.37-bi3kyxcdrfgw3y7vv2k7c5rjxg75qzju/lmod/lmod/init/bash
+++ '[' -z '' ']'
+++ case "$-" in
+++ __lmod_vx=x
+++ '[' -n x ']'
+++ set +x
Shell debugging temporarily silenced: export LMOD_SH_DBG_ON=1 for this output (/usr/local/spack/opt/spack/linux-pop22-cascadelake/gcc-12.2.0/lmod-8.7.37-bi3kyxcdrfgw3y7vv2k7c5rjxg75qzju/lmod/lmod/init/bash)
Shell debugging restarted
+++ unset __lmod_vx
++ export MODULEPATH=/usr/local/spack/share/spack/lmod/linux-pop22-x86_64/Core
++ MODULEPATH=/usr/local/spack/share/spack/lmod/linux-pop22-x86_64/Core
+ module load gcc esmf
+ '[' -z '' ']'
+ case "$-" in
+ __lmod_sh_dbg=x
+ '[' -n x ']'
+ set +x
Shell debugging temporarily silenced: export LMOD_SH_DBG_ON=1 for Lmod's output
Lmod Warning:
-------------------------------------------------------------------------------
The following dependent module(s) are not currently loaded:
gcc-runtime/12.2.0-v5od57d (required by: gmp/6.3.0-wmkgoai, mpfr/4.2.1-z4h5vuo,
mpc/1.3.1-7yebwoe, zlib-ng/2.1.6-sgcwuo6, zstd/1.5.6-4yyzf3h, gcc/13.1.0,
libpciaccess/0.17-m6rj66n, xz/5.4.6-dc2pskh, libxml2/2.10.3-gipjgfv,
ncurses/6.5-tl7ktgt, hwloc/2.9.3-btuxxs7, libfabric/1.21.0-n2upgxo,
yaksa/0.3-snaqoka, bzip2/1.0.8-45glntl, lz4/1.9.4-p5fhvz2,
snappy/1.1.10-557fi2m, c-blosc/1.21.5-ifrgevx, pkgconf/2.2.0-5yfc5u2,
libaec/1.0.6-k3ttt5u)
-------------------------------------------------------------------------------
Shell debugging restarted
+ unset __lmod_sh_dbg
+ return 0
+ rm -rf /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37
+ python3 /home/jenkins/cam_bit_diff_test_workspace/sys_admin/python_nightly_test_suite/run_test.py -c /home/jenkins/cam_bit_diff_test_workspace/bit_diff_cam_config
ERROR: No machine nelson found
Error creating new case
specified use --force to set
ERROR: Undefined env var 'GFORTRAN_NETCDF_FORTRAN'
Traceback (most recent call last):
File "/home/jenkins/cam_bit_diff_test_workspace/repo/run_scripts/set_up_cam_coarse_res.py", line 78, in <module>
os.mkdir("ccs_config/machines/larson-group")
FileExistsError: [Errno 17] File exists: 'ccs_config/machines/larson-group'
ERROR: Unknown Job Queue specified use --force to set
submit_jobs case.run
Submit job case.run
No timing file found in run directory
check for resubmit
dout_s False
mach larson-group
resubmit_num 0
Traceback (most recent call last):
File "/home/jenkins/cam_bit_diff_test_workspace/repo/run_scripts/set_up_cam_coarse_res.py", line 78, in <module>
os.mkdir("ccs_config/machines/larson-group")
FileExistsError: [Errno 17] File exists: 'ccs_config/machines/larson-group'
ERROR: Unknown Job Queue specified use --force to set
submit_jobs case.run
Submit job case.run
No timing file found in run directory
check for resubmit
dout_s False
mach larson-group
resubmit_num 0
=============================== Configuring Test ===============================
Working directory set to: /home/jenkins/cam_bit_diff_test_workspace
Trying to clone repo...
Cloned repo location: /home/jenkins/cam_bit_diff_test_workspace/repo
Output Save Mode: none
Using branch: clubb_silhs_devel
Making list from 'UnresolvedCommits':
Run Output Path: /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37/run/subcol_SILHS_UWM_debug_f10_f10_mg37.cam.rh0.0001-01-02-00000.nc
Run commands: ['rm -rf /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37', '\nrepo/run_scripts/set_up_cam_coarse_res.py --caseroot /home/$USER/cam_output/caseroot/$CASE --reuse_build --mach nelson --silhs 4 repo', '\nrepo/run_scripts/run_cesm_uwm_coarse_res.sh']
Diffing command: python3 repo/run_scripts/netcdf_3D_vars_diff.py
=============================== Running Test ===============================
Parsing message from commit: 7bfe29a485e7b753a079ade0b887e212f7f48989
Parsing message from commit: d90b25c0dd5e712488e84d9668f1e2049ecbd562
Parsing message from commit: b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
Checking Commit: 7bfe29a485e7b753a079ade0b887e212f7f48989
Skipping: not final commit and not labelled BIT_CHANGING
Checking Commit: d90b25c0dd5e712488e84d9668f1e2049ecbd562
Skipping: not final commit and not labelled BIT_CHANGING
Checking Commit: b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
Running: final commit to check, not labelled BIT_CHANGING
Checking out b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
Running: 'rm -rf /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37'
Running: '
repo/run_scripts/set_up_cam_coarse_res.py --caseroot /home/$USER/cam_output/caseroot/$CASE --reuse_build --mach nelson --silhs 4 repo'
Running: '
repo/run_scripts/run_cesm_uwm_coarse_res.sh'
================== WARNING ==================
Commit: b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
There was a problem copying the output.
Did the run complete?
Is the output path correct? /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37/run/subcol_SILHS_UWM_debug_f10_f10_mg37.cam.rh0.0001-01-02-00000.nc
The run will continue, and this commit will be
considered unresolved.
===========================================
Checking out 2be2b0d0d25b26a03f8053fa505c8ce962661dad
Running: 'rm -rf /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37'
Running: '
repo/run_scripts/set_up_cam_coarse_res.py --caseroot /home/$USER/cam_output/caseroot/$CASE --reuse_build --mach nelson --silhs 4 repo'
Running: '
repo/run_scripts/run_cesm_uwm_coarse_res.sh'
================== WARNING ==================
Commit: 2be2b0d0d25b26a03f8053fa505c8ce962661dad
There was a problem copying the output.
Did the run complete?
Is the output path correct? /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37/run/subcol_SILHS_UWM_debug_f10_f10_mg37.cam.rh0.0001-01-02-00000.nc
The run will continue, and this commit will be
considered unresolved.
===========================================
ISSUE WITH COMMIT DETECTED: b70e8c0c5c561fba09acca2b9579b2cf58ec1a85 differs from 2be2b0d0d25b26a03f8053fa505c8ce962661dad
Finding all BIT_CHANGING commits in range: 2be2b0d0d25b26a03f8053fa505c8ce962661dad...b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
Checking out 7bfe29a485e7b753a079ade0b887e212f7f48989
Running: 'rm -rf /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37'
Running: '
repo/run_scripts/set_up_cam_coarse_res.py --caseroot /home/$USER/cam_output/caseroot/$CASE --reuse_build --mach nelson --silhs 4 repo'
Running: '
repo/run_scripts/run_cesm_uwm_coarse_res.sh'
================== WARNING ==================
Commit: 7bfe29a485e7b753a079ade0b887e212f7f48989
There was a problem copying the output.
Did the run complete?
Is the output path correct? /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37/run/subcol_SILHS_UWM_debug_f10_f10_mg37.cam.rh0.0001-01-02-00000.nc
The run will continue, and this commit will be
considered unresolved.
===========================================
Commit 2be2b0d0d25b26a03f8053fa505c8ce962661dad already failed. Will not run again
Checking out d90b25c0dd5e712488e84d9668f1e2049ecbd562
Running: 'rm -rf /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37'
Running: '
repo/run_scripts/set_up_cam_coarse_res.py --caseroot /home/$USER/cam_output/caseroot/$CASE --reuse_build --mach nelson --silhs 4 repo'
Running: '
repo/run_scripts/run_cesm_uwm_coarse_res.sh'
================== WARNING ==================
Commit: d90b25c0dd5e712488e84d9668f1e2049ecbd562
There was a problem copying the output.
Did the run complete?
Is the output path correct? /home/jenkins/cam_output/scratch/subcol_SILHS_UWM_debug_f10_f10_mg37/run/subcol_SILHS_UWM_debug_f10_f10_mg37.cam.rh0.0001-01-02-00000.nc
The run will continue, and this commit will be
considered unresolved.
===========================================
Commit 7bfe29a485e7b753a079ade0b887e212f7f48989 already failed. Will not run again
Commit b70e8c0c5c561fba09acca2b9579b2cf58ec1a85 already failed. Will not run again
Commit d90b25c0dd5e712488e84d9668f1e2049ecbd562 already failed. Will not run again
Found:
7bfe29a485e7b753a079ade0b887e212f7f48989
d90b25c0dd5e712488e84d9668f1e2049ecbd562
b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
=============================== Test Results ===============================
Commit Range Checked = 2be2b0d0d25b26a03f8053fa505c8ce962661dad...b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
New Unresolved Commits:
7bfe29a485e7b753a079ade0b887e212f7f48989
d90b25c0dd5e712488e84d9668f1e2049ecbd562
b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
No fake BIT_CHANGING commits found.
Remaining Unresolved Commits (oldest commit is listed first):
7bfe29a485e7b753a079ade0b887e212f7f48989
d90b25c0dd5e712488e84d9668f1e2049ecbd562
b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
=============================== Updating Config File ===============================
Saving into UnresolvedCommits: 7bfe29a485e7b753a079ade0b887e212f7f48989
d90b25c0dd5e712488e84d9668f1e2049ecbd562
b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
Ignoring b70e8c0c5c561fba09acca2b9579b2cf58ec1a85 since it failed to produce output.
Ignoring d90b25c0dd5e712488e84d9668f1e2049ecbd562 since it failed to produce output.
Ignoring 7bfe29a485e7b753a079ade0b887e212f7f48989 since it failed to produce output.
=============================== Cleanup ===============================
Removed repo.
Removing all generated output
================== Unresolved Commit Status ==================
Remaining Unresolved Commits (oldest commit is listed first):
7bfe29a485e7b753a079ade0b887e212f7f48989
d90b25c0dd5e712488e84d9668f1e2049ecbd562
b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
================== FAILED Commits ==================
These are the commits which entirely failed to produce output:
b70e8c0c5c561fba09acca2b9579b2cf58ec1a85
2be2b0d0d25b26a03f8053fa505c8ce962661dad
7bfe29a485e7b753a079ade0b887e212f7f48989
d90b25c0dd5e712488e84d9668f1e2049ecbd562
Exiting with code: 1
![[Jenkins]](/jenkins/static/57c8a776/images/svgs/logo.svg){kind=logo}
