+ source /etc/profile.d/larson-group.sh
++ export GIT_EDITOR=vi
++ GIT_EDITOR=vi
++ export SVN_EDITOR=vi
++ SVN_EDITOR=vi
++ export OMP_STACKSIZE=1048579
++ OMP_STACKSIZE=1048579
++ export LMOD_ROOT=/opt/lmod/
++ LMOD_ROOT=/opt/lmod/
++ source /opt/lmod//lmod/lmod/init/bash
+++ '[' -z '' ']'
+++ case "$-" in
+++ __lmod_vx=x
+++ '[' -n x ']'
+++ set +x
Shell debugging temporarily silenced: export LMOD_SH_DBG_ON=1 for this output (/usr/local/spack/opt/spack/linux-pop22-cascadelake/gcc-12.2.0/lmod-8.7.37-bi3kyxcdrfgw3y7vv2k7c5rjxg75qzju/lmod/lmod/init/bash)
Shell debugging restarted
+++ unset __lmod_vx
+++ find /usr/local/spack/share/spack/lmod/linux-pop22-x86_64/Core -print -quit
++ export MODULEPATH=/usr/local/spack/share/spack/lmod/linux-pop22-x86_64/Core
++ MODULEPATH=/usr/local/spack/share/spack/lmod/linux-pop22-x86_64/Core
+ module load gcc netcdf-fortran
+ '[' -z '' ']'
+ case "$-" in
+ __lmod_sh_dbg=x
+ '[' -n x ']'
+ set +x
Shell debugging temporarily silenced: export LMOD_SH_DBG_ON=1 for Lmod's output
Lmod Warning:
-------------------------------------------------------------------------------
The following dependent module(s) are not currently loaded:
gcc-runtime/13.1.0-gzedt3f (required by: mpich/4.2.1, hdf5/1.14.3-42x3zrv,
netcdf-c/4.9.2, netcdf-fortran/4.5.3)
-------------------------------------------------------------------------------
Shell debugging restarted
+ unset __lmod_sh_dbg
+ return 0
+ python3 /home/pub/jenkins_BFB_test_configs/sys_admin/python_nightly_test_suite/run_test.py -c /home/pub/jenkins_BFB_test_configs/clubb_BFB_e3sm_flags_gfortran_test_config
Directory 1 is d9d887857da333c2c683aad09311ffcdb2d54440
Directory 2 is d20fd1daad5e5d921607794e258bfde151fd3c89
Using reporting threshold: 0.0
The following cases will be compared: ['arm', 'arm_97', 'atex', 'bomex', 'dycoms2_rf01', 'dycoms2_rf02_ds', 'gabls3_night', 'lba', 'rico', 'twp_ice', 'wangara']
###DIFFING arm netCDF (*.nc) files###
>No differences detected by the linux diff in arm_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case arm.<<<
**********************************************************************************************************
###DIFFING arm_97 netCDF (*.nc) files###
>The linux diff detected differences in arm_97_stats.nc<
>>Differences above threshold were detected in the common fields in file arm_97_stats.nc<<
>>>Differences in common variables detected for case arm_97<<<
**********************************************************************************************************
###DIFFING atex netCDF (*.nc) files###
>No differences detected by the linux diff in atex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case atex.<<<
**********************************************************************************************************
###DIFFING bomex netCDF (*.nc) files###
>No differences detected by the linux diff in bomex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case bomex.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf01 netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf01_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf01.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf02_ds netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf02_ds_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf02_ds.<<<
**********************************************************************************************************
###DIFFING gabls3_night netCDF (*.nc) files###
>No differences detected by the linux diff in gabls3_night_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case gabls3_night.<<<
**********************************************************************************************************
###DIFFING lba netCDF (*.nc) files###
>No differences detected by the linux diff in lba_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case lba.<<<
**********************************************************************************************************
###DIFFING rico netCDF (*.nc) files###
>No differences detected by the linux diff in rico_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case rico.<<<
**********************************************************************************************************
###DIFFING twp_ice netCDF (*.nc) files###
>The linux diff detected differences in twp_ice_stats.nc<
>>Differences above threshold were detected in the common fields in file twp_ice_stats.nc<<
>>>Differences in common variables detected for case twp_ice<<<
**********************************************************************************************************
###DIFFING wangara netCDF (*.nc) files###
>No differences detected by the linux diff in wangara_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case wangara.<<<
**********************************************************************************************************
SUMMARY:
There were differences detected in the common variables in netCDF (*.nc) files.
Directory 1 is 9f2b1e6c9370a897e1da1320521fbc27baf6ff31
Directory 2 is d20fd1daad5e5d921607794e258bfde151fd3c89
Using reporting threshold: 0.0
The following cases will be compared: ['arm', 'arm_97', 'atex', 'bomex', 'dycoms2_rf01', 'dycoms2_rf02_ds', 'gabls3_night', 'lba', 'rico', 'twp_ice', 'wangara']
###DIFFING arm netCDF (*.nc) files###
>No differences detected by the linux diff in arm_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case arm.<<<
**********************************************************************************************************
###DIFFING arm_97 netCDF (*.nc) files###
>No differences detected by the linux diff in arm_97_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case arm_97.<<<
**********************************************************************************************************
###DIFFING atex netCDF (*.nc) files###
>No differences detected by the linux diff in atex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case atex.<<<
**********************************************************************************************************
###DIFFING bomex netCDF (*.nc) files###
>No differences detected by the linux diff in bomex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case bomex.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf01 netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf01_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf01.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf02_ds netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf02_ds_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf02_ds.<<<
**********************************************************************************************************
###DIFFING gabls3_night netCDF (*.nc) files###
>No differences detected by the linux diff in gabls3_night_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case gabls3_night.<<<
**********************************************************************************************************
###DIFFING lba netCDF (*.nc) files###
>No differences detected by the linux diff in lba_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case lba.<<<
**********************************************************************************************************
###DIFFING rico netCDF (*.nc) files###
>No differences detected by the linux diff in rico_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case rico.<<<
**********************************************************************************************************
###DIFFING twp_ice netCDF (*.nc) files###
>No differences detected by the linux diff in twp_ice_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case twp_ice.<<<
**********************************************************************************************************
###DIFFING wangara netCDF (*.nc) files###
>No differences detected by the linux diff in wangara_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case wangara.<<<
**********************************************************************************************************
SUMMARY:
Linux diff did not detect any differences in the compared files.
Directory 1 is 149e61f9bab26c5cb4518fd7bada9b1f4064482e
Directory 2 is 9f2b1e6c9370a897e1da1320521fbc27baf6ff31
Using reporting threshold: 0.0
The following cases will be compared: ['arm', 'arm_97', 'atex', 'bomex', 'dycoms2_rf01', 'dycoms2_rf02_ds', 'gabls3_night', 'lba', 'rico', 'twp_ice', 'wangara']
###DIFFING arm netCDF (*.nc) files###
>No differences detected by the linux diff in arm_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case arm.<<<
**********************************************************************************************************
###DIFFING arm_97 netCDF (*.nc) files###
>The linux diff detected differences in arm_97_stats.nc<
>>Differences above threshold were detected in the common fields in file arm_97_stats.nc<<
>>>Differences in common variables detected for case arm_97<<<
**********************************************************************************************************
###DIFFING atex netCDF (*.nc) files###
>No differences detected by the linux diff in atex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case atex.<<<
**********************************************************************************************************
###DIFFING bomex netCDF (*.nc) files###
>No differences detected by the linux diff in bomex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case bomex.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf01 netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf01_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf01.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf02_ds netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf02_ds_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf02_ds.<<<
**********************************************************************************************************
###DIFFING gabls3_night netCDF (*.nc) files###
>No differences detected by the linux diff in gabls3_night_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case gabls3_night.<<<
**********************************************************************************************************
###DIFFING lba netCDF (*.nc) files###
>No differences detected by the linux diff in lba_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case lba.<<<
**********************************************************************************************************
###DIFFING rico netCDF (*.nc) files###
>No differences detected by the linux diff in rico_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case rico.<<<
**********************************************************************************************************
###DIFFING twp_ice netCDF (*.nc) files###
>The linux diff detected differences in twp_ice_stats.nc<
>>Differences above threshold were detected in the common fields in file twp_ice_stats.nc<<
>>>Differences in common variables detected for case twp_ice<<<
**********************************************************************************************************
###DIFFING wangara netCDF (*.nc) files###
>No differences detected by the linux diff in wangara_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case wangara.<<<
**********************************************************************************************************
SUMMARY:
There were differences detected in the common variables in netCDF (*.nc) files.
Directory 1 is 0fc97f413d522aabb24db8c4a289aa5e5471256d
Directory 2 is 149e61f9bab26c5cb4518fd7bada9b1f4064482e
Using reporting threshold: 0.0
The following cases will be compared: ['arm', 'arm_97', 'atex', 'bomex', 'dycoms2_rf01', 'dycoms2_rf02_ds', 'gabls3_night', 'lba', 'rico', 'twp_ice', 'wangara']
###DIFFING arm netCDF (*.nc) files###
>No differences detected by the linux diff in arm_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case arm.<<<
**********************************************************************************************************
###DIFFING arm_97 netCDF (*.nc) files###
>No differences detected by the linux diff in arm_97_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case arm_97.<<<
**********************************************************************************************************
###DIFFING atex netCDF (*.nc) files###
>No differences detected by the linux diff in atex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case atex.<<<
**********************************************************************************************************
###DIFFING bomex netCDF (*.nc) files###
>No differences detected by the linux diff in bomex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case bomex.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf01 netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf01_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf01.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf02_ds netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf02_ds_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf02_ds.<<<
**********************************************************************************************************
###DIFFING gabls3_night netCDF (*.nc) files###
>No differences detected by the linux diff in gabls3_night_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case gabls3_night.<<<
**********************************************************************************************************
###DIFFING lba netCDF (*.nc) files###
>No differences detected by the linux diff in lba_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case lba.<<<
**********************************************************************************************************
###DIFFING rico netCDF (*.nc) files###
>No differences detected by the linux diff in rico_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case rico.<<<
**********************************************************************************************************
###DIFFING twp_ice netCDF (*.nc) files###
>No differences detected by the linux diff in twp_ice_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case twp_ice.<<<
**********************************************************************************************************
###DIFFING wangara netCDF (*.nc) files###
>No differences detected by the linux diff in wangara_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case wangara.<<<
**********************************************************************************************************
SUMMARY:
Linux diff did not detect any differences in the compared files.
Directory 1 is d9d887857da333c2c683aad09311ffcdb2d54440
Directory 2 is 0fc97f413d522aabb24db8c4a289aa5e5471256d
Using reporting threshold: 0.0
The following cases will be compared: ['arm', 'arm_97', 'atex', 'bomex', 'dycoms2_rf01', 'dycoms2_rf02_ds', 'gabls3_night', 'lba', 'rico', 'twp_ice', 'wangara']
###DIFFING arm netCDF (*.nc) files###
>No differences detected by the linux diff in arm_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case arm.<<<
**********************************************************************************************************
###DIFFING arm_97 netCDF (*.nc) files###
>No differences detected by the linux diff in arm_97_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case arm_97.<<<
**********************************************************************************************************
###DIFFING atex netCDF (*.nc) files###
>No differences detected by the linux diff in atex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case atex.<<<
**********************************************************************************************************
###DIFFING bomex netCDF (*.nc) files###
>No differences detected by the linux diff in bomex_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case bomex.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf01 netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf01_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf01.<<<
**********************************************************************************************************
###DIFFING dycoms2_rf02_ds netCDF (*.nc) files###
>No differences detected by the linux diff in dycoms2_rf02_ds_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case dycoms2_rf02_ds.<<<
**********************************************************************************************************
###DIFFING gabls3_night netCDF (*.nc) files###
>No differences detected by the linux diff in gabls3_night_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case gabls3_night.<<<
**********************************************************************************************************
###DIFFING lba netCDF (*.nc) files###
>No differences detected by the linux diff in lba_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case lba.<<<
**********************************************************************************************************
###DIFFING rico netCDF (*.nc) files###
>No differences detected by the linux diff in rico_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case rico.<<<
**********************************************************************************************************
###DIFFING twp_ice netCDF (*.nc) files###
>No differences detected by the linux diff in twp_ice_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case twp_ice.<<<
**********************************************************************************************************
###DIFFING wangara netCDF (*.nc) files###
>No differences detected by the linux diff in wangara_stats.nc<
>>>The linux diff could not detect any differences in the file pairs for case wangara.<<<
**********************************************************************************************************
SUMMARY:
Linux diff did not detect any differences in the compared files.
=============================== Configuring Test ===============================
Working directory set to: /home/jenkins/workspace/clubb_BFB_e3sm_flags_gfortran_test
Checking specified repo: /home/jenkins/workspace/clubb_BFB_e3sm_flags_gfortran_test
Using /home/jenkins/workspace/clubb_BFB_e3sm_flags_gfortran_test with origin https://github.com/larson-group/clubb.git
Output Save Mode: none
Using branch: master
Making list from 'UnresolvedCommits':
CheckRange = True: Getting commit range
Run Output Path: output
Run commands: ['python compile.py -debug', '\nrm -rf output', '\nmkdir output', '\nmv input/tunable_parameters_e3sm/configurable_model_flags.in input/tunable_parameters', '\npython run_scripts/run_scm_all.py -min_cases']
Diffing command: run_scripts/run_bindiff_all.py
=============================== Running Test ===============================
Parsing message from commit: 9f2b1e6c9370a897e1da1320521fbc27baf6ff31
Parsing message from commit: 149e61f9bab26c5cb4518fd7bada9b1f4064482e
Parsing message from commit: 0fc97f413d522aabb24db8c4a289aa5e5471256d
Parsing message from commit: d9d887857da333c2c683aad09311ffcdb2d54440
Checking Commit: 9f2b1e6c9370a897e1da1320521fbc27baf6ff31
Skipping: not final commit and not labelled BIT_CHANGING
Checking Commit: 149e61f9bab26c5cb4518fd7bada9b1f4064482e
Skipping: not final commit and not labelled BIT_CHANGING
Checking Commit: 0fc97f413d522aabb24db8c4a289aa5e5471256d
Skipping: not final commit and not labelled BIT_CHANGING
Checking Commit: d9d887857da333c2c683aad09311ffcdb2d54440
Running: final commit to check, not labelled BIT_CHANGING
Checking out d9d887857da333c2c683aad09311ffcdb2d54440
Running: 'python compile.py -debug'
Running: 'rm -rf output'
Running: 'mkdir output'
Running: 'mv input/tunable_parameters_e3sm/configurable_model_flags.in input/tunable_parameters'
Running: 'python run_scripts/run_scm_all.py -min_cases'
Checking out d20fd1daad5e5d921607794e258bfde151fd3c89
Running: 'python compile.py -debug'
Running: 'rm -rf output'
Running: 'mkdir output'
Running: 'mv input/tunable_parameters_e3sm/configurable_model_flags.in input/tunable_parameters'
Running: 'python run_scripts/run_scm_all.py -min_cases'
Diffing: 'run_scripts/run_bindiff_all.py d9d887857da333c2c683aad09311ffcdb2d54440 d20fd1daad5e5d921607794e258bfde151fd3c89'
ISSUE WITH COMMIT DETECTED: d9d887857da333c2c683aad09311ffcdb2d54440 differs from d20fd1daad5e5d921607794e258bfde151fd3c89
Finding all BIT_CHANGING commits in range: d20fd1daad5e5d921607794e258bfde151fd3c89...d9d887857da333c2c683aad09311ffcdb2d54440
Checking out 9f2b1e6c9370a897e1da1320521fbc27baf6ff31
Running: 'python compile.py -debug'
Running: 'rm -rf output'
Running: 'mkdir output'
Running: 'mv input/tunable_parameters_e3sm/configurable_model_flags.in input/tunable_parameters'
Running: 'python run_scripts/run_scm_all.py -min_cases'
Output from d20fd1daad5e5d921607794e258bfde151fd3c89 already saved
Diffing: 'run_scripts/run_bindiff_all.py 9f2b1e6c9370a897e1da1320521fbc27baf6ff31 d20fd1daad5e5d921607794e258bfde151fd3c89'
Checking out 149e61f9bab26c5cb4518fd7bada9b1f4064482e
Running: 'python compile.py -debug'
Running: 'rm -rf output'
Running: 'mkdir output'
Running: 'mv input/tunable_parameters_e3sm/configurable_model_flags.in input/tunable_parameters'
Running: 'python run_scripts/run_scm_all.py -min_cases'
Output from 9f2b1e6c9370a897e1da1320521fbc27baf6ff31 already saved
Diffing: 'run_scripts/run_bindiff_all.py 149e61f9bab26c5cb4518fd7bada9b1f4064482e 9f2b1e6c9370a897e1da1320521fbc27baf6ff31'
Checking out 0fc97f413d522aabb24db8c4a289aa5e5471256d
Running: 'python compile.py -debug'
Running: 'rm -rf output'
Running: 'mkdir output'
Running: 'mv input/tunable_parameters_e3sm/configurable_model_flags.in input/tunable_parameters'
Running: 'python run_scripts/run_scm_all.py -min_cases'
Output from 149e61f9bab26c5cb4518fd7bada9b1f4064482e already saved
Diffing: 'run_scripts/run_bindiff_all.py 0fc97f413d522aabb24db8c4a289aa5e5471256d 149e61f9bab26c5cb4518fd7bada9b1f4064482e'
Output from d9d887857da333c2c683aad09311ffcdb2d54440 already saved
Output from 0fc97f413d522aabb24db8c4a289aa5e5471256d already saved
Diffing: 'run_scripts/run_bindiff_all.py d9d887857da333c2c683aad09311ffcdb2d54440 0fc97f413d522aabb24db8c4a289aa5e5471256d'
Found:
149e61f9bab26c5cb4518fd7bada9b1f4064482e
=============================== Test Results ===============================
Commit Range Checked = d20fd1daad5e5d921607794e258bfde151fd3c89...d9d887857da333c2c683aad09311ffcdb2d54440
New Unresolved Commits:
149e61f9bab26c5cb4518fd7bada9b1f4064482e
No fake BIT_CHANGING commits found.
Remaining Unresolved Commits (oldest commit is listed first):
149e61f9bab26c5cb4518fd7bada9b1f4064482e
=============================== Updating Config File ===============================
Saving into UnresolvedCommits: 149e61f9bab26c5cb4518fd7bada9b1f4064482e
Saving into BaselineCommit: d9d887857da333c2c683aad09311ffcdb2d54440
=============================== Cleanup ===============================
Removing all generated output
REMOVING: d20fd1daad5e5d921607794e258bfde151fd3c89
REMOVING: 9f2b1e6c9370a897e1da1320521fbc27baf6ff31
REMOVING: 149e61f9bab26c5cb4518fd7bada9b1f4064482e
REMOVING: 0fc97f413d522aabb24db8c4a289aa5e5471256d
REMOVING: d9d887857da333c2c683aad09311ffcdb2d54440
================== Unresolved Commit Status ==================
Remaining Unresolved Commits (oldest commit is listed first):
149e61f9bab26c5cb4518fd7bada9b1f4064482e
Exiting with code: 1
