problem with too many submition to slurm scheduler

[agrouaze]

to reproduce

micromamba activate /scale/project/lops-siam-airflow/envs_exploit/micromamba/py3_submitit
turboblaster --listing-input /home/agrouaze/sources/slurm_aws_training/submitit_tests/test5_apptainer_sif_dataaccess_massive/wv_ocn_safe_s1d_listing.txt --bash-slurm-exec test5_apptainer_sif_dataaccess_massive/hello_world.slurm --output-dir test5_apptainer_sif_dataaccess_massive/logs/

listing 1329 lines: see attached doc.

wv_ocn_safe_s1d_listing.txt

traceback

srun: error: slurm_send_node_msg: [socket:[10013027]] slurm_bufs_sendto(msg_type=REQUEST_HET_JOB_ALLOC_INFO) failed: Unexpected missing socket error
srun: error: Unable to confirm allocation for job 6479: Unexpected missing socket error
srun: Check SLURM_JOB_ID environment variable. Expired or invalid job 6479

payload

slurm hello_world.slurm

#!/bin/bash
#SBATCH --job-name=test-python
#SBATCH --output=output_%j.txt
#SBATCH --time=00:30:00
#SBATCH --ntasks=1
#SBATCH --mem=2G
#SBATCH --partition=cpu
hostname
#module load python/3.10
# Move to the directory where THIS script is located
set -e  # <--- CRITICAL: Exit immediately if a command fails
set -x  # <--- HELPFUL: Print commands as they are executed
cd "$(dirname "$0")"
which python
python --version
#python helloworld.py 31
# img=/scale/project/lops-siam-airflow/envs_exploit/apptainer/s1ifr-2026.2.9.sif
# img=/scale/project/lops-siam-airflow/envs_exploit/apptainer/unifiedwvalticolocs_2026.2.17.sif
# img=/scale/project/lops-siam-airflow/envs_exploit/apptainer/update_large_netcdf_calibration_2026.2.3.sif
# img=/scale/project/lops-siam-airflow/envs_exploit/apptainer/kpimpc_20260113_apptainer.sif
img=/scale/project/lops-siam-airflow/envs_exploit/apptainer/unifiedwvalticolocs_2026.2.17.sif
exe=/home/agrouaze/sources/slurm_aws_training/submitit_tests/test3_apptainer_sif_dataaccess/helloworld.py
echo "Running Apptainer container with image: $img and executable: $exe"
# binpython=/opt/conda/envs/kpimpc/bin/python
binpython=python
safeinput=$1
apptainer run -B /legacy -B /scale -B /home -B /ontap $img $binpython $exe --safeinput $1

python helloworld.py

import logging
import sys,os
import time
import xarray as xr
from tqdm import tqdm
import glob
import argparse
if __name__ == '__main__':
    # Force logs to stdout so they stay in sync with prints
    logging.basicConfig(
        level=logging.INFO,
        format='%(levelname)s:%(message)s',
        stream=sys.stdout 
    )
    # parse arguments
    parser = argparse.ArgumentParser(description="read SAR for test.")
    parser.add_argument(
        "--safeinput",
        type=str,
        required=True,
        help="full path of OCN SAFE data WV",
    )

    args = parser.parse_args()

    logging.info('eh oui ce log est un hello world! from inside a apptainer container')
    print('hello world from test3', flush=True)
    assert os.path.exists('/legacy/project/cersat/public')

    didi=args.safeinput
    logging.info('Reading files from %s', didi)
    list_of_files = glob.glob(os.path.join(didi, "*.nc"))
    print('number of files to read:', len(list_of_files), flush=True)
    lons = []
    # for ii in tqdm(
    #         range(len(list_of_files)), 
    #         file=sys.stdout,      # Sync with your print statements
    #         mininterval=5,        # Only update every 5 seconds (prevents massive log files)
    #         ascii=True            # Use standard characters instead of smooth blocks (cleaner in logs)
    #     ):
    for ii in tqdm(range(len(list_of_files)), file=sys.stdout, mininterval=1, ascii=True):
        filepath = list_of_files[ii]
        print(f"DEBUG: Attempting to open {filepath}", flush=True) # <--- ADD THIS
        
        try:
            # Try specifying the engine explicitly if you know it
            ds = xr.open_dataset(filepath, engine='h5netcdf')  
            lons.append(ds.oswLon.values.squeeze())
            ds.close()
            print(f"DEBUG: Successfully processed {filepath}", flush=True)
        except Exception as e:
            print(f"ERROR processing {filepath}: {e}", file=sys.stderr, flush=True)
            raise e # Re-raise so the script exits with error
    # display python version
    logging.info('longitudes values : %s', lons)
    print('longitudes values :', lons, flush=True)
    print('python version:', flush=True)
    print(sys.version, flush=True)  
    print('sleep', flush=True)
    time.sleep(3)
    logging.info('Task completed successfully, python talking.!')