Workflow 500, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	2097152000 (2000 MiB)
Max distance for inputs	0.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MANCHESTER, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MANCHESTER, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites
Scope	usertests
Events for this stage

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
3	0	3	0	0	0	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
38546	96	0	0	0	26172	0	49	11300	0	929	0

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

Jobscript

#!/bin/bash
:<<'EOF'
justin simple-workflow \
  --rss-mb 2000 \
  --description 'Automated Workflow Tests' \
  --mql 'rucio-dataset testpro:awt' \
  --jobscript awt.jobscript
EOF

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup python v3_9_13
setup metacat
setup rucio
export X509_USER_PROXY=$JUSTIN_PATH/awt-proxy.pem
export X509_NOROLES_PROXY=$JUSTIN_PATH/justin-jobs-no-roles.proxy.pem
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export RUCIO_ACCOUNT=dunepro
setup justin
unset GFAL_CONFIG_DIR GFAL_PLUGIN_DIR
which metacat
metacat --version
which rucio
rucio --version
python3 -c 'import gfal2 ; print("gfal version",gfal2.get_version())'
metacat auth login -m x509 dunepro
export upload_dataset=`date --utc +'awt-uploads-%Y%W'`

touch results.txt
export now=`date +'%s'`

echo '===Start $JUSTIN_PATH/justin-awt-rse-list.txt==='
cat "$JUSTIN_PATH/justin-awt-rse-list.txt"
echo '===End $JUSTIN_PATH/justin-awt-rse-list.txt==='

cat $JUSTIN_PATH/justin-awt-rse-list.txt | (

while read rse_name write_protocol read_pfn
do
  echo '---------------------------------------------------------------------'
  echo $JUSTIN_SITE_NAME $rse_name $write_protocol $read_pfn

  # Test download of file prepositioned on RSEs using normal jobscript proxy
  X509_USER_PROXY=$X509_NOROLES_PROXY xrdcp --force --nopbar --verbose \
    "$read_pfn" "downloaded.txt"
  download_retval=$?

  echo "'xrdcp --force --nopbar --verbose $read_pfn downloaded.txt' returns $download_retval"

  # Test upload of random file
  fn=`mktemp awt-$now-XXXXXXXXXX`
  echo "$fn" > $fn

  echo '{"namespace":"testpro","name":"'$fn'","size":0}' >tmp.json
  metacat file declare --json -f tmp.json "dune:all"
  ret=$?
  echo "metacat file declare returns $ret"

  echo "GFAL_CONFIG_DIR: $GFAL_CONFIG_DIR   GFAL_PLUGIN_DIR: $GFAL_PLUGIN_DIR"
  for i in 1 2 3 4 5
  do
    echo "justin-rucio-upload attempt $i"
    justin-rucio-upload \
            --rse "$rse_name" \
            --protocol "$write_protocol" \
            --scope testpro \
            --dataset $upload_dataset \
            --timeout 1200 \
            "$fn" 2>&1 
#            | sed -r 's/\x1B\[([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGK]//g'
    upload_retval=$?
    if [ $upload_retval != 63 ] ; then
      break
    fi
  done

  echo "'justin-rucio-upload --rse $rse_name --protocol $write_protocol --scope testpro --dataset $upload_dataset $fn --timeout 1200' returns $upload_retval"
  echo
  echo

  echo "==awt== $JUSTIN_SITE_NAME $rse_name $download_retval $upload_retval" \
    "$read_pfn $write_protocol" >> results.txt

done

)

voms-proxy-info --all --file $X509_USER_PROXY

echo
echo '===== Results ====='
echo
echo 'Download/upload commands:'
echo 'xrdcp --force --nopbar --verbose $read_pfn downloaded.txt'
echo echo \''{"namespace":"testpro","name":"FILENAME","size":0}'\' \>tmp.json
echo 'metacat file declare --json -f tmp.json "dune:all"'
echo 'justin-rucio-upload --rse $rse_name --protocol $write_protocol --scope testpro --dataset '$upload_dataset' --timeout 1200 FILENAME'
echo 'Use the wrapper job link on the page for the job on the justIN Dashboard to find the full log file, with errors from these commands'
echo
echo 'Each line: $JUSTIN_SITE_NAME $rse_name $download_retval $upload_retval $read_pfn $write_protocol'
cat results.txt