MDSS migration

  • "man mdss" for command usage
  • Use copyq queue for migration of data to MDSS
  • If there are lots of small files please tar/gzip them up
  • An example script to migrate a year's worth of data, with one job submits itself to process the next month (run for a month first to work out the elapsed time required)
#PBS -P dp9
#PBS -q copyq
#PBS -e /home/548/ttl548/cpq.err
#PBS -o /home/548/ttl548/cpq.out
#PBS -l walltime=03:00:00
#PBS -l wd
#PBS -l storage=gdata/access+gdata/hh5+gdata/ig2

m=(0 31 28 31 30 31 30 31 31 30 31 30 31)
ymd=$(cat /home/548/ttl548/cron/
yy=$(( $ymd/10000 ))
typeset -RZ2 mm=$(( $ymd%10000/100 ))
cd /g/data/ig2/usavm/1
while [ $ymd -le $end ]; do
  mdss -P ig2 put -r $ymd usavm/1/$yy
  ymd=$(/g/data/access/apps/cawcr-utils/1.0.0/bin/adddate $ymd 1)
# echo $ymd > /home/548/ttl548/cron/
# qsub /home/548/ttl548/cron/
if [ $mm -lt 12 ]; then
  typeset -RZ2 mm=$(( mm+1 ))
  echo $ymd > /home/548/ttl548/cron/
  qsub /home/548/ttl548/cron/
Last modified 8 months ago Last modified on Apr 20, 2021 4:12:02 PM