Link to home
Start Free TrialLog in
Avatar of dcorliss
dcorliss

asked on

Writing large file (2.4 GB) to a multi-tape file.

I have inherited a UNIX script that writes a file to a single tape.  The file has become so large (2.4 GB) that it will not fit on a single tape.  I need to alter the script to accomodate the use of multi-tape.  I am looking for examples of UNIX scripts that allow for the writing of a large file to a multi-tape file.
Avatar of yuzh
yuzh
Avatar of yuzh
yuzh
Just a reminder:

cpio, GUN tar, Solaris ufsdump can handle multi volumes.

The native unix tar is OS depandent.
Avatar of bira
bira
What kind of Unix do you have?
If AIX, the backup utiliy supports multi volume tapes.
Avatar of dcorliss
dcorliss

ASKER

I thank you for your comments but I have a specially written script (running on Solaris 2.8) that does data manipulation.  How can I get a copy of the script to you for study?
Avatar of dcorliss
dcorliss

ASKER

There's a section (exit) written to blow up if the tape count is greater then 1.  It is here I need to handle going to another tape.  I assume I have to close the current tape volume and open another but this I do not know how to do.  
Avatar of dcorliss
dcorliss

ASKER

Script:

#!/usr/bin/ksh

#
# set DEBUG=echo to run in DEBUG mode, which will not access the tape drive
#

DEBUG=
#DEBUG=echo

VALID_NUM_PARMS=8
VALID_NUM_PARMS_OPT=9
USAGE="write_tape.sh source_dir vol_ser_alpha first_vol_ser_num file_name_base lrecl blksize rewind|norewind|eject 3480|3490|3490E [vb_data_write.pl ]"

if [ $# -ne $VALID_NUM_PARMS ]
then
      if [ $# -ne $VALID_NUM_PARMS_OPT ]
      then
            echo "Invalid number of parameters ($#) - Expecting $VALID_NUM_PARMS or $VALID_NUM_PARMS_OPT"
            echo $USAGE
            exit -1
      fi
fi

# Accept parameters
SRC_DIR=$1;shift
VOL_SER_ALPHA=$1;shift
VOL_SER_NUM_FIRST=$1;shift
FILE_NAME_BASE=$1;shift
LRECL=$1;shift
BLKSIZE=$1;shift
EOF_OPTION=$1;shift
CART_TYPE=$1;shift


#!!TAPE_OPTIONS_DATA="conv=block,ebcdic obs=$BLKSIZE cbs=$LRECL "
#!!TAPE_OPTIONS_DATA="$TAPE_OPTIONS_CONVERT obs=$BLKSIZE cbs=$LRECL "

#Default record format: F (Fixed Block)
RECFM=F
# Optional Data Conversion parameter
if [ $# -ge 1 ]
then
      DATA_FILTER_PARM=$1
      if [ "noconvert" = "$DATA_FILTER_PARM" ]
      then
            : # OK
            shift
            NO_CONVERT_CHAR_SET=1
            FILTER_DATA=0
            TAPE_OPTIONS_CONVERT=" "
      else
            # See if DATA_FILTER_PARM is vb_data_write.pl
            if [ "vb_data_write.pl" = "$DATA_FILTER_PARM" ]
            then
                  shift
                  NO_CONVERT_CHAR_SET=1
                  #record format: V (Variable Block)
                  RECFM=V
                  FILTER_DATA=1
                  TAPE_OPTIONS_CONVERT=" "
                  DATA_OPTIONS_CONVERT=$*
                  if [ -n "$DATA_OPTIONS_CONVERT" ]
                  then
                        echo "Data filter options found ==>$DATA_OPTIONS_CONVERT<==. Ignring."
                  fi
            else
                  echo "Invalid noconvert option ==>$NO_CONVERT_CHAR_SET<==.  Valid: noconvert or vb_data_write.pl"
                  echo $USAGE
                  exit -1
            fi
      fi
else
      NO_CONVERT_CHAR_SET=0
      FILTER_DATA=0
      TAPE_OPTIONS_CONVERT="conv=block,ebcdic "
fi


# Validate Destination Directory
if [ -d $SRC_DIR ]
then
      if [ -w $SRC_DIR ]
      then
            : # Cool.  Can write to directory.
      else
            echo $LOGNAME does not have write authority to directory $SRC_DIR
            echo exiting
            exit -1
      fi
else
      echo Input Directory $SRC_DIR is not a directory
      echo exiting
      exit -1
fi

# Validate Destination Directory
if [ -f ${SRC_DIR}/${FILE_NAME_BASE} ]
then
      : # Cool. File exisits
else
      echo Input  File ${SRC_DIR}/${FILE_NAME_BASE} not found
      echo exiting
      exit -1
fi

if [ "rewind" = "$EOF_OPTION" -o "eject" = "$EOF_OPTION" -o "norewind" = "$EOF_OPTION" ]
then
      : # OK
else
      echo "Invalid end of tape option ==>$EOF_OPTION<==.  Valid: rewind|norewind|eject"
      echo $USAGE
      exit -1
fi

if [ "3480" = "$CART_TYPE" -o "3490" = "$CART_TYPE" -o "3490E" = "$CART_TYPE" ]
then
      : # OK
      if [ "3480" = "$CART_TYPE" ]
      then
            CART_CAPACITY=419429996
      else
            if [ "3490" = "$CART_TYPE" ]
            then
                  CART_CAPACITY=838860396
            else
                  # !! CART_CAPACITY=2576979974
                  # !! Future: Use info from ../log/tape_size_hist.txt
                  CART_CAPACITY=2722230600
            fi
      fi
else
      echo "Invalid tape cartridge option ==>$CART_TYPE<==.  \
            Valid: 3480|3490|3490E"
      echo $USAGE
      exit -1
fi

# Validate numeric parms
let TEST_NUM=$VOL_SER_NUM_FIRST+1
if [ $? -ne 0 ]
then
      echo "Non-numeric Vol Ser Numeric portion ==>$VOL_SER_NUM_FIRST<==."
      echo $USAGE
      exit -1
fi

let TEST_NUM=$LRECL+1
if [ $? -ne 0 ]
then
      echo "Logical Record Length is non-numeric ==>$LRECL<==."
      echo $USAGE
      exit -1
fi

let TEST_NUM=$BLKSIZE+1
if [ $? -ne 0 ]
then
      echo "Block size is non-numeric ==>$BLKSIZE<==."
      echo $USAGE
      exit -1
fi


set -x # !! DEBUG - comment out this line when all is cool.

# Calc various info

# Determine # of tapes
FILE_SIZE=`stat --dereference --format='%s' ${SRC_DIR}/${FILE_NAME_BASE}`

LF_COUNT_ALL=`wc -l ${SRC_DIR}/${FILE_NAME_BASE}`
LF_COUNT=`echo $LF_COUNT_ALL | tr -s ' ' | cut -d' ' -f1 | tr -d ' '`

if [ -z "$LF_COUNT" ]
then
      LF_COUNT=`echo $LF_COUNT_ALL | tr -s ' ' | cut -d' ' -f2 | tr -d ' '`
fi

# The following ugly code attempts to round up the TTL_BLOCKS to the next
# integer, since the korn shell uses integer arithmetic.
let MULT_FACTOR=100000
let TTL_BLOCKS_MULT=\(\($FILE_SIZE-$LF_COUNT\)*$MULT_FACTOR\)/$BLKSIZE
# Use mod operator to see if need to round
let REMAINDER=$TTL_BLOCKS_MULT%$MULT_FACTOR
if [ $REMAINDER -ne 0 ]
then
      let TTL_BLOCKS=\(\($FILE_SIZE-$LF_COUNT\)/$BLKSIZE\)+1
else
      let TTL_BLOCKS=\($FILE_SIZE-$LF_COUNT\)/$BLKSIZE
fi

# This does not take into account header/trailer files on each tape,
# nor blksize%cart_capacity
let NUM_TAPES=$FILE_SIZE/$CART_CAPACITY+1

echo FILE_SIZE=$FILE_SIZE, TTL_BLOCKS=$TTL_BLOCKS, NUM_TAPES=$NUM_TAPES
echo LF_COUNT=$LF_COUNT

let VOL_SER_NUM_LAST=$VOL_SER_NUM_FIRST+$NUM_TAPES-1

# pad with zeros if needed
LEN_NUM_FIRST=`echo $VOL_SER_NUM_FIRST | wc -c | tr -d ' '`
LEN_NUM_LAST=`echo $VOL_SER_NUM_LAST | wc -c | tr -d ' '`
echo HEY LEN_NUM_FIRST=$LEN_NUM_FIRST, LEN_NUM_LAST=$LEN_NUM_LAST
let LEN_DIFF=$LEN_NUM_FIRST-$LEN_NUM_LAST
if [ $LEN_DIFF -gt 0 ]
then
      VOL_SER_NUM_LAST=`echo "0000000" | cut -c1-$LEN_DIFF`$VOL_SER_NUM_LAST
fi


if [ $NUM_TAPES -eq 1 ]
then
      TAPE_INFO=tape
      TAPE_INFO2="tape is"
      TAPE_INFO3="it"
else
      TAPE_INFO=tapes
      TAPE_INFO2="tapes are"
      TAPE_INFO3="the first tape"
fi

if [ $NUM_TAPES -gt 10 ]
then
      echo $0 cannot currently handle writing more than 10 tapes.  Sorry.
      echo Modify loop in $0 so that it pauses every 10 tapes for unmounting
      echo written tapes, and mounting x \(up to 10\) blank tapes before going on.
      exit -1
else
      if [ $NUM_TAPES -gt 1 ]
      then
            echo $0 cannot currently handle writing more than 1 tape.  Sorry.
            echo Modify loop in $0 to figure out blocks per tape, and use
            echo dd parms iseek=n and count=n to bust apart file.
            exit -1
      fi
      echo Ensure $NUM_TAPES blank $TAPE_INFO2 in the tape library
      echo and $TAPE_INFO3 is loaded
fi


echo Hit Enter to write $NUM_TAPES $TAPE_INFO from ${SRC_DIR}/${FILE_NAME_BASE}
echo Vol Ser ${VOL_SER_ALPHA}${VOL_SER_NUM_FIRST} to \
      ${VOL_SER_ALPHA}${VOL_SER_NUM_LAST} blksize=$BLKSIZE \
      and $EOF_OPTION last tape
read stuff


# Clobber log files

echo about to "> $SRC_DIR/${FILE_NAME_BASE}.out 2> $SRC_DIR/${FILE_NAME_BASE}.err"
> $SRC_DIR/${FILE_NAME_BASE}.out 2> $SRC_DIR/${FILE_NAME_BASE}.err

if [ -w $SRC_DIR/${FILE_NAME_BASE}.out -a -w $SRC_DIR/${FILE_NAME_BASE}.err ]
then
      : # OK to write in this dir
else
      echo Unable to write files in `pwd`
      exit -1
fi

echo `date '+%D %T'` write_tape.sh going to write $NUM_TAPES $TAPE_INFO \
      starting with VOL_SER=${VOL_SER_ALPHA}${VOL_SER_NUM_FIRST}, \
      last=${VOL_SER_ALPHA}${VOL_SER_NUM_LAST} \
            | tee -a $SRC_DIR/${FILE_NAME_BASE}.out

echo       `date '+%D %T'` Base File Name=$FILE_NAME_BASE, LRECL=$LRECL, \
      Blocksize=$BLKSIZE \
            | tee -a $SRC_DIR/${FILE_NAME_BASE}.out

CURR_TAPE=$VOL_SER_NUM_FIRST

# 2: TAPE_OPTIONS="conv=ascii,unblock ibs=$BLKSIZE cbs=$LRECL "
# 3: TAPE_OPTIONS="conv=ascii,unblock ibs=$BLKSIZE "
# 4: TAPE_OPTIONS="conv=ascii ibs=$BLKSIZE cbs=$LRECL "
#!TAPE_OPTIONS_LABEL="conv=ebcdic,block obs=$BLKSIZE cbs=80 "
TAPE_OPTIONS_LABEL="conv=block,ebcdic cbs=80 "
#!TAPE_OPTIONS_DATA="conv=ebcdic,block obs=$BLKSIZE cbs=$LRECL "
#!!TAPE_OPTIONS_DATA="conv=block,ebcdic obs=$BLKSIZE cbs=$LRECL "
TAPE_OPTIONS_DATA="$TAPE_OPTIONS_CONVERT obs=$BLKSIZE cbs=$LRECL "


time $DEBUG mt status >> ${FILE_NAME_BASE}.out 2>> ${FILE_NAME_BASE}.err

TAPES_WRITTEN=1
while [ $CURR_TAPE -le $VOL_SER_NUM_LAST ]
do
      echo `date '+%D %T'` "working on Tape Vol Ser: ${VOL_SER_ALPHA}${CURR_TAPE}" \
            | tee -a $SRC_DIR/${FILE_NAME_BASE}.out $SRC_DIR/${FILE_NAME_BASE}.err


      ##
      # perl script to generate header file
      # !! To do:  Handle files spanning tapes
      ##
      echo `date '+%D %T'` "Creating vol ser header for ${VOL_SER_ALPHA}${CURR_TAPE}" \
            >> $SRC_DIR/${FILE_NAME_BASE}.out
      $DEBUG write_vol_ser.pl ${VOL_SER_ALPHA}${VOL_SER_NUM_FIRST} \
            ${VOL_SER_ALPHA}${CURR_TAPE} $FILE_NAME_BASE \
            $LRECL $BLKSIZE $TAPES_WRITTEN $NUM_TAPES $RECFM > \
            $SRC_DIR/${FILE_NAME_BASE}_${VOL_SER_ALPHA}${CURR_TAPE}_0

      echo `date '+%D %T'` "Writing vol ser header for ${VOL_SER_ALPHA}${CURR_TAPE}" \
            | tee -a $SRC_DIR/${FILE_NAME_BASE}.out $SRC_DIR/${FILE_NAME_BASE}.err

      time $DEBUG /usr/bin/dd of=/dev/rmt/0n \
            if=$SRC_DIR/${FILE_NAME_BASE}_${VOL_SER_ALPHA}${CURR_TAPE}_0 \
            $TAPE_OPTIONS_LABEL \
            >> $SRC_DIR/${FILE_NAME_BASE}.out 2>> $SRC_DIR/${FILE_NAME_BASE}.err
      time $DEBUG mt status \
            >> $SRC_DIR/${FILE_NAME_BASE}.out 2>> $SRC_DIR/${FILE_NAME_BASE}.err

      ##
      # Write data file
      # !! To do:  Handle files spanning tapes
      ##
      echo `date '+%D %T'` "Writing data for ${VOL_SER_ALPHA}${CURR_TAPE}" \
            | tee -a $SRC_DIR/${FILE_NAME_BASE}.out $SRC_DIR/${FILE_NAME_BASE}.err
      echo `date '+%D %T'` "Options: $TAPE_OPTIONS_DATA " \
            | tee -a $SRC_DIR/${FILE_NAME_BASE}.out $SRC_DIR/${FILE_NAME_BASE}.err

      set -x  # !! TEMP DEBUG
      if [ $FILTER_DATA -eq 1 ]
      then
            # Use dd with no "if" parm, pipe from DATA_FILTER_PARM
            time $DEBUG $DATA_FILTER_PARM blksize=$BLKSIZE  \
                  $SRC_DIR/${FILE_NAME_BASE} \
                  2>> $SRC_DIR/${FILE_NAME_BASE}.err | \
                  /usr/bin/dd of=/dev/rmt/0n \
                  $TAPE_OPTIONS_DATA \
                  >> $SRC_DIR/${FILE_NAME_BASE}.out 2>> $SRC_DIR/${FILE_NAME_BASE}.err
            RC_WRITE_DATA=$?
            # Need to grep $SRC_DIR/${FILE_NAME_BASE}.err to see if
            # $DATA_FILTER_PARM script had a
            # error, since the dd program might have masked a
            # bad RC
            grep '^Error:' $SRC_DIR/${FILE_NAME_BASE}.err
            if [ $? -eq 0 ]
            then
                  # Found string - there was an error
                  RC_WRITE_DATA=123
            fi

            # Need to obtain blocks written if $DATA_FILTER_PARM is vb_data_write.pl
            # since writing VB data means that we can't determine the number of
            # blocks written by simply dividing file size by block size,
            # since each record can be varying in length.
            if [ "vb_data_write.pl" = "$DATA_FILTER_PARM" ]
            then
                  if [ $NUM_TAPES -gt 1 ]
                  then
                        # Blow up for now
                        echo "write_tape.sh: I don't know how you got here 2, but you shouldn't have!"
                        exit -1
                  else
                        TTL_BLOCKS=`grep 'vb_data_write finished.* blocks written' $SRC_DIR/${FILE_NAME_BASE}.err | cut -f 9 -d'|'`
                        #echo "HEY: Just set TTL_BLOCKS to $TTL_BLOCKS "
                        #echo "HEY: TTL_BLOCKS=grep 'vb_data_write finished.* blocks written' $SRC_DIR/${FILE_NAME_BASE}.err | cut -f 9 -d'|'"
                  fi
            fi
      else
            # Write directly to file
            time $DEBUG /usr/bin/dd of=/dev/rmt/0n \
                  if=$SRC_DIR/${FILE_NAME_BASE} \
                  $TAPE_OPTIONS_DATA \
                  >> $SRC_DIR/${FILE_NAME_BASE}.out 2>> $SRC_DIR/${FILE_NAME_BASE}.err
            RC_WRITE_DATA=$?
      fi

      if [ $RC_WRITE_DATA -ne 0 ]
      then
            echo `date '+%D %T'` "Error $RC_WRITE_DATA Writing data " \
            | tee -a $SRC_DIR/${FILE_NAME_BASE}.out $SRC_DIR/${FILE_NAME_BASE}.err
            echo You may need to mt rewind this tape
            exit -1
      fi
      set +x  # !! TEMP DEBUG

      time $DEBUG mt status \
            >> $SRC_DIR/${FILE_NAME_BASE}.out 2>> $SRC_DIR/${FILE_NAME_BASE}.err

      ##
      #
      # perl script to generate trailer file
      # !! To do:  Handle files spanning tapes
      #
      ##
      if [ $NUM_TAPES -gt 1 ]
      then
            # Blow up for now
            echo "write_tape.sh: I don't know how you got here, but you shouldn't have!"
            exit -1
      else
            #let TTL_BLOCKS=$FILE_SIZE/$BLKSIZE
            let CURR_BLOCKS=$TTL_BLOCKS
      fi
      echo `date '+%D %T'` "Creating vol ser trailer for ${VOL_SER_ALPHA}${CURR_TAPE}" \
            >> $SRC_DIR/${FILE_NAME_BASE}.out
      echo `date '+%D %T'` "HEY! CURR_TAPE==>$CURR_TAPE<==, NUM_TAPES==>$NUM_TAPES<==, TAPES_WRITTEN==>$TAPES_WRITTEN<==" \
      echo `date '+%D %T'` "HEY! CURR_BLOCKS==>$CURR_BLOCKS<==, TTL_BLOCKS==>$TTL_BLOCKS<==, TAPES_WRITTEN==>$TAPES_WRITTEN<==" \
            >> $SRC_DIR/${FILE_NAME_BASE}.out
      $DEBUG write_vol_ser.pl ${VOL_SER_ALPHA}${VOL_SER_NUM_FIRST} \
            ${VOL_SER_ALPHA}${CURR_TAPE} $FILE_NAME_BASE \
            $LRECL $BLKSIZE $TAPES_WRITTEN $NUM_TAPES $RECFM $CURR_BLOCKS > \
            $SRC_DIR/${FILE_NAME_BASE}_${VOL_SER_ALPHA}${CURR_TAPE}_2
      echo `date '+%D %T'` "Writing vol ser trailer for ${VOL_SER_ALPHA}${CURR_TAPE}" \
            | tee -a $SRC_DIR/${FILE_NAME_BASE}.out $SRC_DIR/${FILE_NAME_BASE}.err

      time $DEBUG /usr/bin/dd of=/dev/rmt/0n \
            if=$SRC_DIR/${FILE_NAME_BASE}_${VOL_SER_ALPHA}${CURR_TAPE}_2 \
            $TAPE_OPTIONS_LABEL \
            >> $SRC_DIR/${FILE_NAME_BASE}.out 2>> $SRC_DIR/${FILE_NAME_BASE}.err
      time $DEBUG mt status \
            >> $SRC_DIR/${FILE_NAME_BASE}.out 2>> $SRC_DIR/${FILE_NAME_BASE}.err

      if [ $CURR_TAPE -eq $VOL_SER_NUM_LAST ]
      then
            # Last tape.  Rewind or Eject to next tape
            if [ "eject" = "$EOF_OPTION" ]
            then
                  echo `date '+%D %T'` "Switching to next tape " \
                        | tee -a $SRC_DIR/${FILE_NAME_BASE}.out \
                              $SRC_DIR/${FILE_NAME_BASE}.err
                  time $DEBUG mt_swap_tape.sh swap \
                        >> $SRC_DIR/${FILE_NAME_BASE}.out \
                        2>> $SRC_DIR/${FILE_NAME_BASE}.err
            else
                  if [ "rewind" = "$EOF_OPTION" ]
                  then
                        echo `date '+%D %T'` "Rewinding last tape " \
                              | tee -a $SRC_DIR/${FILE_NAME_BASE}.out \
                                    $SRC_DIR/${FILE_NAME_BASE}.err
                        time $DEBUG mt rewind \
                              >> $SRC_DIR/${FILE_NAME_BASE}.out \
                              2>> $SRC_DIR/${FILE_NAME_BASE}.err
                  else
                        echo `date '+%D %T'` "Leaving last tape positioned at EOF" \
                              | tee -a $SRC_DIR/${FILE_NAME_BASE}.out \
                                    $SRC_DIR/${FILE_NAME_BASE}.err
                  fi
            fi
      else
            # !! To Do:  Need to ensure not on last tape slot (10).  If so,
            # !!         pause until tape magazine is reloaded with 10
            # !!         blank tapes.
            # !! Look at /export/home/kabira/common/log/mt_swap_tape_info.txt
            # !! to determine current tape slot in use.
            echo `date '+%D %T'` "Switching to next tape " \
                        | tee -a $SRC_DIR/${FILE_NAME_BASE}.out \
                              $SRC_DIR/${FILE_NAME_BASE}.err
            time $DEBUG mt_swap_tape.sh swap \
                  >> $SRC_DIR/${FILE_NAME_BASE}.out \
                  2>> $SRC_DIR/${FILE_NAME_BASE}.err
      fi

      echo `date '+%D %T'` " - End of Loop" \
            >> $SRC_DIR/${FILE_NAME_BASE}.out
      ps -f -p $$ \
            >> $SRC_DIR/${FILE_NAME_BASE}.out

      # Bump counters
      let CURR_TAPE=$CURR_TAPE+1
      let TAPES_WRITTEN=$TAPES_WRITTEN+1
      # pad with zeros if needed
      LEN_NUM_LAST=`echo $CURR_TAPE | wc -c | tr -d ' '`
      echo HEY LEN_NUM_FIRST=$LEN_NUM_FIRST, LEN_NUM_LAST=$LEN_NUM_LAST
      let LEN_DIFF=$LEN_NUM_FIRST-$LEN_NUM_LAST
      if [ $LEN_DIFF -gt 0 ]
      then
            CURR_TAPE=`echo "0000000" | cut -c1-$LEN_DIFF`$CURR_TAPE
      fi
      echo HEY CURR_TAPE=$CURR_TAPE

done

exit 0



ASKER CERTIFIED SOLUTION
Avatar of yuzh
yuzh
Link to home
membership
This solution is only available to members.
To access this solution, you must be a member of Experts Exchange.
Start Free Trial