#!/usr/bin/perl -w
# "IDFStoPDS_batch.p"
# Run the program IDFStoPDS in a batch mode.  
#   This requires locating data files, discovering 
#   the times covered in each file using "idfs_time" 
#   and then invoking "IDFStoPDS" with the data 
#   file and its appropriate ".I2P" file.  
# 
# The required input is the name of a configuration 
#   file. Within this configuration file are specified 
#   the following [as a minimum]: 
#     - the name of an '.I2P' file 
#           (this file contains information 
#            used by IDFStoPDS to extract data 
#            from the SDDAS archive in IDFS 
#            format and write it to a new 
#            archive in PDS format) 
#     - the name of the IDFS data directory 
#           (this is the source directory for 
#            data to be converted to PDS format) 
#     - the name of the PDS data directory 
#           (this is the destination directory 
#            for data converted to PDS format) 
#  Additionally, the following information may 
#    be specified within a configuration file: 
#     - the name of the SPICE data directory 
#           (SPICE data may be required in order 
#            to fill in currently unknown values 
#            for fields in the PDS files; these 
#            fields will be filled in using a 
#            vendor-supplied program) 
#     - an indication to run in 'verbose mode' 
#          "verbose: 1" to turn verbosity ON
#          "verbose: 0" to turn verbosity OFF
#     - specify which executables to use: 
#          "IDFStoPDS: ${HOME}/mybin/IDFStoPDS.Sun"
#          "idfs_time: {SDDAS_HOME}/test_bin/idfs_time"
#          "vendor: ${SDDAS_HOME}/vendor_bin/vendorExec"
########################################################
# 
# The broad structure of the script is to: 
#   1. Read the config file one line at a time 
#      and act upon that line; should the first 
#      character of a line be '#', then that line 
#      is considered to be a comment; as such, it 
#      will be recorded, in running in verbose 
#      mode, to the log file.  
#   2. Should a line modify an executable or path, 
#      then that file is verified - if it cannot 
#      be verified, the script halts with appropriate 
#      comments written to STDERR and the logfile 
#   3. Anytime a line contains the name of an 'I2P' 
#      file, that file is processed using the executables 
#      already verified; should the line also contain 
#      data directories, then it must contain them 
#      in the following order: 
#      IDFS_dataDirectory PDS_dataDirectory SPICE_dataDirectory 
#      ...if only one directory name is given, that is used 
#         in place of the current IDFS data directory; should 
#         two directory names be given, then the first is taken 
#         as the IDFS data directory and the second is taken 
#         to be the PDS data directory, etc. 
#   4. To assist in configuration clarity when read with an 
#      editor, line continuation may be noted by ending a 
#      line with a backslash, so the following series of lines 
#      is interpreted as one line and results in the execution 
#      of IDFStoPDS on data files in the IDFS data directory 
#      ${SDDAS_DATA}/source_IDFS/, with the resulting PDS files 
#      written to the directory {$PDS_DATA}/new_files/:
#      >ELSSCIL_C_ACC.I2P \
#      >      ${SDDAS_DATA}/source_IDFS/\
#      >      ${PDS_DATA}/new_files/
# 
########################################################
# 
# Given a .I2P file [or files], this script will 
#   locate the appropriate data files and invoke 
#   IDFStoPDS for each file.  For this to be 
#   successful, IDFStoPDS has to be supplied with 
#   arguments (-BTime and -ETime) to override the 
#   time information specified in the .I2P file.  
#   In this way, each PDS file produced contains 
#   data over a time range exactly matching the 
#   corresponding IDFS data file.  
# 
# The default directory path along which data files 
#   are searched for is ${SDDAS_DATA}.  In such cases, 
#   the .I2P file is parsed for a standard SDDAS 
#   data directory pathname (i.e. Project/Satellite/\
#   Experiment/Instrument/) which is appended to 
#   ${SDDAS_DATA} to specify the directories that 
#   are searched for SDDAS data files.  This default 
#   behavior may be over-ridden with the command 
#   line argument '-dataDir directory' where 
#   'directory' specifies a appropriate data 
#   directory; this directory then is searched 
#   for data files.  
# 
# The default directory path for output files will 
#   be the directory from which the program is 
#   invoked.  The user may override this default 
#   behavior by specifying the command line argument 
#   "-destDir directory".  In such cases, all output 
#   files generated will be placed into the specified 
#   directory after their creation by IDFStoPDS (the 
#   files will initially be created in the directory 
#   from which the program is invoked).  
# 
# It should be noted that each .I2P file may relate 
#   to data files located in a unique directory; or 
#   the output files related to a given .I2P file 
#   may be directed to an output directory not 
#   shared by other .I2P files for which the 
#   program is invoked.  To handle this situation 
#   and allow the user to invoke the program only 
#   once, a configuration file may be created in 
#   which the I2P file is listed along with specific 
#   data and destination directories.  For example, 
#   three lines in such a file could be: 
#  "#Comment lines begin with a # - this is a sample:"
#  "ELSSCIL_C_ACC.I2P  /archive/sddasData/  /archive/pdsDataL/day1/"
#  "ELSSCIH_C_ACC.I2P  /archive/sddasData/  /archive/pdsDataH/day1/"
#  In this example, data files pertaining to both .I2P 
#    files are looked for in the directory 
#    "/archive/sddasData/", while the PDS data files 
#    are created in the directory "/archive/pdsDataL/day1/" 
#    for the .I2P file "ELSSCIL_C_ACC.I2P" and in the 
#    directory "/archive/pdsDataH/day1/" for the .I2P 
#    file "ELSSCIH_C_ACC.I2P".  
# 
# JKJ

# The vendor will supply some sort of program to be 
#   used in filling out the .LBL file with the rest 
#   of its information - once the name of this program 
#   is known, assign it to 'vendorProg' and set 
#   'doVendorProg' equal to 1.  (Well, there will 
#   probably be some changes to the invocation, 
#   but without the program - we'll guess!) 
$doVendorProg   = 0;
$vendorProg     = "";
$SPICEdirectory = "";

# What sort of suffixes might the I-files have? 
#   These would be files that work with calls 
#   to "idfs_time", so that for all data files 
#   there is a corresponding I-file.  For examle, 
#   the data file "ELSSCIL20031901858D" belongs 
#   with the I-file "ELSSCIL20030010000V.v3".  
#   Older versions of I-files end with 'I'.  
@iFileSuffixes = ("I", "V.v3");

@ARGV or (&usage($0) and die());

$verbose   =  1;
$idfs_time = "";
$IDFStoPDS = "";
$sddasPath = "";
$destPath  = "";
$autoPath  =  0;

# >>>>>>>> Open the log file, or turn logging off <<<<<<<<<<<<<<<
  $logging = 1;
  $logFile = ".logFile_IDFStoPDS_batch.p";
  $currDir = `pwd`;
  chomp($currDir);
  if(!($currDir =~ /\/$/)){
    $currDir = $currDir."/";
  }
  
  # If we cannot write to the current directory, then 
  #   the output from IDFStoPDS cannot be reviewed and 
  #   validated [most simply, anyway].  
  if(!(-w $currDir)){
    print STDERR "The program needs to be able to write in the";
    print STDERR " current directory\n";
    die();
  }
  
  if(!(open LOG, ">> $logFile")){
    print STDERR "\n\tunable to open logfile \"$logFile\"\n";
    print STDERR "\t...turning logging OFF!\n";
    $logging = 0;
  }else{
    print "Logging notes/errors to file \"$logFile\"\n";
    print LOG `date`;
    print LOG "Invoked as user: ";
    print LOG `whoami`;
    print LOG "Invocation: \"$0 @ARGV\"\n";
    print LOG "Invocation directory: $currDir\n";
  }

# >>>>>>>> Read through each specified configuration file <<<<<<<<<
foreach $file(@ARGV){
  
  if(!(-f $file)){
    print STDERR "Unable to find specified config file: \"$file\"\n";
    print STDERR "\n\t...terminating program \"$0\"\n\n";
    if($logging){
      print LOG "Unable to find specified config file: \"$file\"\n";
      print LOG `date`;
      print LOG "\n\t...terminating program \"$0\"\n\n";
      print LOG "##############################################################\n\n";
    }
    die "";
  }elsif(!(-r $file)){
    print STDERR "Unable to read specified config file: \"$file\"\n";
    print STDERR "\n\t...terminating program \"$0\"\n\n";
    if($logging){
      print LOG "Unable to read specified config file: \"$file\"\n";
      print LOG `date`;
      print LOG "\n\t...terminating program \"$0\"\n\n";
      print LOG "##############################################################\n\n";
    }
    die "";
  }else{
    open CONFIG, $file;
    @configLines = <CONFIG>;
    chomp(@configLines);
    close(CONFIG);
  }
  $parseError = 0;
  
# >>>>>>>> Read through the current config 
#                 file, one line at a time <<<<<<<<<<<<<<<
  $configLine = "";
  $lineCount  = @configLines;
  for($linePtr = 0; $linePtr < $lineCount; $linePtr++){
    $configLine = $configLines[$linePtr];
    # Remove leading whitespace (this leads to 
    #   predictable results from "split"): 
    $configLine =~ s/^\s+//;
    # Concatenate while the line ends in a backslash: 
    while($configLine =~ /\\$/){
      $linePtr++;
      if($linePtr < $lineCount){
        $line = $configLine;
        $line =~ s/\\$/ /;
        $configLine = $configLines[$linePtr];
        # Remove leading whitespace: 
        $configLine =~ s/^\s+//;
        $configLine = $line.$configLine;
      }else{
        print STDERR "Unable to read indicated continuation line\n";
        print STDERR "   -->line #$linePtr in file: \"$file\" ended with a backslash\n";
        print STDERR "\n\t...terminating program \"$0\"\n\n";
        if($logging){
          print LOG "Unable to read indicated continuation line\n";
          print LOG "   -->line #$linePtr in file: \"$file\" ended with a backslash\n";
          print LOG `date`;
          print LOG "\n\t...terminating program \"$0\"\n\n";
          print LOG "##############################################################\n\n";
        }
        die "";
      }
    }
    # Remove trailing whitespace: 
    $configLine =~ s/\s+$//;
    
    # Examine the current configuration line to see 
    #   what action should be taken: 
    @fields = split /\s+/, $configLine;
    $firstField = $fields[0];
    if($firstField =~ /^#/){
      if($logging){
        print LOG "config file comment line: \"$configLine\"\n";
      }
    }elsif(!($firstField =~ /\.I2P$/)){
      # This line does not contain the name of an I2P file, 
      #   so the user must be specifying a path or some 
      #   other configurable quantity.  
      if(@fields == 1){
        # For flexibility, account for the possibility 
        #   of an entry such as: "verbose:0" by 
        #   resplitting the line after replacing 
        #   any [first] colon with a space: 
        $tempLine = $configLine;
        $tempLine =~ s/:/ /;
        @fields   = split /\s+/, $tempLine;
      }
      if(@fields > 1){
        # ignoring case and remove any terminal ':': 
        $firstField = lc($firstField);
        $firstField =~ s/:$//;
        if($firstField eq "verbose"){
          if($fields[1] eq "1" or $fields[1] =~ /^[oO][nN]$/){
            if($logging && $verbose < 1){
              print LOG "   changing to verbose output...\n";
            }
            $verbose = 1;
          }elsif($fields[1] eq "0" or $fields[1] =~ /^[oO][fF][fF]$/){
            if($logging && $verbose > 0){
              print LOG "   turning verbose output off...\n";
            }
            $verbose = 0;
          }else{
            print STDERR "Parsing error in the config file: ";
            print STDERR "bad verbosity indicator \"$fields[1]\"\n";
            if($logging){
              print LOG "Parsing error in the config file: ";
              print LOG "bad verbosity indicator \"$fields[1]\"\n";
            }
            $parseError = 1;
          }
        }elsif($firstField eq "idfs_root"){
          # If only the root directory is specified, 
          #   then the rest of the data path is made 
          #   up from the standard SDDAS data path 
          #   [accomplished by setting autoPath to '1'].  
          $sddasPath = expand_path($fields[1]);
          $autoPath  = 1;
          if(!(-e $sddasPath)){
            print STDERR "\tIDFS root path \"$fields[1]\" is not found\n";
            if($sddasPath ne $fields[1]){
              print STDERR "\tIDFS root path expanded to \"$sddasPath\"\n";
            }
            if($logging){
              print LOG "\tIDFS root path \"$fields[1]\" is not found\n";
              if($sddasPath ne $fields[1]){
                print LOG "\tIDFS root path expanded to \"$sddasPath\"\n";
              }
            }
            $parseError = 1;
          }
        }elsif($firstField eq "idfs_path"){
          $sddasPath = expand_path($fields[1]);
          $autoPath  = 0;
          if(!(-e $sddasPath)){
            print STDERR "\tIDFS path \"$fields[1]\" is not found\n";
            if($sddasPath ne $fields[1]){
              print STDERR "\tIDFS path expanded to \"$sddasPath\"\n";
            }
            if($logging){
              print LOG "\tIDFS path \"$fields[1]\" is not found\n";
              if($sddasPath ne $fields[1]){
                print LOG "\tIDFS path expanded to \"$sddasPath\"\n";
              }
            }
            $parseError = 1;
          }
        }elsif($firstField eq "pds_path"){
          $destPath = expand_path($fields[1]);
          if(!(-e $destPath)){
            print STDERR "\tdestination path \"$fields[1]\" is not found\n";
            if($destPath ne $fields[1]){
              print STDERR "\tdestination path expanded to \"$destPath\"\n";
            }
            if($logging){
              print LOG "\tdestination path \"$fields[1]\" is not found\n";
              if($destPath ne $fields[1]){
                print LOG "\tdestination path expanded to \"$destPath\"\n";
              }
            }
            $parseError = 1;
          }elsif(!(-w $destPath)){
            print STDERR "\tdestination path \"$fields[1]\" is not writeable\n";
            if($destPath ne $fields[1]){
              print STDERR "\tdestination path expanded to \"$destPath\"\n";
            }
            if($logging){
              print LOG "\tdestination path \"$fields[1]\" is not writeable\n";
              if($destPath ne $fields[1]){
                print LOG "\tdestination path expanded to \"$destPath\"\n";
              }
            }
            $parseError = 1;
          }
        }elsif($firstField eq "spice_path"){
          $SPICEdirectory = expand_path($fields[1]);
          if(!(-e $SPICEdirectory)){
            print STDERR "\tSPICE directory \"$fields[1]\" is not found\n";
            if($SPICEdirectory ne $fields[1]){
              print STDERR "\tSPICE directory expanded to \"$SPICEdirectory\"\n";
            }
            if($logging){
              print LOG "\tSPICE directory \"$fields[1]\" is not found\n";
              if($SPICEdirectory ne $fields[1]){
                print LOG "\tSPICE directory expanded to \"$SPICEdirectory\"\n";
              }
            }
            $parseError = 1;
          }
        }elsif($firstField eq "idfstopds"){
          $IDFStoPDS = expand_path($fields[1]);
          if(!(-f $IDFStoPDS)){
            print STDERR "\tIDFStoPDS program \"$fields[1]\" is not found\n";
            if($IDFStoPDS ne $fields[1]){
              print STDERR "\tIDFStoPDS program expanded to \"$IDFStoPDS\"\n";
            }
            if($logging){
              print LOG "\tIDFStoPDS program \"$fields[1]\" is not found\n";
              if($IDFStoPDS ne $fields[1]){
                print LOG "\tIDFStoPDS program expanded to \"$IDFStoPDS\"\n";
              }
            }
            $parseError = 1;
          }elsif(!(-x $IDFStoPDS)){
            print STDERR "\tIDFStoPDS program \"$fields[1]\" is not executable\n";
            if($IDFStoPDS ne $fields[1]){
              print STDERR "\tIDFStoPDS program expanded to \"$IDFStoPDS\"\n";
            }
            if($logging){
              print LOG "\tIDFStoPDS program \"$fields[1]\" is not executable\n";
              if($IDFStoPDS ne $fields[1]){
                print LOG "\tIDFStoPDS program expanded to \"$IDFStoPDS\"\n";
              }
            }
            $parseError = 1;
          }
        }elsif($firstField eq "idfs_time"){
          $idfs_time = expand_path($fields[1]);
          if(!(-f $idfs_time)){
            print STDERR "\tidfs_time program \"$fields[1]\" is not found\n";
            if($idfs_time ne $fields[1]){
              print STDERR "\tidfs_time program expanded to \"$idfs_time\"\n";
            }
            if($logging){
              print LOG "\tidfs_time program \"$fields[1]\" is not found\n";
              if($idfs_time ne $fields[1]){
                print LOG "\tidfs_time program expanded to \"$idfs_time\"\n";
              }
            }
            $parseError = 1;
          }elsif(!(-x $idfs_time)){
            print STDERR "\tidfs_time program \"$fields[1]\" is not executable\n";
            if($idfs_time ne $fields[1]){
              print STDERR "\tidfs_time program expanded to \"$idfs_time\"\n";
            }
            if($logging){
              print LOG "\tidfs_time program \"$fields[1]\" is not executable\n";
              if($idfs_time ne $fields[1]){
                print LOG "\tidfs_time program expanded to \"$idfs_time\"\n";
              }
            }
            $parseError = 1;
          }
        }elsif($firstField eq "vendor"){
          $vendorProg = expand_path($fields[1]);
          if(!(-f $vendorProg)){
            print STDERR "\tvendor program \"$fields[1]\" is not found\n";
            if($vendorProg ne $fields[1]){
              print STDERR "\tvendor program expanded to \"$vendorProg\"\n";
            }
            if($logging){
              print LOG "\tvendor program \"$fields[1]\" is not found\n";
              if($vendorProg ne $fields[1]){
                print LOG "\tvendor program expanded to \"$vendorProg\"\n";
              }
            }
            $parseError = 1;
          }elsif(!(-x $vendorProg)){
            print STDERR "\tvendor program \"$fields[1]\" is not executable\n";
            if($vendorProg ne $fields[1]){
              print STDERR "\tvendor program expanded to \"$vendorProg\"\n";
            }
            if($logging){
              print LOG "\tvendor program \"$fields[1]\" is not executable\n";
              if($vendorProg ne $fields[1]){
                print LOG "\tvendor program expanded to \"$vendorProg\"\n";
              }
            }
            $parseError = 1;
          }
        }else{
          print STDERR "Parsing error in the config file: ";
          print STDERR "unrecognized flag: \"$fields[0]\"\n";
          if($logging){
            print LOG "Parsing error in the config file: ";
            print LOG "unrecognized flag: \"$fields[0]\"\n";
          }
          $parseError = 1;
        }
      }else{
        print STDERR "Parsing error in the config file: no I2P file ";
        print STDERR "and no field:value combination given\n";
        if($logging){
          print LOG "Parsing error in the config file: no I2P file ";
          print LOG "and no field:value combination given\n";
        }
        $parseError = 1;
      }
      if($parseError){
        $currLineNum = $linePtr + 1;
        print STDERR "   -->see line #$currLineNum in file: \"$file\"\n";
        print STDERR "\n\t...terminating program \"$0\"\n\n";
        if($logging){
          print LOG "   -->see line #$currLineNum in file: \"$file\"\n";
          print LOG `date`;
          print LOG "\n\t...terminating program \"$0\"\n\n";
          print LOG "##############################################################\n\n";
        }
        die "";
      }
    }else{
      # This line references an I2P file, so 
      #   go ahead and execute IDFStoPDS after 
      #   any necessary validations...
      $i2pFile = $fields[0];
      if(@fields > 1){
        $sddasPath = $fields[1];
        $autoPath  = 0;
      }
      if(@fields > 2){
        $destPath = $fields[2];
      }
      if(@fields > 3){
        $SPICEdirectory = $fields[2];
      }
      if(@fields > 4){
        $lineNum = $linePtr + 1;
        print STDERR "Parsing error in the config file: \"$file\"";
        print STDERR "\n\tadditional field[s] given on line $lineNum\n";
        print STDERR "I2Pfilename SDDASdirectory PDSdirectory SPICEdirectory _??_\n";
        if($logging){
          print LOG "Parsing error in the config file: \"$file\"";
          print LOG "\n\tadditional field[s] given on line $lineNum\n";
          print LOG "I2Pfilename SDDASdirectory PDSdirectory SPICEdirectory _??_\n";
        }
        $terminate = 1;
      }
      
# >>>>>> check assignment of input/output paths and I2P files <<<<<<<<<<<<
      if($sddasPath eq ""){
        
        $autoPath  = 1;
        $sddasPath = &defaultDATApath($logging, \*LOG);
        
        if($sddasPath eq ""){
          print STDERR "\n   FATAL: Unable to determine SDDAS directory ";
          print STDERR "from environment variables\n\n";
          if($logging){
            print LOG "U\n   FATAL: nable to recognized determine SDDAS ";
            print LOG "directory from environment variables\n\n";
          }
          $terminate = 1;
        }
      }
      if(!($sddasPath =~ /\/$/)){
        $sddasPath = $sddasPath."/";
      }
      
# >>>>>>>>>> Validate the executables to be used <<<<<<<<<<<<<<<<
      if($idfs_time eq ""){
        $idfs_time = "idfs_time";
      }
      if($IDFStoPDS eq ""){
        $IDFStoPDS = "IDFStoPDS";
      }
      if($doVendorProg){
        @execFiles = exec_validate($idfs_time, $IDFStoPDS, $vendorProg);
      }else{
        @execFiles = exec_validate($idfs_time, $IDFStoPDS);
      }
      
      if($execFiles[0] eq "" or $execFiles[1] eq ""){
        print "\n";
        if($execFiles[0] eq ""){
          print STDERR "\n   FATAL: Unable to find an executable version of \"$idfs_time\"\n\n";
          if($logging){
            print LOG "\n   FATAL: Unable to find an executable version of \"$idfs_time\"\n\n";
          }
        }
        if($execFiles[1] eq ""){
          print STDERR "\n   FATAL: Unable to find an executable version of \"$IDFStoPDS\"\n\n";
          if($logging){
            print LOG "\n   FATAL: Unable to find an executable version of \"$IDFStoPDS\"\n\n";
          }
        }
        $terminate = 1;
      }else{
        $idfs_time = $execFiles[0];
        $IDFStoPDS = $execFiles[1];
      }
      if($doVendorProg){
        if($execFiles[4] eq ""){
          print STDERR "\n   FATAL: Unable to find an executable version of \"$vendorProg\"\n\n";
          if($logging){
            print LOG "\n   FATAL: Unable to find an executable version of \"$vendorProg\"\n\n";
          }
          $terminate = 1;
        }else{
          $vendorProg = $execFiles[4];
        }
      }
      
      if($terminate){
        &usage($0);
        print STDERR "\n\t...terminating program \"$0\"\n\n";
        if($logging){
          print LOG `date`;
          print LOG "\n\t...terminating program \"$0\"\n\n";
          print LOG "##############################################################\n\n";
        }
        die "";
      }
      
      # Discover and verify the pathname for data files: 
      $parseInfo = &parse_i2pFile($i2pFile, $logging, \*LOG);
      
      if($parseInfo eq ""){
        print STDERR "Unable to parse virtual name from file: \"$i2pFile\"\n";
        if($logging){
          print LOG "Unable to parse virtual name from file: \"$i2pFile\"\n";
        }
      }else{
        
        @fields = split /\//, $parseInfo;
        $virtualName = $fields[$#fields];
        if($autoPath ne 0){
          $pathSuffix = $parseInfo;
          $pathSuffix =~ s/$virtualName\///;
        }else{
          $pathSuffix = "";
        }
        
        $dataPath = &validateDataPath($sddasPath.$pathSuffix, $logging, \*LOG);
        
#print LOG "  VIRTUAL NAME is \"$virtualName\"\n";
#print LOG "     DATA PATH is \"$dataPath\"\n";
        if($dataPath ne ""){
          @dFiles   = `ls $dataPath | grep $virtualName | grep D\$`;
#print LOG "Before truncating the list, these are the dFiles:\n\"@dFiles\"\n";
          chomp(@dFiles);
          # Well, turns out that there may be multiple 
          #   files which begin with $virtualName when 
          #   that name contains a number [could also 
          #   be other instances, as well, but names 
          #   like IMA_AZ14* and IMA_AZ1* are producing 
          #   matches for IMA_AZ1 which include the 
          #   IMA_AZ14 files].  To avoid this, after 
          #   acquiring a set of H, D, or I files go 
          #   ahead and rescan these files to ensure 
          #   that the virtual name has 11, exactly, 
          #   digits between the virtualName and the 
          #   suffix (D/H/V.v3, etc.)... we have to do 
          #   this in Perl, apparently, because the grep 
          #   available on Solaris does not like to check 
          #   for numbers using "[0-9]"... not sure how 
          #   to immediately and quickly get around that.  
          @temp    = @dFiles;
          $#dFiles = -1;
          foreach $tempName(@temp){
            if($tempName =~ /$virtualName...........D/){
              push @dFiles, $tempName;
            }
          }
#print LOG " AFTER truncating the list, these are the dFiles:\n\"@dFiles\"\n";
          $dFileCnt = @dFiles;
          
          @hFiles   = `ls $dataPath | grep $virtualName | grep H\$`;
          chomp(@hFiles);
          # validate these filenames, as annotated above, 
          #   before the hFiles array is finalized: 
          @temp    = @hFiles;
          $#hFiles = -1;
          foreach $tempName(@temp){
            if($tempName =~ /$virtualName...........H/){
              push @hFiles, $tempName;
            }
          }
#print LOG " AFTER truncating the list, these are the hFiles:\n\"@hFiles\"\n";
          $hFileCnt = @hFiles;
          
          if($hFileCnt != $dFileCnt){
            print STDERR "Unequal number of header/data files: $hFileCnt/$dFileCnt\n";
            if($logging){
              print LOG "Unequal number of header/data files: $hFileCnt/$dFileCnt\n";
            }
            goto NEXT_INPUT_LINE;
          }
          
          if($logging){
            print LOG "I2P file: \"$i2pFile\"\n";
            print LOG "\t       data path: \"$dataPath\"\n";
            print LOG "\tdestination path: \"$destPath\"\n";
          }
          
          foreach $iFileSuffix(@iFileSuffixes){
            @iFiles = `ls $dataPath | grep $virtualName | grep $iFileSuffix\$`;
#print LOG "Before truncating the list, these are the iFiles:\n\"@iFiles\"\n";
            chomp(@iFiles);
#           # validate these filenames, as annotated above, 
#           #   before the dFiles array is finalized: 
            @temp    = @iFiles;
            $#iFiles = -1;
            foreach $tempName(@temp){
#print LOG "virtualName: \"$virtualName\" and suffix \"$iFileSuffix\"\n";
              if($tempName =~ /$virtualName...........$iFileSuffix/){
                push @iFiles, $tempName;
              }
            }
#print LOG " AFTER truncating the list, these are the iFiles:\n\"@iFiles\"\n";
            
            if(@iFiles){
              foreach $iFile(@iFiles){
                if($iFile =~ /$virtualName...........$iFileSuffix/){
                  foreach $dFile(@dFiles){
                    if($dFile =~ /$virtualName...........D/){
                      $command = "cd ".$dataPath."; ".$idfs_time." ".$iFile." ".$dFile;
                      $timeInfo = `$command`;
                      chomp($timeInfo);
                      if($timeInfo eq ""){
                        print STDERR "...failed execution of \"$command\" for ";
                        print STDERR "file: \"$dFile\"\n";
                        if($logging){
                          print LOG "...failed execution of \"$command\" for ";
                          print LOG "file: \"$dFile\"\n";
                        }
                      }else{
                        if($verbose){
                          print STDERR "$dFile: $timeInfo\n";
                        }
                        if($verbose && $logging){
                          print LOG "$dFile: $timeInfo\n";
                        }
                        
                        # Sample time information returned from idfs_time: 
                        # 2003 190 18 58 09 958 68289958 2003 190 20 32 24 299 73944299
                        #{year doy hr mn sc msc nanosec} 
                        # Format needed for input to IDFStoPDS: 
                        # -BTime 2003/190:19:18:58.958 -ETime 2003/190:20:32:24.299
                        $timeInfo =~ s/^\s+//;
                        $timeInfo =~ s/\s+$//;
                        @fields = split /\s+/, $timeInfo;
                        if($timeInfo =~ /a-zA-z/ or @fields != 14){
                          print STDERR "\tInvalid return from \"$idfs_time\": \"$timeInfo\"\n";
                          if($logging){
                            print LOG "\tInvalid return from \"$idfs_time\": \"$timeInfo\"\n";
                            print LOG "\t...bypassing file \"$dFile\"\n";
                          }
                        }else{
                          $Btime = sprintf("%04i", $fields[0]);
                          $Btime = $Btime."/".sprintf("%03i", $fields[1]);
                          $Btime = $Btime.":".sprintf("%02i", $fields[2]);
                          $Btime = $Btime.":".sprintf("%02i", $fields[3]);
                          $Btime = $Btime.":".sprintf("%02i", $fields[4]);
                          $Btime = $Btime.".".sprintf("%03i", $fields[5]);
                          
                          $Etime = sprintf("%04i", $fields[7]);
                          $Etime = $Etime."/".sprintf("%03i", $fields[8]);
                          $Etime = $Etime.":".sprintf("%02i", $fields[9]);
                          $Etime = $Etime.":".sprintf("%02i", $fields[10]);
                          $Etime = $Etime.":".sprintf("%02i", $fields[11]);
                          $Etime = $Etime.".".sprintf("%03i", $fields[12]);
                          
                          $command = $IDFStoPDS." -FName ".$i2pFile;
                          $command =~ s/\.I2P$//;
                          $command = $command." -BTime ".$Btime." -ETime ".$Etime;
                          if($verbose){
                            print STDERR "$command\n";
                          }
                          if($verbose && $logging){
                            print LOG "$command\n";
                          }
                          $results = `$command`;
                          chomp($results);
                          # As long as the string 'has been generated' appears in 
                          #   the output, then we should be able to extract a CSV 
                          #   filename from the output.  Several CSV files, however, 
                          #   may be produced and there may be a failure to report, 
                          #   so the output is scanned for 'has been generated' as 
                          #   an indicator of success, then coarsely scanned for 
                          #   the presence of a failure by confirming the presence 
                          #   of a CSV filename in every 6th field.  Should a CSV 
                          #   filename fail to appear where expected, then the output 
                          #   from the command is recorded to the logfile and echo'd 
                          #   to STDERR.  
                          # All files produced are then moved to the destination 
                          #   directory (unless the vendor has supplied a program 
                          #   to invoke at this point).  
                          if($results =~ /has been generated/){
                            if($verbose){
                              print STDERR "$dFile: $results\n";
                            }
                            if($verbose && $logging){
                              print LOG "$dFile: $results\n";
                            }
                            $results =~ s/^\s+//;
                            $results =~ s/\s+$//;
                            @fields = split /\s+/, $results;
                            $cnt    = @fields;
                            if($cnt != 6 or !($fields[2] =~ /\.CSV$/)){
                              $flagError = 0;
                              for($fld_ptr = 0; $fld_ptr < $cnt - 1; $fld_ptr = $fld_ptr + 6){
                                if(!($fields[$fld_ptr + 2] =~ /\.CSV$/)){
                                  $flagError = 1;
                                }
                              }
                              if($flagError){
                                print STDERR "Unexpected format in output line from \"$IDFStoPDS\"\n";
                                print STDERR "\twhere 6 fields were expected, with the 3rd field \n";
                                print STDERR "\tbeing a filename ($cnt fields were found in line):\n";
                                print STDERR "\t\"$results\"\n";
                                if($logging){
                                  print LOG "Unexpected format in output line from \"$IDFStoPDS\"\n";
                                  print LOG "\twhere 6 fields were expected, with the 3rd field \n";
                                  print LOG "\tbeing a filename ($cnt fields were found in line):\n";
                                  print LOG "\t\"$results\"\n";
                                }
                              }
                            }
                            $file_CSV   = "";
                            $#CSV_files = -1;
                            foreach $field(@fields){
                              if($field =~ /\.CSV$/){
                                $file_CSV = $field;
                                push @CSV_files, $field;
                              }
                            }
                            if($file_CSV eq ""){
                              print STDERR "Could not identify output file in line:\n";
                              print STDERR "\t\"$results\"\n";
                              if($logging){
                                print LOG "Could not identify output file in line:\n";
                                print LOG "\t\"$results\"\n";
                              }
                            }else{
                              foreach $file_CSV(@CSV_files){
                                $file_LBL = $file_CSV;
                                $file_LBL =~ s/\.CSV$/.LBL/;
                                if(!(-e $file_CSV) or !(-e $file_LBL)){
                                  if(!(-e $file_CSV)){
                                    print STDERR "Could not locate expected output file: \"$file_CSV\"\n";
                                    if($logging){
                                      print LOG "Could not locate expected output file: \"$file_CSV\"\n";
                                    }
                                  }
                                  if(!(-e $file_LBL)){
                                    print STDERR "Could not locate expected output file: \"$file_LBL\"\n";
                                    if($logging){
                                      print LOG "Could not locate expected output file: \"$file_LBL\"\n";
                                    }
                                  }
                                }else{
                                  if($doVendorProg){
                                    $command = $vendorProg." ".$file_LBL;
                                    $results = `$command`;
                                    if($results eq ""){
                                    }
                                  }
                                  $command = "mv ".$file_CSV." ".$destPath;
                                  `$command`;
                                  $command = "mv ".$file_LBL." ".$destPath;
                                  `$command`;
                                }
                              }
                            }
                          }else{
                            print STDERR "...failed execution of \"$command\"\n\tfor ";
                            print STDERR "file: \"$dFile\" using \"$i2pFile\"\n";
                            if($logging){
                              print LOG "...failed execution of \"$command\"\n\tfor ";
                              print LOG "file: \"$dFile\" using \"$i2pFile\"\n";
                            }
                          }
                        }
                      }
                    }
                  }
                }else{
                  print STDERR "...parse failure with I-file named \"$iFile\"\n";
                  print STDERR "   expecting $virtualName+several-digits+$iFileSuffix\n";
                  if($logging){
                    print LOG "...parse failure with I-file named \"$iFile\"\n";
                    print LOG "   expecting $virtualName+several-digits+$iFileSuffix\n";
                  }
                }
              }
            }elsif($logging){
              print LOG "No I-files with suffix \"$iFileSuffix\" located ";
              print LOG "along the path: \"$dataPath\"\n";
            }
          }
        }else{
          print STDERR "...invalid path, no processing attempted for file \"$i2pFile\"\n";
          if($logging){
            print LOG "...invalid path, no processing attempted for file \"$i2pFile\"\n";
          }
        }
      }
    }
    
    NEXT_INPUT_LINE:
    # Reset the config line before continuing to read the file... 
    $configLine = "";
  }
}
  
# Finished... 
if($logging){
  print LOG `date`;
  print LOG "##############################################################\n\n";
}

##############################################################
sub usage{
  $progName = $_[0];
  
  print STDERR "\n$progName configFilename[s]\n";
  print STDERR "\n";
}

##############################################################
# Expand a pathname - needed in case the user incorporates 
#   environment variables in the path/filenames.  
sub expand_path{
  my ($in_name, $prefix, $suffix, $var, $out_name, $temp);
  
  $in_name = $_[0];
  $temp = $in_name;
  if($temp =~ /(.*)\{(.+)\}(.*)/){
    $prefix = $1;
    $var    = $2;
    $suffix = $3;
    if($ENV{$var}){
      $prefix   =~ s/\$$//;
      $out_name = $prefix.$ENV{$var}."/".$suffix;
    }else{
      print STDERR "Not found: \"$var\"\n";
      $out_name = "Unable to expand environment variable \"$var\"\n";
      $out_name = $out_name."found in \"$in_name\"\n";
    }
  }else{
    $out_name = $in_name;
  }
  
  $out_name =~ s/\/+/\//g;
  return $out_name;
}

##############################################################
# Check out the required programs and make sure they 
#   may be found.  If not found along the environment 
#   PATH, then look for them along the SDDAS_HOME path.  
# For every executable that is found, fill in the 
#   array "replies" with the fully-qualified name of 
#   the program.  For every executable that is not 
#   found, return an empty string.  The order the 
#   programs are listed when the routine is called 
#   determines their order in the returned array.  
sub exec_validate{
  my ($i, $prog, @replies, $limit);
  
  @replies = @_;
  
  $limit = @_;
  for($i = 0; $i < $limit; $i++){
    $prog = $_[$i];
    
    $result = `which $prog`;
    if($result eq ""){
      $prog = $ENV{"SDDAS_HOME"}."/bin/".$prog;
      $result = `which $prog`;
      if($result eq ""){
        $replies[$i] = "";
      }else{
        chomp($result);
        $replies[$i] = $result;
      }
    }else{
      chomp($result);
      $replies[$i] = $result;
    }
  }
  
  return @replies;
}

##############################################################
sub parse_i2pFile(){
  my ($file, $path, $start, $doLog, $log);
  
  $file  = $_[0];
  $doLog = $_[1];
  $log   = $_[2];
  $path  = "";
  
  $start = 0;
  $cntr  = 0;
  if(open INPUT, "$file"){
    @contents = <INPUT>;
    foreach $line(@contents){
      $line =~ s/^\s+//;
      $line =~ s/\s+$//;
      if($line eq "END"){
        if($start gt 0){
          goto TERM;
        }
      }
      if($start gt 0){
        @fields = split /\s+/, $line;
        if(@fields == 3){
          $path = $path.$fields[1]."/";
          $cntr = $cntr + 1;
        }
      }
      if($line eq "IdfsSource"){
        $start = 1;
      }
    }
  }
  TERM:
  
  if($cntr != 5){
    print STDERR "\nPathname parsed from file \"$file\" is abnormal\n";
    print STDERR "    where 5 components were expected, there are $cntr\n";
    if($doLog){
      print $log "\nPathname parsed from file \"$file\" is abnormal\n";
      print $log "    where 5 components were expected, there are $cntr\n";
    }
  }
  
  return $path;
}

##############################################################
sub validateDataPath(){
  my ($path, $doLog, $log);
  
  $path  = $_[0];
  $doLog = $_[1];
  $log   = $_[2];
  
  if(-d $path){
    if(!(-r $path)){
      print STDERR "Unable to read from specified ";
      print STDERR "directory for SDDAS data files: \"$path\"\n";
      if($doLog){
        print $log "Unable to read from specified ";
        print $log "directory for SDDAS data files: \"$path\"\n";
      }
      $path = "";
    }
  }else{
    print STDERR "Unable to locate specified ";
    print STDERR "directory for SDDAS data files: \"$path\"\n";
    if($doLog){
      print $log "Unable to locate specified ";
      print $log "directory for SDDAS data files: \"$path\"\n";
    }
    $path = "";
  }
  
  return $path;
}

##############################################################
sub validateDestPath(){
  my ($path, $doLog, $log);
  
  $path  = $_[0];
  $doLog = $_[1];
  $log   = $_[2];
  
  if(-d $path){
    if(!(-w $path)){
      print STDERR "Unable to write to specified ";
      print STDERR "directory for output files: \"$path\"\n";
      if($doLog){
        print $log "Unable to write to specified ";
        print $log "directory for output files: \"$path\"\n";
      }
      $path = "";
    }
  }else{
    print STDERR "Unable to locate specified ";
    print STDERR "directory for output files: \"$path\"\n";
    if($doLog){
      print $log "Unable to locate specified ";
      print $log "directory for output files: \"$path\"\n";
    }
    $path = "";
  }
  
  return $path;
}
##############################################################
sub defaultDATApath(){
  my ($path, $sddasHome, $doLog, $log);
  
  $path  = $ENV{"SDDAS_DATA"};
  $doLog = $_[0];
  $log   = $_[1];
  
  if($path eq ""){
    print STDERR "SDDAS_DATA should be set! trying \${SDDAS_HOME}/data/\n";
    if($doLog){
      print $log "SDDAS_DATA should be set! trying \${SDDAS_HOME}/data/\n";
    }
    $sddasHome = $ENV{"SDDAS_HOME"};
    if($sddasHome ne ""){
      $path = $sddasHome."/data/";
    }
  }
  
  if($path eq ""){
    print STDERR "SDDAS_DATA must be set! or invoke with \"-data_path\"\n";
    if($doLog){
      print $log "SDDAS_DATA must be set! or invoke with \"-data_path\"\n";
    }
  }else{
    if(-d $path){
      if(-r $path){
        if($doLog){
          print $log "SDDAS_DATA path: \"$path\"\n";
        }
      }else{
        print STDERR "Unable to read from specified ";
        print STDERR "directory for SDDAS data files: \"$path\"\n";
        if($doLog){
          print $log "Unable to read from specified ";
          print $log "directory for SDDAS data files: \"$path\"\n";
        }
        $path = "";
      }
    }else{
      print STDERR "Unable to locate specified ";
      print STDERR "directory for SDDAS data files: \"$path\"\n";
      if($doLog){
        print $log "Unable to locate specified ";
        print $log "directory for SDDAS data files: \"$path\"\n";
      }
      $path = "";
    }
  }
  
  return $path;
}

##############################################################

