package cal_pipe2;

use flat_ir2;
use flat_vis2;
use dark_vis;
use specific_energy;
use solar_remove;
use isis_geo;
use cal_occultation;
use cal_point;

require Exporter;

@ISA      = qw(Exporter);
@EXPORT   = qw(cal_pipe2);

use strict;

sub cal_pipe2 {

  # Get arguments passed from vims_cal_pipe.pl
  my ($thedir, $descend, %cmds) = @_;

  my $file;
  opendir DATA, $thedir or die "Can't open $thedir: $!\n";
  my @filesWithPath = map "$thedir/$_", grep !/^\./, readdir(DATA);
  closedir DATA;

  my ($ir_state, $vis_state, $ir_power_state, $vis_power_state, $line);
  my ($core_item_type, $power_state_flag, $smpl, $bnd, $ln, $core_name);
  my ($instrument_mode_id, $file2, $path, $file2_nopath, $fileCM, $temp);
  my ($file_nopath, $ff, $cmd, $sat, %head_vals, $ir_exp, $vis_exp);
  my ($start_sample, $start_line, $sampling_mode, $ir_sampling_mode);
  my ($vis_sampling_mode, $key, $background_sampling_mode);
  my ($vis_back_samp_mode, $ir_back_samp_mode, $gain_mode);
  my ($vis_gain_mode, $ir_gain_mode, $dsp1, $dsp2, $dsp3, $dsp4);
  my ($doperf, $doflat, $do_specific_energy, $garbage, $spkfil);
  my ($vissm, $vres, $vexp, $dovisdark, $fib, $sef, $ssf);
  my ($compressor_id, $compid, $target);

  my $idldir = $ENV{IDL_DIR};

  # The program must determine whether or not the chosen calibration steps
  # are compatible.

  # Do this for each file in the given directory.
FILE:  foreach $file (@filesWithPath) {
    print "file = $file\n";

    if (-d $file) {
      # If one of the files is a directory and 'descend' is 'yes' then
      # recursively descend the directory tree.
      print("$file is a directory\n");
      if ($descend eq "yes") {
	cal_pipe2($file, $descend, %cmds);
      }
    } else {
      # Otherwise, examine the file to determine what type of calibration
      # to perform.
      print("\n\n");
      print("$file "); 

      # If it isn't a cube, skip.
      # If first 4 characters aren't "CCSD".
      open(DAT,"<$file");
      $line=<DAT>;
      if ($line !~ "CCSD.*") {
        print("not a valid cube file, skipping...\n");
        next FILE;
      }
      print "is a cube!\n";

      # Peek around in the header a smidge.
      # Put various keyword values pairs in a hash.
      %head_vals = ();

      while ($line !~ 'END.*' || $line =~ 'END_.*') {
        # Get CORE_ITEM_TYPE.
        if ($line=~ "\s*CORE_ITEM_TYPE.*") {
          ($core_item_type)=$line=~/\s*\w+\s*=\s*(\w+)/;
          print ("CORE_ITEM_TYPE = $core_item_type\n");
          $head_vals{'core_item_type'} = $core_item_type;
        }

        # Get CORE_ITEMS.
        if ($line=~ "\s*CORE_ITEMS.*") {
          ($smpl, $bnd, $ln) = $line=~
            /\s*\w+\s*=\s*\((\d+)\,\s*(\d+)\,\s*(\d+)\s*\)/;
          print ("CORE_ITEMS = $smpl,$bnd,$ln\n");
	  $head_vals{'samples'} = $smpl;
	  $head_vals{'lines'} = $ln;
	  $head_vals{'bands'} = $bnd;
        }

        # Get EXPOSURE_DURATION.
        if ($line=~ "\s*EXPOSURE_DURATION.*") {
          ($ir_exp, $vis_exp) = $line=~
            /\s*\w+\s*=\s*\(([\.\d]+)\,\s*([\.\d-]+)\s*\)/;
          print ("EXPOSURE_DURATION = $ir_exp,$vis_exp\n");
	  $head_vals{'ir_exp'} = $ir_exp;
	  $head_vals{'vis_exp'} = $vis_exp;
        }

        # Check the INSTRUMENT_MODE_ID, this will effect branch points in
	# the pipeline.
        if ($line=~ "\s*INSTRUMENT_MODE_ID.*") {
          ($instrument_mode_id) = $line=~ /\s*\w+\s*=\s*"(\w+)"/;
          print ("INSTRUMENT_MODE_ID = $instrument_mode_id\n");
          $head_vals{'instrument_mode_id'} = $instrument_mode_id;
        }

        # Check the TARGET, this will effect the I/F calculation as
	# we will use this info to determine distance from the sun.
        if ($line=~ "\s*TARGET_NAME.*") {
          ($target) = $line=~ /\s*\w+\s*=\s*"(\w+)"/;
          print ("TARGET = $target\n");
          $head_vals{'target'} = $target;
        }

        # Check for compressor off, this means background also off.
	# COMPRESSOR_ID should be '1'
        if ($line=~ "\s*COMPRESSOR_ID.*") {
          ($compressor_id) = $line=~ /\s*\w+\s*=\s*(.*)/;
          print ("COMPRESSOR_ID = $compressor_id\n");
          $head_vals{'compressor_id'} = $compressor_id;
        }

	# Check to see if this is a vis-only or ir-only cube.
        if ($line=~ "\s*POWER_STATE_FLAG.*") {
          ($power_state_flag)=$line=~/\s*\w+\s*=\s*\(([\s",\w]+)\)\s*/;
          print ("POWER_STATE_FLAG = $power_state_flag\n");
          ($ir_power_state, $vis_power_state) = $power_state_flag=~
                                                /\s*"(\w+)",\s*"(\w+)"\s*/;
          $head_vals{'ir_power_state'} = $ir_power_state;
          $head_vals{'vis_power_state'} = $vis_power_state;
        }

        # Get X_OFFSET.
        if ($line=~ "\s*X_OFFSET.*") {
          ($start_sample) = $line=~
            /\s*\w+\s*=\s*(\d+)\s*/;
          print ("START_SAMPLE = $start_sample\n");
	  $head_vals{'start_sample'} = $start_sample;
        }

        # Get Z_OFFSET.
        if ($line=~ "\s*Z_OFFSET.*") {
          ($start_line) = $line=~
            /\s*\w+\s*=\s*(\d+)\s*/;
          print ("START_LINE = $start_line\n");
	  $head_vals{'start_line'} = $start_line;
        }

        # Get SPECTRAL_EDITING_FLAG.
        if ($line=~ "\s*SPECTRAL_EDITING_FLAG.*") {
          ($sef) = $line=~
            /\s*\w+\s*=\s*"(\w+)"\s*/;
          print ("SPECTRAL_EDITING_FLAG = $sef\n");
	  $head_vals{'sef'} = $sef;
        }

        # Get SPECTRAL_SUMMING_FLAG.
        if ($line=~ "\s*SPECTRAL_SUMMING_FLAG.*") {
          ($ssf) = $line=~
            /\s*\w+\s*=\s*"(\w+)"\s*/;
          print ("SPECTRAL_SUMMING_FLAG = $ssf\n");
	  $head_vals{'ssf'} = $ssf;
        }

	# Get the BACKGROUND_SAMPLING_MODE_ID, check for on/off modes.
        if ($line=~ "\s*BACKGROUND_SAMPLING_MODE_ID.*") {
          ($background_sampling_mode)=$line=~/\s*\w+\s*=\s*\(([\s",\w\/]+)\)/;
          print ("BACKGROUND_SAMPLING_MODE = $background_sampling_mode\n");
          ($ir_back_samp_mode, $vis_back_samp_mode)= $background_sampling_mode=~
                                                /\s*"(\w+)",\s*"([\w\/]+)"\s*/;
	  # Some files only have one value for BACKGROUND_SAMPLING_MODE.
	  # Duplicate this into both IR and VIS.
	  if ($vis_back_samp_mode eq "") {
	    ($background_sampling_mode)=$line=~/\s*\w+\s*=\s*([",\w]+)/;
	    print ("BACKGROUND_SAMPLING_MODE = $background_sampling_mode\n");
            ($ir_back_samp_mode) = $background_sampling_mode=~ /\s*"(\w+)"\s*/;
	    $vis_back_samp_mode = $ir_back_samp_mode;
	  }
          $head_vals{'ir_back_samp_mode'} = $ir_back_samp_mode;
          $head_vals{'vis_back_samp_mode'} = $vis_back_samp_mode;
        }

	# Get the SAMPLING_MODE_ID, check for hi-res modes.
        if ($line=~ "\s*SAMPLING_MODE_ID.*" && !($line=~ "\s*BACKGROUND.*")) {
          ($sampling_mode)=$line=~/\s*\w+\s*=\s*\(([-\s",\w\/]+)\)/;
          print ("SAMPLING_MODE = $sampling_mode\n");
          ($ir_sampling_mode, $vis_sampling_mode) = $sampling_mode=~
                                         /\s*"([-\w\/]+)",\s*"([-\w\/]+)"\s*/;
          $head_vals{'ir_sampling_mode'} = $ir_sampling_mode;
          $head_vals{'vis_sampling_mode'} = $vis_sampling_mode;
        }

	# Get the GAIN_MODE_ID, check for high gain.
        if ($line=~ "\s*GAIN_MODE_ID.*") {
          ($gain_mode)=$line=~/\s*\w+\s*=\s*\(([\s",\w\/]+)\)/;
          print ("GAIN_MODE = $gain_mode\n");
          ($ir_gain_mode, $vis_gain_mode) = $gain_mode=~
                                                /\s*"(\w+)",\s*"([\w\/]+)"\s*/;
          $head_vals{'ir_gain_mode'} = $ir_gain_mode;
          $head_vals{'vis_gain_mode'} = $vis_gain_mode;
        }

        $line=<DAT>;
      }

#      # Print the header hash for debugging.
#      print ("\n\n head_vals \n\n");
#      foreach $key (keys %head_vals) {
#        print ("$key = $head_vals{$key}\n");
#      }

      # The state flags indicate which instruments are turned on.
      # We assume that at least one instrument is on since we have a cube!
      my $ir_state = 1;
      my $vis_state = 1;
      if ($head_vals{ir_power_state} eq "OFF") { $ir_state = 0; }
      if ($head_vals{vis_power_state} eq "OFF") { $vis_state = 0; }

      if ($ir_state == 0 and $vis_state == 0) {
        # No data, both instruments are off!
        print ("Both POWER_STATE_FLAGs are OFF(!), skipping\n");
        next FILE;
      }

      close (DAT);

      # If CORE_ITEM_TYPE is PC_REAL, skip this cube.
      # The cube has probably already been calibrated.
      if ($head_vals{core_item_type} eq 'PC_REAL') {
        print ("Cube CORE_ITEM_TYPE is PC_REAL, skipping\n");
        next FILE;
      }

      # If either of the background sampleing modes is NOBACK, skip this cube.
      if (($head_vals{'ir_back_samp_mode'} eq 'NOBACK') or ($head_vals{'vis_back_samp_mode'} eq 'NOBACK')) {
        print ("Cube background sampling is NOBACK, skipping\n");
        next FILE;
      }

      # If INSTRUMENT_MODE_ID is CAL_BACKGROUND or CAL_SPECTRAL skip
      # this cube.
      if (($head_vals{instrument_mode_id} eq 'CAL_BACKGROUND') or ($head_vals{instrument_mode_id} eq 'CAL_SPECTRAL')) {
        print ("Cube INSTRUMENT_MODE_ID is ");
	print ("$head_vals{instrument_mode_id}");
	print (", skipping\n");
        next FILE;
      }

      # If INSTRUMENT_MODE_ID is OCCULTATION, call cal_occultation.pm on
      # this cube.
      if ($head_vals{instrument_mode_id} eq 'OCCULTATION') {
        print ("Cube INSTRUMENT_MODE_ID is ");
	print ("$head_vals{instrument_mode_id}");
	print (", calling cal_occultation\n");
	cal_occultation($file,\%head_vals,\%cmds);
        next FILE;
      }

      # If INSTRUMENT_MODE_ID is POINT, call cal_point on this cube.
      if ($head_vals{instrument_mode_id} eq 'POINT') {
        print ("Cube INSTRUMENT_MODE_ID is ");
	print ("$head_vals{instrument_mode_id}");
	print (", calling cal_point\n");
	cal_point($file,\%head_vals,\%cmds);
        next FILE;
      }

      # Image mode, not summed cube.

      # Find the core cube name.
      ($path,$file_nopath) = $file =~ /(.*\/)(.*)/;
      ($core_name) = $file =~ /.*\/[A-Z,a-z]*([_\d]*).*/;
      print("cube core_name = $core_name\n");

      # Run 'cube_prep' on the cube.
      $file2 = $path . $core_name;
      ($path,$file2_nopath) = $file2 =~ /(.*\/)(.*)/;
      $fileCM = $path . "CM_" . $file2_nopath . ".cub";

      # cube_prep adds calibration keyword/values pairs to the header,
      # looks up and translates the back plane data, and swaps bytes if
      # needed.
      if ($head_vals{core_item_type} eq "SUN_INTEGER") {
        # Swap bytes.
	$cmd = "$ENV{caldir}/software/bin/cube_prep $file $fileCM s\n";
	print ("$cmd");
	system ("$cmd");
      } elsif ($head_vals{core_item_type} eq "PC_INTEGER") {
        # Don't swap bytes.
	$cmd = "$ENV{caldir}/software/bin/cube_prep $file $fileCM\n";
	print ("$cmd");
	system ("$cmd");
      } else {
	print("cube core_item_type is strange, skipping...\n");
	next FILE;
      }

      # If the compressor was turned off, we need to subtract
      # the background values from the cube.
      $temp = "tempcb.cub";
      if ($head_vals{ssf} ne "ON") {
	$temp = "tempcb.cub";
	$garbage = "garbage_data";
	$compid  = $head_vals{"compressor_id"};
	print("compid = $compid\n");
	if ($compid == 'N/A') {
	  $cmd = "$ENV{caldir}/software/bin/ir_bg_sub $fileCM $temp\n";
	  print("$cmd");
	  system("$cmd");
	  system("mv $temp $fileCM");
	}
      }

      # Run 'ir_bg' on the 'CM_' file.  This program adds
      # the background back into each pixel, calculates new
      # background values based on a linear fit to the
      # values in each band, and subtracts this from the data.
      # Don't do this if the number of lines is less than 4.
      # Don't do this for spectrally summed cubes.
      if ($head_vals{ssf} ne "ON") {
	if ($head_vals{'lines'} >= 4) {
	  $temp = "tempcb.cub";
	  $garbage = "garbage_data";
	  $fib  = $cmds{"ir_background"};
	  # Also, don't do the background fix if spectral editing is on.
	  if ($fib eq "fix" and $head_vals{sef} eq "OFF") {
	    $cmd = "$ENV{caldir}/software/bin/ir_bg $fileCM $temp\n";
	    print("$cmd");
	    system("$cmd");
	    system("mv $temp $fileCM");

	    # Update the header to show that this step has been run.
	    $cmd = "labels from=$fileCM" .
		   "  list=SUMMARY" .
		   "  dest=FILE" .
		   "  listfile=listfile" .
		   "  grpnam=CALIBRATION" .
		   "  keyword=IR_BGND" .
		   "  keyvalue=FIXED" .
		   "  keytype=STRING";
	    system($cmd);
	    system("rm listfile.lis");
	  }
	}
      }

      # Run 'mark_saturated' on the 'CM_' file.  This program adds
      # the background back into each pixel and checks to see if
      # it is greater than or equal to 4095.  If it is, it marks it
      # as saturated (pixel value set to -32765).
      # Don't do this for spectrally summed cubes.
      if ($head_vals{ssf} ne "ON") {
	$temp = "tempcb.cub";
	$garbage = "garbage_data";
	$sat  = $cmds{"mark_saturated"};
	if ($sat eq "yes") {
	  $cmd = "$ENV{caldir}/software/bin/mark_saturated $fileCM $temp\n";
	  print("$cmd");
	  system("$cmd");
	  system("mv $temp $fileCM");

	  # Update the header to show that this step has been run.
	  $cmd = "labels from=$fileCM" .
		 "  list=SUMMARY" .
		 "  dest=FILE" .
		 "  listfile=listfile" .
		 "  grpnam=CALIBRATION" .
		 "  keyword=SET_SATURATED" .
		 "  keyvalue=YES" .
		 "  keytype=STRING";
	  system($cmd);
	  system("rm listfile.lis");
	}
      }

      # Construct a command file for "make_dark" containing the command
      # to make_dark (an IDL routine) with the appropriate arguments.
      $vissm = $head_vals{vis_sampling_mode};
      if ($vissm eq "HIGH" || $vissm eq "HI-RES") {
        $vres = "hi";
      } else {
        $vres = "lo";
      }
      $vexp = $head_vals{'vis_exp'};

      system("rm md_cmd");
      open(MD_CMD, "> md_cmd") or die "Can't open md_cmd: $!\n";
      print MD_CMD "restore, \"$ENV{caldir}/software/bin/make_dark.sav\"\n";
      if ($head_vals{ssf} eq "ON") {
        print MD_CMD "make_dark,\"",$vres,"\",",$vexp,",\"tmp_vis_dark.cub\",specsum=\"yes\"\n";
      } else {
        print MD_CMD "make_dark,\"",$vres,"\",",$vexp,",\"tmp_vis_dark.cub\"\n";
      }
      print MD_CMD "exit\n";
      close (MD_CMD);

      print ("idl md_cmd\n");
      system("$idldir/bin/idl md_cmd");

      # Remove the dark from the VIS data
      my $vbsm = $head_vals{'vis_back_samp_mode'};
      my $dovisdark = $cmds{"vis_background"};
      if ($dovisdark eq "yes" and $vbsm = "ZERO_SUB") {
	if ($vis_state) {
	  $temp = "tempcb.cub";
	  print("dark_vis.pl $fileCM $temp head_vals\n");
	  dark_vis ($fileCM, $temp, %head_vals);
  	  system("mv $temp $fileCM");
	}
      }
      system("rm tmp_vis_dark.cub");
      system("rm md_cmd");

      # Transpose the cube to make some things run faster.
      $cmd = "transpose from=$fileCM to=$temp order=BSQ\n";
      print("$cmd");
      system($cmd);
      system("mv $temp $fileCM");

      # Flat field the IR cube then flat field the VIS cube (if wanted).
      my $doirflat = $cmds{"ir_flat_field"};
      if ($doirflat eq "yes") {
	if ($ir_state) {
	  print("flat_ir2 $fileCM $temp head_vals\n");
	  flat_ir2 ($fileCM, $temp, %head_vals);
  	  system("mv $temp $fileCM");
	}
      }

      my $dovisflat = $cmds{"vis_flat_field"};
      if ($dovisflat eq "yes") {
	if ($vis_state) {
	  print("flat_vis2.pl $fileCM $temp head_vals\n");
	  flat_vis2 ($fileCM, $temp, %head_vals);
  	  system("mv $temp $fileCM");
	}
      }

      # This step removes the performance model, divides by the exposure time,
      # and then converts the resulting electrons/second to ergs/sec/spectel.
      my $to_specific_energy = $cmds{"to_specific_energy"};
      if ($to_specific_energy eq "yes") {
	print("specific_energy $fileCM $temp head_vals\n");
	specific_energy($fileCM, $temp, %head_vals);
  	system("mv $temp $fileCM");

        # Change CORE_NAME to SPECIFIC_ENERGY and CORE_UNIT to ERGS_PER_SECOND
	$cmd = "labels from=$fileCM" .
	       " list=NONE" .
	       " dest=FILE" .
	       " listfile=listfile" .
	       " objtyp=QUBE" .
	       " keyword=CORE_NAME" .
	       " keyvalue=SPECIFIC_ENERGY" .
	       " keytype=LITERAL";
	print("$cmd\n");
	system($cmd);
	system("rm listfile.lis");

	$cmd = "labels" .
	       " from=$fileCM" .
	       " list=NONE" .
	       " dest=FILE" .
	       " listfile=listfile" .
	       " objtyp=QUBE" .
	       " keyword=CORE_UNIT" .
	       " keyvalue=ERGS_PER_SECOND" .
	       " keytype=LITERAL";
	print("$cmd\n");
	system($cmd);
	system("rm listfile.lis");
      }


      # Converting to I/F involves dividing by the solar spectrum at
      # the radius from the sun where the data was taken.  This involves
      # using the solar spectrum at one AU and the distance in AU at the
      # observation point.
      my $to_I_over_F = $cmds{"to_I_over_F"};
      my $times_pi = $cmds{"times_pi"};
      if ($to_I_over_F eq "yes" and $to_specific_energy eq "yes") {
	print("solar_remove.pl $fileCM $temp $times_pi head_vals\n");
	solar_remove($fileCM, $temp, $times_pi, %head_vals);
	system("mv $temp $fileCM");

        # Change CORE_NAME to I_OVER_F and CORE_UNIT to DIMENSIONLESS
	$cmd = "labels from=$fileCM" .
	       " list=NONE" .
	       " dest=FILE" .
	       " listfile=listfile" .
	       " objtyp=QUBE" .
	       " keyword=CORE_NAME" .
	       " keyvalue=I_OVER_F" .
	       " keytype=LITERAL";
	print("$cmd\n");
	system($cmd);
	system("rm listfile.lis");

	$cmd = "labels" .
	       " from=$fileCM" .
	       " list=NONE" .
	       " dest=FILE" .
	       " listfile=listfile" .
	       " objtyp=QUBE" .
	       " keyword=CORE_UNIT" .
	       " keyvalue=DIMENSIONLESS" .
	       " keytype=LITERAL";
	print("$cmd\n");
	system($cmd);
	system("rm listfile.lis");
      }

      system("rm $fileCM" . ".tmp\n");

      my $splitcubes    = $cmds{"splitcubes"};
      my $backplanes    = $cmds{"backplanes"};
      my $keepcomposite = $cmds{"keepcomposite"};

      # Transpose the cube back to BIL.
      $cmd = "transpose from=$fileCM to=$temp order=BIL\n";
      print("$cmd");
      system($cmd);
      system("mv $temp $fileCM");

      if ($splitcubes eq "yes" or $backplanes eq "yes") {
	if ($splitcubes eq "yes" and $backplanes eq "no") {
	  isis_geo($fileCM, 1, 0, 1);
	} elsif ($backplanes eq "yes" and $splitcubes eq "no") {
	  isis_geo($fileCM, 0, 1, 1);
	} else {
	  isis_geo($fileCM, 1, 1, 1);
	}
      }

      # Transpose the cube back to BIL.
      $cmd = "transpose from=$fileCM to=$temp order=BIL\n";
      print("$cmd");
      system($cmd);
      system("mv $temp $fileCM");

      if ($keepcomposite ne "yes") {
	system("rm $fileCM");
      }

    }   # end of if-then-else directories vs. files.
  }   # end of foreach on files.
}   # end of subroutine.