;+
; NAME: batch_rivm
;
; DESCRIPTION:
; 	Runs RIVM in a loop to reduce all the magnetograms in a day.
;
; CALLING SEQUENCE:
;	batch_rivm, cdir, wdir
;
; INPUT PARAMETERS:
;	cdir = directory that contains the raw calibration files
;	wdir  directory that contains the raw data files
;
; OPTIONAL INPUT KEYWORDS:
;	save_binarr - If set, save traditional IMG0*_unb/_bn binary array file
;	nodelete - If set, keep the intermediate files.  Default behavior,
;		whereas earlier versions would save by default.  2012.04.24
;	delete - Retained for backward compatibility; nodelete keyword
;		takes precedence.
;	calib - If set, use the CAL and FLAT files that are already
;		processed (*inv files).  Default processes the CAL
;		and FLAT files.
;	noscan - If set, do not run quality scan on the data files.
;		Default runs IVM_SCAN and IVM_DATA_PATCH to correct bad images.
;	AND all the keywords used by RIVM.
;
; RESTRICTIONS:
;	The best way to use this procedure is to make a copy of
;	it and edit for the details of the call to RIVM.
;
; HISTORY:
;	Written  1998  Barry LaBonte, one a long line of attempts.
;	Add keyword inheritance, better preprocessing Sept 2, 1999 BJL
;	Modify for daily flat processing  June 22, 2001  BJL
;	KDL: warning, does not work when combined with 'jump' keyword.
;	BJL Oct 3, 2002  Replace SPAWN with FILE_DELETE
;	ELW 2011-2012 - Rework to run with updated rivm v5.0
;	ELW 20140325 - 'err' flag passed through from rivmcal, and checked
;-

pro batch_rivm, cdir, wdir, delete=delete, nodelete=nodelete, calib=calib, $
	scat=scat, noscan=noscan, _EXTRA=ex, unblur=unblur, nounblur=nounblur, $
	sodium=sodium, save_binarr=save_binarr, binstokes=binstokes, $
	err=err, verbose=verbose

; Avoid bulky .mycshrc by using Bourne shell for system calls.   ELW-20091223
setenv,'SHELL=/bin/sh'

err=0	; Start with a clean slate

; Do not keep intermediate files by default
keep_int = keyword_set(nodelete)

; Get the exact directory names to enable exact comparisons
CD, cdir, CURRENT=here
CD, wdir, CURRENT=caldir
CD, here, CURRENT=datdir

; ELW20121228 - unblur by default, keep unblur keyword for backward compatibility
unblur = keyword_set(unblur) || ~keyword_set(nounblur)

; Calibrate or not?
redc = 1
redf = 1
IF( ~ KEYWORD_SET(calib) ) THEN BEGIN
	CD, caldir
	dfiles = FINDFILE('ID*', count=ndfiles)
	ffiles = FINDFILE('IF*', count=nffiles)
	cfiles = FINDFILE('IC*', count=ncfiles)

	if (ndfiles eq 0 || nffiles eq 0 || ncfiles eq 0) then begin
	    print, 'Missing calibration files.  Returning.'
	    err=1
	    return
	endif

	print, 'Calling RIVMCAL with the following arguments:'
	print, ' dfiles=',dfiles
	print, ' ffiles=',ffiles
	print, ' cfiles=',cfiles

	RIVMCAL, dfiles, ffiles, cfiles, dayflat, scat=scat, _EXTRA=ex, KEEP_INT=keep_int, err=err
	if (err gt 0) then begin 	; ELW-20140325
	    print, 'ERROR!  batch_rivm exiting due to problem with rivmcal.'
	    return
	endif
	print, ' RIVMCAL set dayflat to ',dayflat
	CD, here
ENDIF

; Are the cal files here?
IF( caldir NE here ) THEN BEGIN
        ; List the calibration files
	cfiles = FINDFILE( caldir + '/ICA* ' + caldir + '/IDK* ' + $
		caldir + '/IF*' )
	ncfiles = N_ELEMENTS(cfiles)
        ; Link them in locally
	FOR i=0,ncfiles-1 DO SPAWN, 'ln -s ' + cfiles(i) + ' .'
ENDIF
; Now the actual cal files - not the suffixed versions
calfiles = FINDFILE( 'ICA*' )
gcal = WHERE( STRPOS( calfiles, 'inv' ) LT 0 AND $
        STRPOS( calfiles, 'instr' ) LT 0, ngcal )
IF( ngcal GT 0 ) THEN calfiles = calfiles(gcal)

; List the magnetogram files, and exit if none are found
CD, datdir
; (Not a typo - FIND_FILE is different from FINDFILE)
files = FIND_FILE('IMGR*', count=nimgr)
if (nimgr lt 1) then begin
   print, 'No IMGR files found.  Returning.'
   err=-1
   return
endif
ns = strpos(files,'_dpa')
noneedtoscan = where(ns GE 0)
needtoscan = kill_index(indgen(n_elements(files)),noneedtoscan)
; Quality check unless already done
IF( ~ keyword_set(noscan) )  && (needtoscan[0] ge 0) THEN BEGIN
   for iscan = 0,n_elements(needtoscan)-1 do begin
	IVM_SCAN, files[needtoscan[iscan]], flags
	IF( MAX(flags) GE 0 ) THEN BEGIN
		IVM_DATA_PATCH, files[needtoscan[iscan]], flags
        ENDIF
   endfor
   files = FIND_FILE('IMGR*', count=nimgr)
ENDIF
print,files
CD, here
nfiles = nimgr

; Match the Cal files with the data files
calmatch = FILE_TIME_MATCH( files, calfiles, /PREVIOUS )

; Loop over files in working directory

FOR i=0,nfiles-1 DO BEGIN &

   ; Link the next data file in if needed
   IF( datdir NE here ) THEN $
	SPAWN,'ln -s ' + datdir + '/' + files(i) + ' .'
   datf = files(i)

   ; Specify the Cal file for this data file
   calf = calmatch(i)

   print, 'Arguments for the call to rivm:'
   print, '  calf=',calf
   print, '  datf=',datf
   print, '  keep_int=',keep_int
   print, '  redf=',redf
   print, '  redc=',redc
   if (keyword_set(dayflat)) then print, '  dayflat=',dayflat
   if (keyword_set(unblur)) then print, '  unblur=',unblur
   if (keyword_set(scat)) then print, '  scat=',scat
   if (keyword_set(binstokes)) then print, '  binstokes=',binstokes

if ~keyword_set(scat) then scat=1

if (keyword_set(sodium)) then begin
   print, 'USE batch_rivm_na instead. -- ELW20110610.  Continue at your own risk...'
   stop
   destretchonly=1 & unbias=1
endif

; Call RIVM - set some defaults, if not overridden by inheritance
  rval=rivm(calf,datf,$	    ; names of cal, dat file from above
; fitmap, $         	    ; variable to return for goodness of fit image
  keep_int = keep_int, $    ; keep intermediate files 
  redf = redf, $            ; flats have been reduced already
  redc = redc, $            ; calibration matrix has been computed already
  dayflat = dayflat, $      ; daily flat file for interpolation
;  regparm = [bx,by,hx,hy,no_lim],$ ; optional coalignment parameters
;	   		    ; defaults: [128,128,nx/2,ny/2,0]
  scat=scat, $               ; amount (or flag to) remove scattered light.
  			    ; set to [1.0e-4,-1.15] for Na data.
  save_binarr=save_binarr,$ ; Save binary arrays, not just fits
  gif = 1, $	            ; produce a GIF file as output
  unblur = unblur, $        ; flag to unblur (call ivm_desee)
  sodium = sodium, $        ; sodium / chromospheric
  destretchonly=1, $	    ; call ivm_desee only for warp corrections
  unbias = 1, $             ; remove continuum and residual polarization biases
  warp = 1, $	            ; warp geometry cam. images to data camera.
  parasitic = 1, $          ; correct for parasitic light
  spurious = 1, $           ; spurious modulation XXXX ELW-20120306 turned [back?] on tentatively
  detrend = 1, $            ; take out limb darkening when needed
  seeing = 1, $             ; if set, remove worst-seeing images
  verbose=verbose, $	    ; verbosity
  binstokes = binstokes,$   ;flag (if=1) or amount (integer > 1) to bin before inverting
  _EXTRA=ex $    ; Finally pass all the inherited keywords
;  jump = 'blur', $         ; jump to this point in the program
;  d0fil = d0fil, $         ; IDK0yymmdd.hhmmss file, if other than in header
;  f0fil = f0fil, $         ; IFA0yymmdd.hhmmss file, if other than in header
;  npoints = npoints, $     ; number of spectral points, if other than in header
;  nfus = nfus, $           ; number of F-P steps, if other than in header
;  shut = 1, $               ; subtract leaky-shutter light (pre-199X data)
;  old_data = 1, $          ; pre-199X data
;  no_reg = 1, $            ; do not attempt to coalign images
;  fourstep = 1, $          ; use (override header) four-step demodulation
;  calsmooth = 1, $         ; width of smoothing function for cal matricies
;  matched ='mfile',$       ; file with 4x2 coalignment transform matrix
;  fitfunct = 'voigt', $    ;use voigt function (can be 'gaussian')
;  no_geomreflat=no_geomreflat, $ ; set this to avoid new geom de-fringing.
; /no_bgram, $	            ; return header information and do not run xbgram
 ) ; KEEP THIS (closes rivm)

; Delete the link to the data file if needed
   IF( datdir NE here ) THEN FILE_DELETE, files(i)

; Delete the last step Geometry file if intermediate files not wanted
;   IF( KEYWORD_SET(delete) ) THEN FILE_DELETE, 'IMG1*'

ENDFOR

RETURN
END