/* $Id: casu_sky.c,v 1.2 2015/08/07 13:06:54 jim Exp $
 *
 * This file is part of the CASU Pipeline utilities
 * Copyright (C) 2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jim $
 * $Date: 2015/08/07 13:06:54 $
 * $Revision: 1.2 $
 * $Name:  $
 */

/* Includes */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <cpl.h>
#include <math.h>
#include <string.h>

#include "casu_sky.h"
#include "casu_mask.h"
#include "casu_fits.h"
#include "casu_mods.h"
#include "casu_utils.h"
#include "casu_stats.h"
#include "casu_wcsutils.h"

static void casu_sky_mask_grow(cpl_image *dith, float rad);
static float casu_sky_med(casu_fits *sky);
static void casu_sky_joffs(casu_fits **inlist, int nfiles, float **xoffs,
                             float **yoffs);
static void casu_sky_xytoxy(cpl_wcs *inwcs, cpl_wcs *outwcs, 
                            cpl_matrix *inxy, cpl_matrix **outxy);

/**@{*/

/*---------------------------------------------------------------------------*/
/**
    \ingroup casu_modules
    \brief Work out a masked sky estimate from an input jitter series

    \par Name:
        casu_pawsky_mask
    \par Purpose:
        Work out a masked sky estimate from an input jitter series
    \par Description:
        Given an input jitter series stars are iteratively masked and clipped
        to form a final sky estimate.
    \par Language:
        C
    \param inlist
        The input jitter series
    \param invar
        The variance maps for the input jitter series
    \param nfiles
        The number of images in the jitter series
    \param conf
        The input confidence map for the jitter series
    \param mask
        The bad pixel mask derived from the confidence map.
    \param skyout
        The output sky map
    \param skyvar
        The output sky variance map
    \param niter
        The number of masking and clipping iterations to do
    \param ipix
        The minimum object size for masking
    \param thresh
        The threshold in sigma above sky for the object detection alrorithm
    \param nbsize
        The cell size for the background following part of the object 
        detection algorithm
    \param smkern
        The size in pixels of the smoothing kernal for the object detection
        algorithm
    \param status
        An input/output status that is the same as the returned values below.
    \retval CASU_OK
        if everything is ok
    \retval CASU_FATAL
        if something went wrong. An error message will appear.
    \par DRS headers:
        The following values will go into the extension propertylist
        - \b SKYALGO
            The sky estimation algorithm used
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int casu_pawsky_mask(casu_fits **inlist, casu_fits **invar, int nfiles, 
                            casu_fits *conf, casu_mask *mask, 
                            casu_fits **skyout, casu_fits **skyvar, int niter, 
                            int ipix, float thresh, int nbsize, float smkern, 
                            int *status) {
    int i,nx,ny,nbad0,lastone,iter,npts,nbad_init,nbad,dbad;
    int xx1,xx2,yy1,yy2,nfiles2,*confdata;
    const char *fctid = "casu_pawsky_mask";
    cpl_image *dith,*dithc,*skyim,*im,*newim,*outim,*outimv;
    cpl_mask *cplmask,*dithmask,*newmask,*curmask;
    casu_fits **list_ss,*dithf,*dithcf,**inlist2,**invar2;
    unsigned char *inbpm,*rejmask,*rejplus;
    float medsky,*xoffs,*yoffs,mindx,mindy,fbad;
    cpl_propertylist *p,*drs;

    /* Inherited status */

    *skyout = NULL;
    *skyvar = NULL;
    if (*status != CASU_OK) 
        return(*status);

    /* If there aren't any images, then get out of here */

    if (nfiles == 0) {
        cpl_msg_error(fctid,"Sky correction impossible. No science frames");
        FATAL_ERROR
    }

    /* Check to see which files are good */

    inlist2 = cpl_malloc(nfiles*sizeof(casu_fits *));
    if (invar != NULL) 
        invar2 = cpl_malloc(nfiles*sizeof(casu_fits *));
    else
        invar2 = NULL;
    nfiles2 = 0;
    for (i = 0; i < nfiles; i++) {
        if (casu_fits_get_status(inlist[i]) == CASU_OK) {
            inlist2[nfiles2] = inlist[i];
            if (invar != NULL) 
                invar2[nfiles2] = invar[i];
            nfiles2++;
        }
    }

    /* If no good images are present then wrap a dummy image and return it */

    if (nfiles2 == 0) {
        outim = casu_dummy_image(inlist[0]);
        *skyout = casu_fits_wrap(outim,inlist[0],NULL,NULL);
        casu_dummy_property(casu_fits_get_ehu(*skyout));
        casu_fits_set_status(*skyout,CASU_FATAL);
        if (invar != NULL) {
            outimv = casu_dummy_image(inlist[0]);
            *skyvar = casu_fits_wrap(outimv,inlist[0],NULL,NULL);
            casu_dummy_property(casu_fits_get_ehu(*skyvar));
        } else {
            *skyvar = NULL;
        }
        cpl_msg_warning(fctid,"No good images in input list");
        freespace(inlist2);
        freespace(invar2);
        *status = CASU_WARN;
        return(CASU_WARN);
    }

    /* Wrap the input mask into a cpl_mask structure and use it to set the
       internal mask for each of the input images */

    inbpm = casu_mask_get_data(mask);
    nx = casu_mask_get_size_x(mask);
    ny = casu_mask_get_size_y(mask);
    cplmask = cpl_mask_wrap((cpl_size)nx,(cpl_size)ny,(cpl_binary *)inbpm);
    for (i = 0; i < nfiles2; i++)
        cpl_image_reject_from_mask(casu_fits_get_image(inlist2[i]),cplmask);
    cpl_mask_unwrap(cplmask);

    /* Do an initial sky combination */

    casu_imcombine(inlist2,invar2,nfiles2,1,1,0,2.0,"EXPTIME",&outim,&outimv,
                   &rejmask,&rejplus,&drs,status);
    *skyout = casu_fits_wrap(outim,inlist2[0],NULL,NULL);    
    if (invar != NULL) 
        *skyvar = casu_fits_wrap(outimv,invar2[0],NULL,NULL);    
    freespace(rejmask);
    freespace(rejplus);
    freepropertylist(drs);

    /* Clean up any blank bits */

    (void)casu_inpaint(*skyout,nbsize,status);

    /* If this is the only iteration then get out of here now */

    if (niter == 0)
        return(*status);

    /* Work out the median sky, ignoring the bad bits */

    medsky = casu_sky_med(*skyout);

    /* Get jitter offsets from wcs */

    casu_sky_joffs(inlist2,nfiles2,&xoffs,&yoffs);
    mindx = xoffs[0];
    mindy = yoffs[0];
    for (i = 1; i < nfiles2; i++) {
        mindx = min(mindx,xoffs[i]);
        mindy = min(mindy,yoffs[i]);
    }

    /* Do an initial subtraction */

    list_ss = cpl_malloc(nfiles2*sizeof(casu_fits *));
    skyim = casu_fits_get_image(*skyout);
    for (i = 0; i < nfiles2; i++) {
        im = casu_fits_get_image(inlist2[i]);
        newim = cpl_image_subtract_create(im,skyim);
        cpl_image_add_scalar(newim,(double)medsky);
        list_ss[i] = casu_fits_wrap(newim,inlist2[i],NULL,NULL);
    }
    casu_fits_delete(*skyout);
    if (invar != NULL)
        casu_fits_delete(*skyvar);

    /* Set up some counters and begin the iteration loop */

    nbad0 = 0;
    lastone = 0;
    for (iter = 1; iter <= niter; iter++) {
        if (lastone)
            break;
        lastone = (iter == niter);

        /* Dither the sky subtracted input images */

        (void)casu_imdither(list_ss,&conf,nfiles2,1,5.0,5.0,&p,"EXPTIME",&dith,
                            &dithc,status);
        cpl_propertylist_delete(p);

        /* How many dead pixels are there in the dithered image? */

        confdata = cpl_image_get_data_int(dithc);
        npts = (int)cpl_image_get_size_x(dithc)*(int)cpl_image_get_size_y(dithc);
        nbad_init = 0;
        for (i = 0; i < npts; i++)
            if (confdata[i] == 0)
                nbad_init++;

        /* Get rid of the sky subtracted images */

        for (i = 0; i < nfiles2; i++)
            casu_fits_delete(list_ss[i]);

        /* Wrap the result */

        dithf = casu_fits_wrap(dith,inlist2[0],NULL,NULL);
        dithcf = casu_fits_wrap(dithc,conf,NULL,NULL);

        /* Now get an object mask from this dithered image */

        (void)casu_opm(dithf,dithcf,ipix,thresh,nbsize,smkern,2,status);

        /* How many flagged pixels are there, not counting the ones already
           flagged in the confidence map */

        nbad = (int)cpl_image_count_rejected((const cpl_image *)dith) - 
            nbad_init;
        dbad = nbad - nbad0;
        fbad = (iter > 1 ? (float)abs(dbad)/(float)nbad0 : 10000.0);
        cpl_msg_info(fctid,
                     "Iteration: %" CPL_SIZE_FORMAT ", Nreject: %" CPL_SIZE_FORMAT " %" CPL_SIZE_FORMAT,
                     (cpl_size)iter,(cpl_size)nbad,(cpl_size)nbad0);
        if (fbad < 0.025 || dbad < 0) {
            lastone = 1;
        } else {
            nbad0 = nbad;
        }

        /* Allow the mask to grow before the next iteration */

        casu_sky_mask_grow(dith,2);

        /* Right, now decide which part of the mask belongs to each of the 
           input images and update the input mask. */

        dithmask = cpl_image_get_bpm(dith);
        for (i = 0; i < nfiles2; i++) {
            xx1 = (int)(-mindx + xoffs[i] + 1.5);
            xx2 = xx1 + nx - 1;
            yy1 = (int)(-mindy + yoffs[i] + 1.5);
            yy2 = yy1 + ny - 1;
            newmask = cpl_mask_extract(dithmask,(cpl_size)xx1,(cpl_size)yy1,
                                       (cpl_size)xx2,(cpl_size)yy2);
            curmask = cpl_image_get_bpm(casu_fits_get_image(inlist2[i]));
            cpl_mask_or(curmask,newmask);
            cpl_mask_delete(newmask);
        }

        /* Delete the dither and its confidence map */
        
        casu_fits_delete(dithf);
        casu_fits_delete(dithcf);

        /* Do a new sky combination using these new object masks */

        casu_imcombine(inlist2,invar2,nfiles2,1,1,0,2.0,"EXPTIME",&outim,
                       &outimv,&rejmask,&rejplus,&drs,status);
        *skyout = casu_fits_wrap(outim,inlist2[0],NULL,NULL);    
        if (invar != NULL)
            *skyvar = casu_fits_wrap(outimv,invar2[0],NULL,NULL);    
        freespace(rejmask);
        freespace(rejplus);
        freepropertylist(drs);
        p = casu_fits_get_ehu(*skyout);
        cpl_propertylist_update_string(p,"ESO DRS SKYALGO","pawsky_mask");
        cpl_propertylist_set_comment(p,"ESO DRS SKYALGO",
                                     "Sky estimation algorithm");        

        /* Clean up any blank bits */

        (void)casu_inpaint(*skyout,nbsize,status);

        /* Do the sky subtraction again so long as this isn't 
           the last iteration */

        if (! lastone) {
            skyim = casu_fits_get_image(*skyout);
            for (i = 0; i < nfiles2; i++) {
                im = casu_fits_get_image(inlist2[i]);
                newim = cpl_image_subtract_create(im,skyim);
                cpl_image_add_scalar(newim,(double)medsky);
                list_ss[i] = casu_fits_wrap(newim,inlist2[i],NULL,NULL);
            }
            casu_fits_delete(*skyout);
            if (invar != NULL) 
                casu_fits_delete(*skyvar);
        }
    }

    /* Free up some workspace that we've collected along the way */

    freespace(xoffs);
    freespace(yoffs);
    freespace(list_ss);
    freespace(inlist2);
    freespace(invar2);
    return(*status);
}

/*---------------------------------------------------------------------------*/
/**
    \ingroup casu_modules
    \brief Work out a sky estimate from an input jitter series and a
           pre-existing object mask

    \par Name:
        casu_pawsky_mask_pre
    \par Purpose:
        Work out a sky estimate from an input jitter series and a
        pre-existing object mask.
    \par Description:
        Given an input jitter series and an object mask, the mask is
        partitioned to the regions covered by each input image. These
        are used to mask out objects during the stack to find the 
        background map.
    \par Language:
        C
    \param inlist
        The input jitter series
    \param invar
        The input jitter series variance maps
    \param nfiles
        The number of images in the jitter series
    \param mask
        The bad pixel mask derived from the confidence map.
    \param objmask
        The input object mask
    \param nbsize
        The cell size for the background following
    \param skyout
        The output sky map
    \param skyvar
        The output sky variance map
    \param status
        An input/output status that is the same as the returned values below.
    \retval CASU_OK
        if everything is ok
    \retval CASU_FATAL
        if something went wrong. An error message will appear.
    \par DRS headers:
        The following values will go into the extension propertylist
        - \b SKYALGO
            The sky estimation algorithm used
        - \b MASKUSED
            The object mask used
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int casu_pawsky_mask_pre(casu_fits **inlist, casu_fits **invar,
                                int nfiles, casu_mask *mask, 
                                casu_fits *objmask, int nbsize, 
                                casu_fits **skyout, casu_fits **skyvar, 
                                int *status) {
    const char *fctid = "casu_pawsky_mask_pre";
    unsigned char *inbpm,*rejplus,*rejmask;
    int nx,ny,npts,ncoor,i,j,nx_objmask,ny_objmask,k,kk,ix,iy,ind,*opm;
    int nfiles2;
    cpl_mask *cplmask;
    cpl_wcs *wcs_objmask,*wcs;
    cpl_matrix *in_xy,*out_xy,*ddx,*ddx_out;
    cpl_image *im,*outim,*outimv;
    cpl_propertylist *drs,*p;
    double *inxy_data,xx,yy,*ddx_data,shiftx,shifty;
    casu_fits **inlist2,**invar2;

    /* Inherited status */

    *skyout = NULL;
    *skyvar = NULL;
    if (*status != CASU_OK) 
        return(*status);

    /* If there aren't any images, then get out of here */

    if (nfiles == 0) {
        cpl_msg_error(fctid,"Sky correction impossible. No science frames");
        FATAL_ERROR
    }

    /* Check to see which files are good */

    inlist2 = cpl_malloc(nfiles*sizeof(casu_fits *));
    if (invar != NULL)
        invar2 = cpl_malloc(nfiles*sizeof(casu_fits *));
    else
        invar2 = NULL;
    nfiles2 = 0;
    for (i = 0; i < nfiles; i++) {
        if (casu_fits_get_status(inlist[i]) == CASU_OK) {
            inlist2[nfiles2] = inlist[i];
            if (invar != NULL) 
                invar2[nfiles2] = invar[i];
            nfiles2++;
        }
    }

    /* If no good images are present then wrap a dummy image and return it */

    if (nfiles2 == 0) {
        outim = casu_dummy_image(inlist[0]);
        *skyout = casu_fits_wrap(outim,inlist[0],NULL,NULL);
        casu_dummy_property(casu_fits_get_ehu(*skyout));
        casu_fits_set_status(*skyout,CASU_FATAL);
        if (invar != NULL) {
            outimv = casu_dummy_image(inlist[0]);
            *skyvar = casu_fits_wrap(outimv,invar[0],NULL,NULL);
            casu_dummy_property(casu_fits_get_ehu(*skyvar));
        } else {
            *skyvar = NULL;
        }
        cpl_msg_warning(fctid,"No good images in input list");
        freespace(inlist2);
        *status = CASU_WARN;
        return(CASU_WARN);
    }

    /* Wrap the input mask into a cpl_mask structure and use it to set the
       internal mask for each of the input images */

    inbpm = casu_mask_get_data(mask);
    nx = casu_mask_get_size_x(mask);
    ny = casu_mask_get_size_y(mask);
    cplmask = cpl_mask_wrap((cpl_size)nx,(cpl_size)ny,(cpl_binary *)inbpm);
    for (i = 0; i < nfiles2; i++)
        cpl_image_reject_from_mask(casu_fits_get_image(inlist2[i]),cplmask);
    cpl_mask_unwrap(cplmask);

    /* Open the WCS of the object mask and get the pixel mask info */
    
    wcs_objmask = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(objmask));
    opm = cpl_image_get_data(casu_fits_get_image(objmask));
    nx_objmask = cpl_image_get_size_x(casu_fits_get_image(objmask));
    ny_objmask = cpl_image_get_size_y(casu_fits_get_image(objmask));

    /* Get some memory so that we can do a transformation between the coordinate
       systems of the input frames and that of the object mask */
    
    npts = nx*ny;
    ncoor = 2;
    in_xy = cpl_matrix_new((cpl_size)npts,(cpl_size)ncoor);

    /* Initialise the data for the first x,y matrix */

    inxy_data = cpl_matrix_get_data(in_xy);
    k = 0;
    for (j = 0; j < ny; j++) {
        for (i = 0; i < nx; i++) {
            inxy_data[k++] = (double)(i+1);
            inxy_data[k++] = (double)(j+1);
        }
    }

    /* Initialise data for an offset matrix */

    ddx = cpl_matrix_new(1,2);
    ddx_data = cpl_matrix_get_data(ddx);
    ddx_data[0] = 1.0;
    ddx_data[1] = 1.0;

    /* Loop for each input file. Convert the xy matrix to an xy in the object
       mask frame of reference for the first file. Then use the offsets between
       the first image and the subsequent ones to define the location of
       the object pixels in the following images */

    for (k = 0; k < nfiles2; k++) {
        wcs = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(inlist2[k]));
        im = casu_fits_get_image(inlist2[k]);
        if (k == 0) 
            casu_sky_xytoxy(wcs,wcs_objmask,in_xy,&out_xy);
        casu_sky_xytoxy(wcs,wcs_objmask,ddx,&ddx_out);
        cpl_wcs_delete(wcs);
        shiftx =  cpl_matrix_get(out_xy,0,0) - cpl_matrix_get(ddx_out,0,0);
        shifty =  cpl_matrix_get(out_xy,0,1) - cpl_matrix_get(ddx_out,0,1);
        kk = 0;
        for (j = 0; j < ny; j++) {
            for (i = 0; i < nx; i++) {
                xx = cpl_matrix_get(out_xy,kk,0) - shiftx;
                yy = cpl_matrix_get(out_xy,kk,1) - shifty;
                kk++;
                ix = casu_nint(xx);
                iy = casu_nint(yy);
                if (ix < 1 || ix > nx_objmask  || iy < 1 || iy > ny_objmask)
                    continue;
                ind = (iy-1)*nx_objmask + ix - 1;
                if (opm[ind])
                    cpl_image_reject(im,i+1,j+1);
            }
        }
        cpl_matrix_delete(ddx_out);
    }
    cpl_matrix_delete(in_xy);
    cpl_matrix_delete(out_xy);
    cpl_wcs_delete(wcs_objmask);

    /* Do a sky combination */

    casu_imcombine(inlist2,invar2,nfiles2,1,1,0,2.0,"EXPTIME",&outim,&outimv,
                   &rejmask,&rejplus,&drs,status);
    *skyout = casu_fits_wrap(outim,inlist2[0],NULL,NULL);    
    if (invar != NULL)
        *skyvar = casu_fits_wrap(outimv,invar2[0],NULL,NULL);    
    freespace(rejmask);
    freespace(rejplus);
    freepropertylist(drs);
    freespace(inlist2);
    p = casu_fits_get_ehu(*skyout);
    cpl_propertylist_update_string(p,"ESO DRS SKYALGO","pawsky_mask_pre");
    cpl_propertylist_set_comment(p,"ESO DRS SKYALGO",
                                 "Sky estimation algorithm");        
    cpl_propertylist_update_string(p,"ESO DRS MASKUSED",
                                   casu_fits_get_filename(objmask));
    cpl_propertylist_set_comment(p,"ESO DRS MASKUSED",
                                 "Object masked used to make sky");

    /* Clean up any blank bits */

    (void)casu_inpaint(*skyout,nbsize,status);

    /* Get out of here */

    return(*status);
}

/*---------------------------------------------------------------------------*/
/**
    \ingroup casu_modules
    \brief Work out a masked sky estimate from a jitter series with large 
           offsets

    \par Name:
        casu_simplesky_mask
    \par Purpose:
        Work out a masked sky estimate from a jitter series with large offsets
    \par Description:
        Given an input jitter series stars are iteratively masked and clipped
        to form a final sky estimate.
    \par Language:
        C
    \param inlist
        The input jitter series
    \param invar
        The input jitter series variance maps
    \param nfiles
        The number of images in the jitter series
    \param conf
        The input confidence map for the jitter series
    \param mask
        The bad pixel mask derived from the confidence map.
    \param skyout
        The output sky map
    \param skyvar
        The output sky variance map
    \param niter
        The number of masking and clipping iterations to do
    \param ipix
        The minimum object size for masking
    \param thresh
        The threshold in sigma above sky for the object detection alrorithm
    \param nbsize
        The cell size for the background following part of the object 
        detection algorithm
    \param smkern
        The size in pixels of the smoothing kernal for the object detection
        algorithm
    \param status
        An input/output status that is the same as the returned values below.
    \retval CASU_OK
        if everything is ok
    \retval CASU_FATAL
        if something went wrong. An error message will appear.
    \par DRS headers:
        The following values will go into the extension propertylist
        - \b SKYALGO
            The sky estimation algorithm used
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int casu_simplesky_mask(casu_fits **inlist, casu_fits **invar, 
                               int nfiles, casu_fits *conf, casu_mask *mask, 
                               casu_fits **skyout, casu_fits **skyvar, 
                               int niter, int ipix, float thresh, int nbsize, 
                               float smkern, int *status) {
    const char *fctid = "casu_simplesky_mask";
    unsigned char *inbpm,*rejmask,*rejplus;
    int nx,ny,i,ninit,nbad0,iter,nbad,nb,dbad,lastone,nfiles2;
    cpl_mask *cplmask,*newmask,*oldmask;
    cpl_image *outim,*skyim,*im,*outimv;
    cpl_propertylist *drs,*p;
    casu_fits *skysub_im,**inlist2,**invar2;
    float fbad,medsky;

    /* Inherited status */

    *skyout = NULL;
    *skyvar = NULL;
    if (*status != CASU_OK) 
        return(*status);

    /* If there aren't any images, then get out of here */

    if (nfiles == 0) {
        cpl_msg_error(fctid,"Sky correction impossible. No science frames");
        FATAL_ERROR
    }

    /* Check to see which files are good */

    inlist2 = cpl_malloc(nfiles*sizeof(casu_fits *));
    if (invar != NULL)
        invar2 = cpl_malloc(nfiles*sizeof(casu_fits*));
    else
        invar2 = NULL;
    nfiles2 = 0;
    for (i = 0; i < nfiles; i++) {
        if (casu_fits_get_status(inlist[i]) == CASU_OK) {
            inlist2[nfiles2] = inlist[i];
            if (invar != NULL)
                invar2[nfiles2] = invar[i];
            nfiles2++;
        }
    }

    /* If no good images are present then wrap a dummy image and return it */

    if (nfiles2 == 0) {
        outim = casu_dummy_image(inlist[0]);
        *skyout = casu_fits_wrap(outim,inlist[0],NULL,NULL);
        casu_dummy_property(casu_fits_get_ehu(*skyout));
        casu_fits_set_status(*skyout,CASU_FATAL);
        if (invar != NULL) {
            outimv = casu_dummy_image(inlist[0]);
            *skyvar = casu_fits_wrap(outimv,inlist[0],NULL,NULL);
            casu_dummy_property(casu_fits_get_ehu(*skyvar));
        } else {
            *skyvar = NULL;
        }
        cpl_msg_warning(fctid,"No good images in input list");
        freespace(inlist2);
        *status = CASU_WARN;
        return(CASU_WARN);
    }

    /* Wrap the input mask into a cpl_mask structure and use it to set the
       internal mask for each of the input images */

    inbpm = casu_mask_get_data(mask);
    nx = casu_mask_get_size_x(mask);
    ny = casu_mask_get_size_y(mask);
    cplmask = cpl_mask_wrap((cpl_size)nx,(cpl_size)ny,(cpl_binary *)inbpm);
    for (i = 0; i < nfiles2; i++)
        cpl_image_reject_from_mask(casu_fits_get_image(inlist2[i]),cplmask);
    cpl_mask_unwrap(cplmask);
    ninit = cpl_image_count_rejected((const cpl_image *)casu_fits_get_image(inlist2[0]));

    /* Do an initial sky combination */

    casu_imcombine(inlist2,invar2,nfiles2,1,1,0,2.0,"EXPTIME",&outim,&outimv,
                   &rejmask,&rejplus,&drs,status);
    *skyout = casu_fits_wrap(outim,inlist2[0],NULL,NULL);    
    if (invar != NULL)
        *skyvar = casu_fits_wrap(outimv,invar2[0],NULL,NULL);    
    freespace(rejmask);
    freespace(rejplus);
    freepropertylist(drs);

    /* Clean up any blank bits */

    (void)casu_inpaint(*skyout,nbsize,status);

    /* If this is the only iteration then get out of here now */

    if (niter == 0)
        return(*status);

    /* Begin the iteration loop */

    nbad0 = 0;
    for (iter = 1; iter <= niter; iter++) {

        /* For each image, sky subtract using the latest sky. Then create
           an object mask from the sky subtracted image and count the number
           of bad pixels */

        skyim = casu_fits_get_image(*skyout);
        medsky = casu_sky_med(*skyout);
        nbad = 0;
        for (i = 0; i < nfiles2; i++) {
            im = casu_fits_get_image(inlist2[i]);
            outim = cpl_image_subtract_create(im,skyim);
            cpl_image_add_scalar(outim,(double)medsky);
            skysub_im = casu_fits_wrap(outim,inlist2[i],NULL,NULL);
            *status = CASU_OK;
            (void)casu_opm(skysub_im,conf,ipix,thresh,nbsize,smkern,2,
                           status);
            nb = (int)cpl_image_count_rejected((const cpl_image *)outim) - 
                ninit;
            nbad += nb;
            newmask = cpl_image_get_bpm(outim);
            oldmask = cpl_image_get_bpm(im);
            cpl_mask_or(oldmask,newmask);
            freefits(skysub_im);
        }

        /* Update report on the number of pixels rejected */

        dbad = nbad - nbad0;
        fbad = (iter > 1 ? (float)abs(dbad)/(float)nbad0 : 10000.0);
        cpl_msg_info(fctid,
                     "Iteration: %" CPL_SIZE_FORMAT ", Nreject: %" CPL_SIZE_FORMAT " %" CPL_SIZE_FORMAT,
                     (cpl_size)iter,(cpl_size)nbad,(cpl_size)nbad0);        
        
        if (fbad < 0.025 || dbad < 0) {
            lastone = 1;
        } else {
            nbad0 = nbad;
            lastone = 0;
        }

        /* If this has converged or this is the last iteration, then
           get out of here now. */

        if (lastone || iter == niter) {
            break;

        /* Otherwise, delete the current sky and create a new one */

        } else {
            freefits(*skyout);
            casu_imcombine(inlist2,invar2,nfiles2,1,1,0,2.0,"EXPTIME",&outim,
                           &outimv,&rejmask,&rejplus,&drs,status);
            *skyout = casu_fits_wrap(outim,inlist2[0],NULL,NULL);   
            if (invar != NULL) 
                *skyvar = casu_fits_wrap(outimv,invar2[0],NULL,NULL);   
            (void)casu_inpaint(*skyout,nbsize,status);
            freespace(rejmask);
            freespace(rejplus);
            freepropertylist(drs);
            p = casu_fits_get_ehu(*skyout);
            cpl_propertylist_update_string(p,"ESO DRS SKYALGO",
                                           "simplesky_mask");
            cpl_propertylist_set_comment(p,"ESO DRS SKYALGO",
                                         "Sky estimation algorithm");        
        }
    }
    freespace(inlist2);
    freespace(invar2);
    return(*status);
}
            
/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_sky_mask_grow
    \par Purpose:
        Grow the bad pixels in a frame out to a given radius
    \par Description:
        The mask associated with an input image is modified to grow the
        bad pixels out to a given radius.
    \par Language:
        C
    \param dith
        The input image
    \param rad
        The grow radius
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static void casu_sky_mask_grow(cpl_image *dith, float rad) {
    cpl_binary *inmap,*outmap;
    int nx,ny,ir,i,j,indx,ixmin,ixmax,iymin,iymax,ii,jj,indx2;
    float dx,dy,radius;

    /* Get the input map */

    inmap = cpl_mask_get_data(cpl_image_get_bpm(dith));
    nx = (int)cpl_image_get_size_x(dith);
    ny = (int)cpl_image_get_size_y(dith);

    /* Get an output map of the same size. Copy the input map to it */

    outmap = cpl_malloc(nx*ny*sizeof(*outmap));
    memmove(outmap,inmap,nx*ny*sizeof(*inmap));

    /* What is the minimum cell size that we need to consider given the 
       input radius */

    ir = casu_nint(rad);

    /* Right, loop through the input image. If the current input pixel is
       not flagged, then move on to the next one. If it is, then look at
       a cell around it. Find all the pixels in that cell that are within
       the grow radius and flag them */

    for (j = 0; j < ny; j++) {
        for (i = 0; i < nx; i++) {
            indx = j*nx + i;
            if (inmap[indx] != 2)
                continue;
            ixmin = max(0,i-ir);
            ixmax = min(nx-1,i+ir);
            iymin = max(0,j-ir);
            iymax = min(ny-1,j+ir);
            for (jj = iymin; jj <= iymax; jj++) {
                dy = (float)(jj - j);
                for (ii = ixmin; ii <= ixmax; ii++) {
                    dx = (float)(ii - i);
                    radius = (float)sqrt(pow(dx,2.0) + pow(dy,2.0));
                    if (radius <= rad) { 
                        indx2 = jj*nx + ii;
                        outmap[indx2] = 1;
                    }
                }
            }
        }
    }

    /* Now move the result back into the input mask. Free up workspace
       and get out of here */
   
    memmove(inmap,outmap,nx*ny*sizeof(*inmap));
    cpl_free(outmap);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_sky_med
    \par Purpose:
        Work out the median of a sky frame
    \par Description:
        Work out the median of a sky frame taking the associated bad
        pixel mask into account
    \par Language:
        C
    \param sky
        The input sky frame
    \returns
        The background median
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static float casu_sky_med(casu_fits *sky) {
    int npts;
    float *data,med;
    unsigned char *bpm;
    cpl_image *skyim;

    /* Get the size of the data array */

    skyim = casu_fits_get_image(sky);
    npts = (int)cpl_image_get_size_x(skyim)*(int)cpl_image_get_size_y(skyim);

    /* Get the data */

    data = cpl_image_get_data_float(skyim);
    bpm = (unsigned char *)cpl_mask_get_data(cpl_image_get_bpm(skyim));

    /* Get the median */

    med = casu_med(data,bpm,(long)npts);
    return(med);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_sky_joffs
    \par Purpose:
        Work out the jitter offsets for a list of files
    \par Description:
        Work out the jitter offsets for a list of files using the WCS
        information in the headers
    \par Language:
        C
    \param inlist
        The input frame list
    \param nfiles
        The number of files in the input list
    \param xoffs
        The X coordinate offset relative to the first good image in the list
    \param yoffs
        The Y coordinate offset relative to the first good image in the list
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static void casu_sky_joffs(casu_fits **inlist, int nfiles, float **xoffs,
                           float **yoffs) {
    float xoff,yoff;
    cpl_wcs *wcsref,*wcs;
    int refset,i,status;
    const double maxoffset = 12000;
    casu_fits *ff;
    const char *fctid = "casu_sky_joffs";

    /* Get a bit of workspace to hold the offsets */

    *xoffs = cpl_malloc(nfiles*sizeof(float));
    *yoffs = cpl_malloc(nfiles*sizeof(float));

    /* Work out the jitter offsets from the WCS. First loop for each image */

    refset = 0;
    wcsref = NULL;
    for (i = 0; i < nfiles; i++) {
        ff = inlist[i];
        wcs = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(ff));

        /* If we can't get a WCS for this image, then signal that with
           a warning */

        if (wcs == NULL) {
            cpl_msg_warning(fctid,"Unable to get WCS for %s",
                            casu_fits_get_filename(ff));
            (*xoffs)[i] = 0.0;
            (*yoffs)[i] = 0.0;
            casu_fits_set_error(ff,CASU_WARN);
            continue;
        }

        /* Define the reference wcs pointer and set the first offset to zero */

        if (! refset) {
            (*xoffs)[0] = 0.0;
            (*yoffs)[0] = 0.0;
            refset = 1;
            wcsref = wcs;
            continue;
        }

        /* Work out the x,y offset */

        status = CASU_OK;
        (void)casu_diffxywcs(wcs,wcsref,&xoff,&yoff,&status);

        /* Did it work? If not the set a warning status for this file */

        if (status != CASU_OK) {
            (*xoffs)[i] = 0.0;
            (*yoffs)[i] = 0.0;
            cpl_msg_warning(fctid,"Unable to WCS difference for %s",
                            casu_fits_get_filename(ff));
        } else if (fabs((double)xoff) > maxoffset || 
                   fabs((double)yoff) > maxoffset) {
            casu_fits_set_error(ff,CASU_FATAL);
            cpl_msg_error(fctid,"WCS offsets for %s are >%g: %g %g -- ignoring",
                          casu_fits_get_filename(ff),maxoffset,xoff,yoff);
        } else {
            (*xoffs)[i] = xoff;
            (*yoffs)[i] = yoff;
        }
        cpl_wcs_delete(wcs);
    }
    if (wcsref != NULL) 
        cpl_wcs_delete(wcsref);

    /* Write the results to the headers */

    for (i = 0; i < nfiles; i++) {
        ff = inlist[i];
        cpl_propertylist_update_double(casu_fits_get_ehu(ff),
                                       "ESO DRS XOFFDITHER",
                                       (double)(*xoffs)[i]);
        cpl_propertylist_update_double(casu_fits_get_ehu(ff),
                                       "ESO DRS YOFFDITHER",
                                       (double)(*yoffs)[i]);
    }
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        casu_sky_xytoxy
    \par Purpose:
        Convert xy positions between two images
    \par Description:
        Convert xy positions from one image to the reference frame
        of another
    \par Language:
        C
    \param inwcs
        The input WCS descriptor
    \param outwcs
        The output WCS descriptor
    \param inxy
        The xy coordinates in the frame of the input WCS
    \param outxy
        The xy coordinates in the frame of the output WCS
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static void casu_sky_xytoxy(cpl_wcs *inwcs, cpl_wcs *outwcs, 
                            cpl_matrix *inxy, cpl_matrix **outxy) {
    cpl_matrix *radec;
    cpl_array *wstatus;

    /* Convert the input xy data to RA/Dec */

    cpl_wcs_convert(inwcs,inxy,&radec,&wstatus,CPL_WCS_PHYS2WORLD);
    cpl_array_delete(wstatus);

    /* Now convert the RA/Dec to the xy coordinates of the second image */

    cpl_wcs_convert(outwcs,radec,outxy,&wstatus,CPL_WCS_WORLD2PHYS);
    cpl_array_delete(wstatus);
    cpl_matrix_delete(radec);
}
    
/**@}*/

/*

$Log: casu_sky.c,v $
Revision 1.2  2015/08/07 13:06:54  jim
Fixed copyright to ESO

Revision 1.1.1.1  2015/06/12 10:44:32  jim
Initial import

Revision 1.12  2015/04/30 12:07:43  jim
raised the maximum offset to be used in pawsky_mask

Revision 1.11  2015/03/03 10:48:11  jim
Fixed some memory leaks

Revision 1.10  2015/01/29 11:56:27  jim
modified comments

Revision 1.9  2015/01/09 12:13:15  jim
*** empty log message ***

Revision 1.8  2014/12/11 12:23:33  jim
new version

Revision 1.7  2014/04/09 11:08:21  jim
Get rid of a couple of compiler moans

Revision 1.6  2014/03/26 15:58:46  jim
Uses floating point confidence

Revision 1.5  2013/11/27 10:35:27  jim
Fixed routines so that add the DRS SKYALGO header keyword

Revision 1.4  2013/11/21 09:38:14  jim
detabbed

Revision 1.3  2013-10-24 09:26:28  jim
traps for dummy input files and tries to remove them from the analysis

Revision 1.2  2013-09-30 18:12:24  jim
Added casu_simplesky_mask

Revision 1.1.1.1  2013-08-27 12:07:48  jim
Imported


*/
