/* $Id: hawki_utils.c,v 1.12 2015/11/27 12:22:51 jim Exp $
 *
 * This file is part of the HAWKI Pipeline
 * Copyright (C) 2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jim $
 * $Date: 2015/11/27 12:22:51 $
 * $Revision: 1.12 $
 * $Name:  $
 */

/* Includes */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <sys/time.h>
#include <time.h>
#include <libgen.h>
#include <string.h>
#include <unistd.h>

#include <cpl.h>
#include <math.h>
#include <casu_utils.h>
#include <casu_stats.h>
#include <casu_fits.h>
#include <casu_tfits.h>
#include <casu_mods.h>
#include <casu_wcsutils.h>

#include "hawki_utils.h"

/**
    \defgroup hawki_utils hawki_utils
    \ingroup supportroutines

    \brief
    These are utility routines of various types.

    \author
    Jim Lewis, CASU
*/

/**@{*/

static void
hawki_propagate_table(cpl_size ext, const char* filename_from,
                          const char* filename_to);

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_get_license
    \par Purpose:
        Get the pipeline copyright and license
    \par Description:
        The function returns a pointer to the statically allocated license 
        string. This string should not be modified using the returned pointer.
    \par Language:
        C
    \return The copyright and license string
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/
extern const char *hawki_get_license(void) {
    const char  *hawki_license = 
        "This file is part of the HAWKI Instrument Pipeline\n"
        "Copyright (C) 2015 European Southern Observatory\n"
        "\n"
        "This program is free software; you can redistribute it and/or modify\n"
        "it under the terms of the GNU General Public License as published by\n"
        "the Free Software Foundation; either version 2 of the License, or\n"
        "(at your option) any later version.\n"
        "\n"
        "This program is distributed in the hope that it will be useful,\n"
        "but WITHOUT ANY WARRANTY; without even the implied warranty of\n"
        "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n"
        "GNU General Public License for more details.\n"
        "\n"
        "You should have received a copy of the GNU General Public License\n"
        "along with this program; if not, write to the Free Software\n"
        "Foundation, Inc., 59 Temple Place, Suite 330, Boston, \n"
        "MA  02111-1307  USA";
    return(hawki_license);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_create_diffimg_stats
    \par Purpose:
        Create an empty difference image stats table
    \par Description:
        Create an empty difference image stats table
    \par Language:
        C
    \param nrows
        The number of rows for the table
    \returns
        The cpl_table pointer for the new stats table
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern cpl_table *hawki_create_diffimg_stats(int nrows) {
    cpl_table *diffimstats;

    diffimstats = cpl_table_new((cpl_size)nrows);
    cpl_table_new_column(diffimstats,"xmin",CPL_TYPE_INT);
    cpl_table_set_column_unit(diffimstats,"xmin","pixels");
    cpl_table_new_column(diffimstats,"xmax",CPL_TYPE_INT);
    cpl_table_set_column_unit(diffimstats,"xmax","pixels");
    cpl_table_new_column(diffimstats,"ymin",CPL_TYPE_INT);
    cpl_table_set_column_unit(diffimstats,"ymin","pixels");
    cpl_table_new_column(diffimstats,"ymax",CPL_TYPE_INT);
    cpl_table_set_column_unit(diffimstats,"ymax","pixels");
    cpl_table_new_column(diffimstats,"chan",CPL_TYPE_INT);
    cpl_table_set_column_unit(diffimstats,"chan","pixels");
    cpl_table_new_column(diffimstats,"mean",CPL_TYPE_FLOAT);
    cpl_table_set_column_unit(diffimstats,"mean","ADU");
    cpl_table_new_column(diffimstats,"median",CPL_TYPE_FLOAT);
    cpl_table_set_column_unit(diffimstats,"median","ADU");
    cpl_table_new_column(diffimstats,"variance",CPL_TYPE_FLOAT);
    cpl_table_set_column_unit(diffimstats,"variance","ADU**2");
    cpl_table_new_column(diffimstats,"mad",CPL_TYPE_FLOAT);
    cpl_table_set_column_unit(diffimstats,"mad","ADU");
    return(diffimstats);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_difference_image
    \par Purpose:
        Create a difference/ratio image and difference/ratio image stats table
    \par Description:
        A difference/ratio image is created from an input image and a master 
        image. A global difference and RMS are calculated from the difference 
        image. Then a difference image stats table will be created. This 
        breaks up the difference image into cells and calculates some basic 
        stats in each cell.
    \par Language:
        C
    \param master
        The master calibration image
    \param prog
        The new mean image
    \param bpm 
        Input bad pixel mask
    \param ncells
        The number of cells per channel
    \param oper
        The operation to be performed:
        - 1. Subtract the images
        - 2. Divide the images
    \param global_diff 
        The median difference over the whole difference image
    \param global_rms  
        The rms difference over the whole difference image
    \param diffim
        The output difference/ratio image
    \param diffimstats 
        The output difference/ratio image statistics table
    \return   
        Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void hawki_difference_image(cpl_image *master, cpl_image *prog, 
                                   unsigned char *bpm, int ncells, int oper, 
                                   float *global_diff, float *global_rms, 
                                   cpl_image **diffim, 
                                   cpl_table **diffimstats) {
    float *ddata,*work,mean,sig,med,mad;
    long nx,ny,npts;
    int nrows,i,nc1,nc2,nr,ixmin,ixmax,iymin,iymax,cnum,cx,cy,idx,idy;
    int icx,icy,indy1,indy2,indx1,indx2,jp,jcx,jj,jcy,ii,ncx,ncy;
    const char *fctid = "hawki_difference_image";

    /* Initialise the output */

    *diffim = NULL;
    *diffimstats = NULL;
    *global_diff = 0.0;
    *global_rms = 0.0;

    /* Is there a programme frame or a master? */

    if (prog == NULL || master == NULL)
        return;

    /* Start by subtracting the master image from the programme image */

    switch (oper) {
    case 1:
        *diffim = cpl_image_subtract_create(prog,master);
        break;
    case 2:
        *diffim = cpl_image_divide_create(prog,master);
        break;
    default:
        *diffim = NULL;
        cpl_msg_error(fctid,"Invalid operation requested %" CPL_SIZE_FORMAT,
                      (cpl_size)oper);
        break;
    }           
    if (*diffim == NULL)
        return;

    /* Work out a median difference */

    ddata = cpl_image_get_data_float(*diffim);
    nx = (int)cpl_image_get_size_x(*diffim);
    ny = (int)cpl_image_get_size_y(*diffim);
    npts = nx*ny;
    casu_medmad(ddata,bpm,npts,global_diff,global_rms);
    *global_rms *= 1.48;

    /* Work out how to divide the channels */

    switch (ncells) {
    case 1:
        nc1 = 1;
        nc2 = 1;
        break;
    case 2:
        nc1 = 2;
        nc2 = 1;
        break;
    case 4:
        nc1 = 4;
        nc2 = 1;
        break;
    case 8:
        nc1 = 8;
        nc2 = 1;
        break;
    case 16:
        nc1 = 16;
        nc2 = 1;
        break;
    case 32:
        nc1 = 16;
        nc2 = 2;
        break;
    case 64:
        nc1 = 32;
        nc2 = 2;
        break;
    default:
        nc1 = 32;
        nc2 = 2;
        break;
    }

    /* Set up the limits of the channels */

    nrows = HAWKI_NX/HAWKI_CHAN_WIDTH;

    /* Create a difference image stats table */

    *diffimstats = hawki_create_diffimg_stats(nrows*nc1*nc2);
    
    /* Loop for each data channel now */

    nr = 0;
    for (i = 0; i < nrows; i++) {
        ixmin = i*HAWKI_CHAN_WIDTH + 1;
        ixmax = ixmin + HAWKI_CHAN_WIDTH - 1;
        iymin = 1;
        iymax = HAWKI_NY;
        cnum = i + 1;

        /* Which is the long axis? If the channels are rectangular then
           divide the long axis by the greater number of cells. If the number
           of cells is less than 8, then just divide the long axis. If the
           channel is square, then divide the X axis more finely */

        cx = ixmax - ixmin + 1;
        cy = iymax - iymin + 1;
        if (cx > cy) {
            ncx = max(nc1,nc2);
            ncy = min(nc1,nc2);
        } else if (cx < cy) {
            ncy = max(nc1,nc2);
            ncx = min(nc1,nc2);
        } else {
            ncx = max(nc1,nc2);
            ncy = min(nc1,nc2);
        }
        
        /* How big is the cell? */

        idy = cy/ncy;
        idx = cx/ncx;
        work = cpl_malloc(idx*idy*sizeof(*work));

        /* Now loop for each cell */

        for (icy = 0; icy < ncy; icy++) {
            indy1 = idy*icy + iymin - 1;
            indy2 = min(iymax,indy1+idy-1);
            for (icx = 0; icx < ncx; icx++) {
                indx1 = idx*icx + ixmin - 1;
                indx2 = min(ixmax,indx1+idx-1);
                jp = 0;
                for (jcy = indy1; jcy < indy2; jcy++) {
                    jj = jcy*nx;
                    for (jcx = indx1; jcx < indx2; jcx++) {
                        ii = jj + jcx;
                        if (bpm != NULL && bpm[ii] == 0)
                            work[jp++] = ddata[ii];
                        else if (bpm == NULL)
                            work[jp++] = ddata[ii];
                    }
                }
                (void)casu_meansig(work,NULL,(long)jp,&mean,&sig);
                (void)casu_medmad(work,NULL,(long)jp,&med,&mad);
                cpl_table_set_int(*diffimstats,"xmin",(cpl_size)nr,indx1+1);
                cpl_table_set_int(*diffimstats,"xmax",(cpl_size)nr,indx2+1);
                cpl_table_set_int(*diffimstats,"ymin",(cpl_size)nr,indy1+1);
                cpl_table_set_int(*diffimstats,"ymax",(cpl_size)nr,indy2+1);
                cpl_table_set_int(*diffimstats,"chan",(cpl_size)nr,cnum);
                cpl_table_set_float(*diffimstats,"mean",(cpl_size)nr,mean);
                cpl_table_set_float(*diffimstats,"median",(cpl_size)nr,med);
                cpl_table_set_float(*diffimstats,"variance",(cpl_size)nr,
                                    (sig*sig));
                cpl_table_set_float(*diffimstats,"mad",(cpl_size)(nr++),mad);
            }
        }
        cpl_free(work);
    }
}   

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_wcsfit_multi
    \par Purpose:
        Do a WCS fit to four images in a given exposure simultaneously
    \par Description:
        Create a source catalogue for each image if one isn't given in the
        argument list. Transform the coordinates into the reference frame
        of the first image and fit a WCS to all the objects simultaneously.
    \par Language:
        C
    \param in
        A list of casu_fits structures for the input images
    \param incat
        A list of casu_tfits structures for the input source catalogues.
    \param catname
        The name of the standards catalogue
    \param catpath
        The full path to the catalogue index file
    \param cdssearch
        The CDS catalogue to be searched for standards. 0 if using local
        catalogues.
    \param cacheloc
        The location of the standard star cache
    \param keepms
        True if we want to keep the matched standards catalogues for
        later use
    \param outms
        An array of casu_tfits structures to be used for storing the
        matched standards catalogues if we want to keep them
    \returns
        Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void hawki_wcsfit_multi(casu_fits **in, casu_tfits **incat, 
                               char *catname, char *catpath, int cdssearch, 
                               char *cacheloc, int keepms, casu_tfits **outms) {
    const char *fctid = "hawki_wcsfit_multi";
    double forward[HAWKI_NEXTN][6] = {
        {1.00000000,0.00000000,0.00000000,1.00000000,0.00000000,0.00000000},
        {0.99989338,-0.00082458,2202.75199936,0.99989338,0.00082458,2.52486889},
        {0.99954089,0.00033954,7.35707930,0.99954089,-0.00033954,2189.67113908},
        {0.99984901,0.00057963,2207.25004571,0.99984901,-0.00057963,2191.08171555}};
    double backward[HAWKI_NEXTN][6] = {
        {1.00000000,0.00000000,0.00000000,1.00000000,0.00000000,0.00000000},
        {1.00010595,0.00082485,-2202.98755123,1.00010595,-0.00082485,-0.70890643},
        {1.00045920,-0.00033976,-6.61657784,1.00045920,0.00033976,-2190.67905279},
        {1.00015067,-0.00057971,-2206.31252191,1.00015067,0.00057971,-2192.69133148}
};
    int j,status,ncat,slevel,nstd,nmaster,i,nr,k,n,firstgood;
    casu_tfits *tcat;
    cpl_table *stdscat,*tmp,*tmp2,*cat,*master;
    cpl_table *matchstds[HAWKI_NEXTN];
    cpl_propertylist *p;
    float *xptr_old,*yptr_old,*xptr_new,*yptr_new,*x,*y,*ra,*dec;
    double *rac,*decc,*dra,*ddec,*ra_d,*dec_d,r,d;
    cpl_wcs *wcs;

    /* Loop for each extension. If we need a source list, then generate
       one. Also get standards for each extension */

    for (j = 0; j < HAWKI_NEXTN; j++) {
        status = CASU_OK;
        matchstds[j] = NULL;
        if (casu_fits_get_status(in[j]) != CASU_OK) 
            continue;

        /* Dereference a few things */
    
        if (incat[j] == NULL)
            continue;
        tcat = incat[j];
        cat = casu_tfits_get_table(tcat);
        ncat = (int)cpl_table_get_nrow(cat);
    
        /* Get some standard stars */

        (void)casu_getstds(casu_fits_get_ehu(in[j]),1,catpath,catname,cdssearch,
                           cacheloc,&stdscat,&status);
        status = CASU_OK;
        nstd = (int)cpl_table_get_nrow(stdscat);

        /* If there are too many objects in the catalogue then first restrict
           ourselves by ellipticity. Cut so that there are similar numbers of
           objects in the standards and the object catalogues by retaining the
           brighter objects */

        if (ncat > 500 && ncat > 2.0*nstd) {
            tmp = cpl_table_duplicate(cat);
            (void)cpl_table_or_selected_float(tmp,"Ellipticity",
                                              CPL_LESS_THAN,0.5);
            tmp2 = cpl_table_extract_selected(tmp);
            ncat = (int)cpl_table_get_nrow(tmp2);
            freetable(tmp);
            p = cpl_propertylist_new();
            cpl_propertylist_append_bool(p,"Isophotal_flux",TRUE);            
            cpl_table_sort(tmp2,(const cpl_propertylist *)p);
            cpl_propertylist_delete(p);
            slevel = min(ncat,max(1,min(5000,max(500,2*nstd))));
            tmp = cpl_table_extract(tmp2,1,(cpl_size)slevel);
            freetable(tmp2);
            ncat = (int)cpl_table_get_nrow(tmp);
            cat = tmp;
        } else {
            tmp = NULL;
        }

        /* Now match this against the catalogue */

        (void)casu_matchstds(cat,stdscat,300.0,&(matchstds[j]),&status);
        freetable(stdscat);
        freetable(tmp);

        /* Duplicate the x,y position rows for use later */

        cpl_table_duplicate_column(matchstds[j],"X_orig",matchstds[j],
                                   "X_coordinate");
        cpl_table_duplicate_column(matchstds[j],"Y_orig",matchstds[j],
                                   "Y_coordinate");
    }

    /* Look for the first good matched standards catalogue */

    firstgood = -1;
    for (j = 0; j < HAWKI_NEXTN; j++) {
        if (matchstds[j] != NULL) {
            firstgood = j;
            break;
        }
    }

    /* If none are good, then get out of here now... */

    if (firstgood == -1) {
        cpl_msg_error(fctid,"No good extensions. No WCS fit possible");
        return;
    }

    /* OK, now loop for each extension again. Start by modifiying the x,y
       positions in the match stds catalogues so that they are on in the
       reference frame of each extension in turn. */

    for (j = 0; j < HAWKI_NEXTN; j++) {

        /* The first time around, take each of the matched standards files
           and convert the x,y coordinates to the standard system. Merge
           the tables into a single master table, then get rid of the
           original matched tables. */

        master = cpl_table_duplicate(matchstds[firstgood]);
        cpl_table_select_all(master);
        cpl_table_erase_selected(master);
        nmaster = 0;
        for (i = 0; i < HAWKI_NEXTN; i++) {
            if (matchstds[i] == NULL)
                continue;
            nr = cpl_table_get_nrow(matchstds[i]);
            xptr_old = cpl_table_get_data_float(matchstds[i],"X_orig");
            yptr_old = cpl_table_get_data_float(matchstds[i],"Y_orig");
            xptr_new = cpl_table_get_data_float(matchstds[i],"X_coordinate");
            yptr_new = cpl_table_get_data_float(matchstds[i],"Y_coordinate");
            for (k = 0; k < nr; k++) {
                xptr_new[k] = forward[i][0]*xptr_old[k] + 
                    forward[i][1]*yptr_old[k] + forward[i][2];
                yptr_new[k] = forward[i][3]*yptr_old[k] + 
                    forward[i][4]*xptr_old[k] + forward[i][5];
            }
            cpl_table_insert(master,matchstds[i],nmaster);
            nmaster = cpl_table_get_nrow(master);
        }
        xptr_new = cpl_table_get_data_float(master,"X_coordinate");
        yptr_new = cpl_table_get_data_float(master,"Y_coordinate");
        xptr_old = cpl_table_get_data_float(master,"X_orig");
        yptr_old = cpl_table_get_data_float(master,"Y_orig");
        memmove(xptr_old,xptr_new,nmaster*sizeof(float));
        memmove(yptr_old,yptr_new,nmaster*sizeof(float));
        for (k = 0; k < nmaster; k++) {
            xptr_new[k] = backward[j][0]*xptr_old[k] + 
                backward[j][1]*yptr_old[k] + backward[j][2];
            yptr_new[k] = backward[j][3]*yptr_old[k] + 
                backward[j][4]*xptr_old[k] + backward[j][5];
        }            

        /* Do the plate solution now for the current extensions */

        status = CASU_OK;
        (void)casu_platesol(casu_fits_get_ehu(in[j]),
                            casu_tfits_get_ehu(incat[j]),master,6,0,
                            &status);
        if (status != CASU_OK) {
            cpl_msg_error(fctid,"Failed to fit WCS for extn %" CPL_SIZE_FORMAT,
                          (cpl_size)(j+1));
            freetable(master);
            continue;
        }

        /* Update the RA and DEC of the objects in the object catalogue */

        if (casu_fits_get_status(in[j]) == CASU_OK) {
            cat = casu_tfits_get_table(incat[j]);
            if (cat == NULL) {
                freetable(master);
                continue;
            }
            n = (int)cpl_table_get_nrow(cat);
            wcs = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(in[j]));
            if (wcs == NULL) {
                cpl_msg_error(fctid,
                              "Failed to fill RA and Dec in catalogue");
                freetable(master);
                continue;
            }
            x = cpl_table_get_data_float(cat,"X_coordinate"); 
            y = cpl_table_get_data_float(cat,"Y_coordinate");
            ra = cpl_table_get_data_float(cat,"RA");
            dec = cpl_table_get_data_float(cat,"DEC");
            for (i = 0; i < n; i++) {
                casu_xytoradec(wcs,(double)x[i],(double)y[i],&r,&d);
                ra[i] = (float)r;
                dec[i] = (float)d;
            }
            cpl_wcs_delete(wcs);
        }
        freetable(master);
    }

    /* If we're keeping the matched standards catalogue then tidy a few
       things up. If not then just get rid of them */

    if (keepms) {
        for (j = 0; j < HAWKI_NEXTN; j++) {
            outms[j] = casu_tfits_wrap(matchstds[j],incat[j],NULL,NULL);

            /* First get rid of the modified coordinates and restore the
               original ones */

            cpl_table_erase_column(matchstds[j],"X_coordinate");
            cpl_table_name_column(matchstds[j],"X_orig","X_coordinate");
            cpl_table_erase_column(matchstds[j],"Y_coordinate");
            cpl_table_name_column(matchstds[j],"Y_orig","Y_coordinate");

            /* Add some extra columns for the computed coordinates and
               the coordinate differences */

            cpl_table_duplicate_column(matchstds[j],"RA_calc",matchstds[j],
                                       "RA");
            rac = cpl_table_get_data_double(matchstds[j],"RA_calc");
            cpl_table_duplicate_column(matchstds[j],"diffRA",matchstds[j],"RA");
            dra = cpl_table_get_data_double(matchstds[j],"diffRA");
            cpl_table_duplicate_column(matchstds[j],"Dec_calc",matchstds[j],
                                       "Dec");
            decc = cpl_table_get_data_double(matchstds[j],"Dec_calc");
            cpl_table_duplicate_column(matchstds[j],"diffDec",matchstds[j],"Dec");
            ddec = cpl_table_get_data_double(matchstds[j],"diffDec");

            /* Now compute the equatorial coordinates and compare with the 
               standard values */

            n = (int)cpl_table_get_nrow(matchstds[j]);
            x = cpl_table_get_data_float(matchstds[j],"X_coordinate");
            y = cpl_table_get_data_float(matchstds[j],"Y_coordinate");
            ra_d = cpl_table_get_data_double(matchstds[j],"RA");
            dec_d = cpl_table_get_data_double(matchstds[j],"Dec");
            wcs = cpl_wcs_new_from_propertylist(casu_fits_get_ehu(in[j]));
            for (i = 0; i < n; i++) {
                casu_xytoradec(wcs,(double)x[i],(double)y[i],&r,&d);
                rac[i] = (float)r;
                decc[i] = (float)d;
                dra[i] = rac[i] - ra_d[i];
                ddec[i] = decc[i] - dec_d[i];
            }
            cpl_wcs_delete(wcs);
        }
    } else {
        for (i = 0; i < HAWKI_NEXTN; i++)
            freetable(matchstds[i]);
    }

    /* Get out of here */
    
    return;
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_testfrms
    \par Purpose:
        Test the frames in a frameset to make sure they are what you 
        expect
    \par Description:
        Each of the frames in a frameset are checked to make sure they
        have the right number of frames and that each image/table
        is loadable.
    \par Language:
        C
    \param frms
        The input frameset
    \param nextn_expected
        The number of extensions expected in each image/table file
    \param isimg
        TRUE if these are image frames. Otherwise we check for tables
    \param checkwcs
        TRUE if we need to check for a valid WCS
    \return
        The number of errors
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int hawki_testfrms(cpl_frameset *frms, int nextn_expected, int isimg, 
                          int checkwcs) {
    int i,nf,nerr;
    cpl_frame *fr;

    /* Return immediately if given nonsense */

    if (frms == NULL) 
        return(0);

    /* Loop for each frame in the frameset */

    nf = cpl_frameset_get_size(frms);
    nerr = 0;
    for (i = 0; i < nf; i++) {
        fr = cpl_frameset_get_position(frms,i);
        nerr += hawki_testfrm_1(fr,nextn_expected,isimg,checkwcs);
    }
    
    /* Return value */

    return(nerr);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_testfrm_1
    \par Purpose:
        Test a frame to make sure it is what you expect
    \par Description:
        The frame is checked to make sure it has the right number of 
        extensions and that each image/table is loadable.
    \par Language:
        C
    \param fr
        The input frame
    \param nextn_expected
        The number of extensions expected in each image/table file
    \param isimg
        TRUE if this is an image frame. Otherwise we check for tables
    \param checkwcs
        TRUE if we need to check for a valid WCS
    \return
        The number of errors
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int hawki_testfrm_1(cpl_frame *fr, int nextn_expected, int isimg,
                           int checkwcs) {
    int nextn,nerr,j;
    casu_fits *test;
    casu_tfits *testt;
    cpl_propertylist *p;
    cpl_wcs *wcs;
    const char *fctid="hawki_testfrm_1";

    /* Return immediately if given nonsense */

    if (fr == NULL)
        return(0);

    /* Test to see how many extensions there are and compare to see
       if it matches the number expected */

    nextn = cpl_frame_get_nextensions(fr);

    nextn = hawki_aodata_nextn_correct(nextn);

    if (nextn != nextn_expected) {
        cpl_msg_error(fctid,"Frame %s has %" CPL_SIZE_FORMAT " extensions, expected %" CPL_SIZE_FORMAT "\n",
                      cpl_frame_get_filename(fr),(cpl_size)nextn,
                      (cpl_size)nextn_expected);
        return(1);
    }

    /* Test to see if you can load each of the extensions */

    nerr = 0;
    for (j = 1; j <= nextn; j++) {
        if (isimg) {
            test = casu_fits_load(fr,CPL_TYPE_UNSPECIFIED,j);
            if (test == NULL) {
                cpl_msg_error(fctid,
                              "Frame image %s[%" CPL_SIZE_FORMAT "] won't load",
                              cpl_frame_get_filename(fr),(cpl_size)j);
                nerr++;
                continue;
            }
            if (checkwcs) {
                p = casu_fits_get_ehu(test);
                wcs = cpl_wcs_new_from_propertylist(p);
                if (wcs == NULL) {
                    cpl_msg_error(fctid,
                                  "Frame image %s[%" CPL_SIZE_FORMAT "] WCS invalid",
                                  cpl_frame_get_filename(fr),(cpl_size)j);
                    cpl_error_reset();
                    nerr++;
                    continue;
                } 
                freewcs(wcs);
            }
            freefits(test);
        } else {
            testt = casu_tfits_load(fr,j);
            if (testt == NULL) {
                cpl_msg_error(fctid,
                              "Frame table %s[%" CPL_SIZE_FORMAT "] won't load\n",
                              cpl_frame_get_filename(fr),(cpl_size)j);
                nerr++;
                continue;
            }
            freetfits(testt);
        }
    }
    return(nerr);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_testrdgn
    \par Purpose:
        Test the read/gain table
    \par Description:
        The read/gain table is tested to make sure it's readable and to
        make sure there is one row for each detector. No testing is
        done to the actual values in the rows.
    \par Language:
        C
    \param fr
        An example input frame that can be used to find the extension names
    \param readgain
        The read/gain table
    \return
        The number of errors
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int hawki_testrdgn(cpl_frame *fr, cpl_frame *readgain) {
    int i,nerr;
    cpl_table *rdgn;
    casu_fits *f;
    const char *fctid = "hawki_testrdgn";

    /* Can we load the table? */

    nerr = 0;
    rdgn = cpl_table_load(cpl_frame_get_filename(readgain),1,0);
    if (cpl_error_get_code() != 0) {
        cpl_msg_error(fctid,"Read/gain table %s[1] won't load\n",
                      cpl_frame_get_filename(readgain));
        nerr++;
        return(nerr);
    }

    /* Loop for each extension and make sure there is an entry for each one */

    for (i = 1; i <= HAWKI_NEXTN; i++) {
        cpl_table_unselect_all(rdgn);
        f = casu_fits_load(fr,CPL_TYPE_UNSPECIFIED,i);
        cpl_table_or_selected_string(rdgn,"EXTNAME",CPL_EQUAL_TO,
                                     casu_fits_get_extname(f));
        if (cpl_table_count_selected(rdgn) != 1) {
            cpl_msg_error(fctid,"No read/gain entry for %s",
                          casu_fits_get_extname(f));
            nerr++;
        }
        freefits(f);
    }
    cpl_table_delete(rdgn);
    return(nerr);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_getrdgn
    \par Purpose:
        Get the readnoise and gain estimate for the current detector
    \par Description:
        The read/gain table is searched for the current detector name.
        The readnoise and gain estimates are returned for that detector.
    \par Language:
        C
    \param readgain
        Input frame with the master read/gain table
    \param extname
        The extension name for which we want readnoise and gain information
    \param readnoise
        The output readnoise in ADUs
    \param gain
        The output gain in e-/ADUs
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void hawki_getrdgn(cpl_frame *readgain, char *extname, float *readnoise,
                          float *gain) {
    cpl_table *rgtab,*subset;
    int isnull;

    /* Read the input table */

    rgtab = cpl_table_load(cpl_frame_get_filename(readgain),1,0);

    /* Search the table for the correct row. NB: the read/gain table has
       already been tested so there should be no error here */

    cpl_table_unselect_all(rgtab);
    cpl_table_or_selected_string(rgtab,"EXTNAME",CPL_EQUAL_TO,extname);
    subset = cpl_table_extract_selected(rgtab);
    *readnoise = cpl_table_get_float(subset,"READNOISE",0,&isnull);
    *gain = cpl_table_get_float(subset,"GAIN",0,&isnull);
    cpl_table_delete(subset);
    cpl_table_delete(rgtab);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_updatewcs
    \par Purpose:
        Modify the WCS to the correct projection
    \par Description:
        The input WCS of a propertylist is updated so that it contains
        the ZPN projection required for the instrument
    \par Language:
        C
    \param p
        The input propertylist
    \return
        Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void hawki_updatewcs(cpl_propertylist *p) {
    if (cpl_propertylist_has(p,"CTYPE1")) 
        cpl_propertylist_update_string(p,"CTYPE1",CTYPE1);
    else
        cpl_propertylist_append_string(p,"CTYPE1",CTYPE1);
    if (cpl_propertylist_has(p,"CTYPE2")) 
        cpl_propertylist_update_string(p,"CTYPE2",CTYPE2);
    else
        cpl_propertylist_append_string(p,"CTYPE2",CTYPE2);

    if (cpl_propertylist_has(p,"PV2_1")) 
        cpl_propertylist_update_float(p,"PV2_1",PV2_1);
    else
        cpl_propertylist_append_float(p,"PV2_1",PV2_1);
    if (cpl_propertylist_has(p,"PV2_3")) 
        cpl_propertylist_update_float(p,"PV2_3",PV2_3);
    else
        cpl_propertylist_append_float(p,"PV2_3",PV2_3);
    if (cpl_propertylist_has(p,"PV2_5")) 
        cpl_propertylist_update_float(p,"PV2_5",PV2_5);
    else
        cpl_propertylist_append_float(p,"PV2_5",PV2_5);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_copywcs
    \par Purpose:
        Copy a WCS from one propertylist to another
    \par Description:
        Copy a full WCS from the first propertylist into the second one
    \par Language:
        C
    \param p1
        The first input propertylist. This is the source of the WCS
    \param p2
        The second input propertylist. The wCS will be copied to this
    \return
        Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void hawki_copywcs(cpl_propertylist *p1, cpl_propertylist *p2) {
    (void)cpl_propertylist_erase_regexp(p2,CPL_WCS_REGEXP,0);
    cpl_propertylist_copy_property_regexp(p2,p1,CPL_WCS_REGEXP,0);
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Correct the number of retrieved extensions for adaptive optic data
  @param    nextn   Number of retrieved extensions
  @return   corrected number of extensions for AO data
 */
/*----------------------------------------------------------------------------*/

int hawki_aodata_nextn_correct(int nextn)
{
    if (nextn > HAWKI_NEXTN) {
        /* Assuming data from adaptive optics are in the last extensions that
         * should be ignored by the pipeline */
        cpl_msg_info(cpl_func, "Number of extension exceeds %d, ignoring all"
                        " extensions larger than %d", HAWKI_NEXTN, HAWKI_NEXTN);
        return HAWKI_NEXTN;
    }
    else {
        return nextn;
    }
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Propagates the adaptive optics extension from the raw to the product
  @param    frameset  The frameset with the raw and product files
  @param    from  Tag of the frames with the AO extesnions
  @param    to Tag of the frames where the AO extensions should be copied
  @return   CPL_ERROR_NONE if no real error happened.
 */
/*----------------------------------------------------------------------------*/

cpl_error_code hawki_propagate_aoextensions(cpl_frameset * frameset,
                                            const char * from, const char * to)
{
    /* Check input in not empty*/

    cpl_error_ensure(frameset != NULL, CPL_ERROR_NULL_INPUT,
                    return CPL_ERROR_NULL_INPUT, "Framelist is empty");
    cpl_error_ensure(from != NULL, CPL_ERROR_NULL_INPUT,
                     return CPL_ERROR_NULL_INPUT, "from string is pointing to NULL");
    cpl_error_ensure(to != NULL, CPL_ERROR_NULL_INPUT,
                     return CPL_ERROR_NULL_INPUT, "to string is pointing to NULL");


    cpl_frameset * fs_from = cpl_frameset_new();
    cpl_frameset * fs_to = cpl_frameset_new();

    /* populate from and to framesets */
    for (cpl_size i = 0; i < cpl_frameset_get_size(frameset); i++) {
        cpl_frame * frm =
                        cpl_frame_duplicate(cpl_frameset_get_position(frameset, i));
        if (!strcmp(cpl_frame_get_tag(frm), from)) {
            cpl_frameset_insert(fs_from, frm);
        }
        else if (!strcmp(cpl_frame_get_tag(frm), to)) {
            cpl_frameset_insert(fs_to, frm);
        }
        else{
            cpl_frame_delete(frm);
        }
    }

    /* Check that the number of from and to are the same and larger then 0 */
    if (cpl_frameset_get_size(fs_to) == 0 || cpl_frameset_get_size(fs_from) == 0 ||
                    cpl_frameset_get_size(fs_to) != cpl_frameset_get_size(fs_from)) {
        cpl_frameset_delete(fs_to);
        cpl_frameset_delete(fs_from);

        cpl_msg_info(cpl_func, "No AO extensions propagated");
        return CPL_ERROR_NONE;
    }

    cpl_size ext = 0;
    cpl_frame * frm_from = NULL;
    cpl_frame * frm_to = NULL;
    const char * filename_from = NULL;
    const char * filename_to = NULL;

    /* propagate the AO extensions */
    for (cpl_size i = 0; i < cpl_frameset_get_size(fs_from); i++) {

        frm_from = cpl_frameset_get_position(fs_from, i);
        frm_to = cpl_frameset_get_position(fs_to, i);
        filename_from = cpl_frame_get_filename(frm_from);
        filename_to = cpl_frame_get_filename(frm_to);

        ext = cpl_fits_find_extension(filename_from, "SPARTA_ATM_DATA");
        hawki_propagate_table(ext, filename_from, filename_to);

        ext = cpl_fits_find_extension(filename_from, "TCS_DATA");
        hawki_propagate_table(ext, filename_from, filename_to);
    }

    cpl_frameset_delete(fs_to);
    cpl_frameset_delete(fs_from);

    return cpl_error_get_code();
}

static void
hawki_propagate_table(cpl_size ext, const char* filename_from,
                          const char* filename_to)
{
    if (ext > 0) {
        cpl_propertylist* plist = cpl_propertylist_load(filename_from, ext);
        /* only copy if it is a table */
        const char* checktable = cpl_propertylist_get_string(plist, "XTENSION");
        if (checktable != NULL && strcmp(checktable, "BINTABLE") == 0) {
            cpl_table* tbl_from = cpl_table_load(filename_from, ext, 0);
            cpl_table_save(tbl_from, NULL, plist, filename_to, CPL_IO_EXTEND);
            cpl_table_delete(tbl_from);
        }
        cpl_propertylist_delete(plist);
    }
}



/**@}*/

/*

$Log: hawki_utils.c,v $
Revision 1.12  2015/11/27 12:22:51  jim
Added cacheloc parameter

Revision 1.11  2015/09/15 10:29:13  jim
added hawki_copywcs

Revision 1.10  2015/09/11 09:30:33  jim
Fixed testfrms_1 so that if wcs fails the cpl error is reset

Revision 1.9  2015/08/07 13:07:06  jim
Fixed copyright to ESO

Revision 1.8  2015/08/04 08:39:59  jim
Plugged memory leak that happens in the case where there are no standards
in the wcs fitting

Revision 1.7  2015/05/13 11:56:25  jim
A few changes to hawki_wcsfit_multi to remove 2mass explicit stuff and to
make it a little more robust

Revision 1.6  2015/03/12 09:16:51  jim
Modified to remove some compiler moans

Revision 1.5  2015/02/14 12:36:05  jim
Expanded and modified hawki_wcsfit_multi

Revision 1.4  2015/01/29 11:59:01  jim
Added support routines that used to be in with the recipe source code

Revision 1.3  2014/12/11 12:24:35  jim
Lots of upgrades

Revision 1.2  2013/11/21 09:38:14  jim
detabbed

Revision 1.1.1.1  2013-08-27 12:07:48  jim
Imported


*/

