/* $Id: hawki_detector_noise.c,v 1.6 2015/09/11 09:31:20 jim Exp $
 *
 * This file is part of the HAWKI Pipeline
 * Copyright (C) 2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jim $
 * $Date: 2015/09/11 09:31:20 $
 * $Revision: 1.6 $
 * $Name:  $
 */

/* Includes */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <cpl.h>
#include <math.h>
#include <string.h>

#include "hawki_utils.h"
#include "hawki_dfs.h"
#include "hawki_pfits.h"
#include "casu_stats.h"
#include "casu_fits.h"
#include "casu_mask.h"
#include "casu_utils.h"
#include "casu_mods.h"

/* Structure definitions */

typedef struct {

    /* Input */

    float       thresh;

    /* Output */

    char        extname[11];
    float       readnoise;
    float       gain;
    float       covar;

} configstruct;

typedef struct {
    cpl_size          *labels;
    cpl_frameset      *darklist;
    cpl_frameset      *flatlist;
    casu_fits         *darkim1;
    casu_fits         *darkim2;
    casu_fits         *flatim1;
    casu_fits         *flatim2;
    casu_mask         *master_mask;
    cpl_table         *tab;
} memstruct;

static char hawki_detector_noise_description[] =
"hawki_detector_noise -- HAWKI recipe to find readnoise and gain.\n\n"
"Compute the readnoise and gain for the hawki detectors using two\n"
"dark frames and two flat field frames. Both must have been exposed\n"
"with NDIT=1\n"
"The program accepts the following files in the SOF:\n\n"
"    Tag                   Description\n"
"    -----------------------------------------------------------------------\n"
"    %-21s A list of raw dark images\n"
"    %-21s A list of raw flat images\n"
"    %-21s Optional master bad pixel mask or\n"
"    %-21s Optional master confidence map\n"
"If a master bad pixel mask or confidence map is included then the"
"bad pixels will be flagged during the statistical analysis\n";

/* Function prototypes */

static int hawki_detector_noise_create(cpl_plugin *);
static int hawki_detector_noise_exec(cpl_plugin *);
static int hawki_detector_noise_destroy(cpl_plugin *);
static int hawki_detector_noise(cpl_parameterlist *, cpl_frameset *) ;
static int hawki_detector_noise_save(cpl_frameset *framelist, 
                                     cpl_parameterlist *parlist, 
                                     configstruct *cs, memstruct *ps,
                                     int nexten, cpl_propertylist *phu,
                                     cpl_frame **product_frame);
static void hawki_detector_noise_init(memstruct *ps);
static void hawki_detector_noise_tidy(memstruct *ps, int level);
static void hawki_detector_noise_autocorr(float *data, unsigned char *bpm, 
                                          int nx, int ny, float sig, 
                                          float *covar);
static cpl_error_code hawki_setqc_detector_noise(const cpl_table *tab,
                                                 cpl_propertylist * qclist);
/**@{*/

/**

    \ingroup recipelist
    \defgroup hawki_detector_noise hawki_detector_noise
    \brief Compute the detector readnoise, gain and covariance for HAWKI

    \par Name: 
        hawki_detector_noise
    \par Purpose: 
        Compute the detector readnoise, gain and covariance for HAWKI 
    \par Description: 
        A pair of dark frames and a pair of flat field frames are used to
        work out the readnoise, gain and covariance of the HAWKI detectors. 
    \par Language:
        C
    \par Parameters:
        - \b thresh (float): The rejection threshold in numbers of sigmas.
    \par Input File Types:
        The following list gives the file types that can appear in the SOF
        file. The word in bold is the DO CATG value.
        - \b DARK (required): A list of raw dark images. Only 2 will be used
        - \b FLAT_TWILIGHT (required): A list of raw flat images. Only 2 
             will be used
        - \b MASTER_BPM or \b MASTER_CONF (optional): If either is given, then
             it will be used to mask out bad pixels during and statistical 
             analysis that is done.
    \par Output Products:
        The following list gives the output data products that are generated
        by this recipe. The word in bold gives the PRO CATG keyword value for
        each product:
        - The output readgain table (\b MASTER_READGAIN)
    \par Output QC Parameters:
        None
    \par Notes
        None.
    \par Fatal Error Conditions:
        - NULL input frameset
        - Input frameset headers incorrect meaning that RAW and CALIB frame
          can't be distinguished
        - No dark or flat frames in the input frameset
        - Unable to save data products
    \par Non-Fatal Error Conditions:
        - No master bad pixel or confidence map. All pixels considered to be 
          good.
    \par Conditions Leading To Dummy Products:
        - Image extensions wouldn't load.
        - The detector for the current image extension is flagged dead
    \par Author:
        Jim Lewis, CASU
    \par Code Reference: 
        hawki_detector_noise.c

*/


/* Function code */


/*---------------------------------------------------------------------------*/
/**
  @brief    Build the list of available plugins, for this module.
  @param    list    the plugin list
  @return   0 if everything is ok

  This function is exported.
 */
/*---------------------------------------------------------------------------*/

int cpl_plugin_get_info(cpl_pluginlist *list) {
    cpl_recipe  *recipe = cpl_calloc(1,sizeof(*recipe));
    cpl_plugin  *plugin = &recipe->interface;
    char alldesc[SZ_ALLDESC];
    (void)snprintf(alldesc,SZ_ALLDESC,hawki_detector_noise_description,
                   HAWKI_DARK_RAW,HAWKI_TWI_RAW,HAWKI_CAL_BPM,
                   HAWKI_CAL_CONF);

    cpl_plugin_init(plugin,
                    CPL_PLUGIN_API,
                    HAWKI_BINARY_VERSION,
                    CPL_PLUGIN_TYPE_RECIPE,
                    "hawki_detector_noise",
                    "HAWKI detector noise recipe",
                    alldesc,
                    "Jim Lewis",
                    "jrl@ast.cam.ac.uk",
                    hawki_get_license(),
                    hawki_detector_noise_create,
                    hawki_detector_noise_exec,
                    hawki_detector_noise_destroy);

    cpl_pluginlist_append(list,plugin);

    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Setup the recipe options
  @param    plugin  the plugin
  @return   0 if everything is ok

  Create the recipe instance and make it available to the application using the
  interface.
 */
/*---------------------------------------------------------------------------*/

static int hawki_detector_noise_create(cpl_plugin *plugin) {
    cpl_recipe      *recipe;
    cpl_parameter   *p;

    /* Get the recipe out of the plugin */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin;
    else 
        return(-1);

    /* Create the parameters list in the cpl_recipe object */

    recipe->parameters = cpl_parameterlist_new();

    /* Rejection threshold */

    p = cpl_parameter_new_range("hawki.hawki_detector_noise.thresh",
                                CPL_TYPE_DOUBLE,
                                "Rejection threshold in sigma above background",
                                "hawki.hawki_detector_noise",5.0,1.0e-6,
                                1.0e10);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"thresh");
    cpl_parameterlist_append(recipe->parameters,p);

    /* Get out of here */

    return(0);
}
    
    
/*---------------------------------------------------------------------------*/
/**
  @brief    Execute the plugin instance given by the interface
  @param    plugin  the plugin
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_detector_noise_exec(cpl_plugin *plugin) {
    cpl_recipe  *recipe;

    /* Get the recipe out of the plugin */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin;
    else 
        return(-1);

    return(hawki_detector_noise(recipe->parameters,recipe->frames));
}
                                
/*---------------------------------------------------------------------------*/
/**
  @brief    Destroy what has been created by the 'create' function
  @param    plugin  the plugin
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_detector_noise_destroy(cpl_plugin *plugin) {
    cpl_recipe *recipe ;

    /* Get the recipe out of the plugin */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin;
    else 
        return(-1);

    cpl_parameterlist_delete(recipe->parameters);
    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    parlist     the parameters list
  @param    framelist   the frames list
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_detector_noise(cpl_parameterlist *parlist, 
                                cpl_frameset *framelist) {
    const char *fctid="hawki_detector_noise";
    cpl_parameter *p;
    cpl_propertylist *phu;
    cpl_frame *prod;
    memstruct ps;
    configstruct cs;
    int j,live,nx,ny,npts,retval,ndarks,nflats,i,status,nbsize=64;
    cpl_size nlab;
    unsigned char *bpm;
    cpl_binary *bpmcpl;
    float *datadark1,*datadark2,*dataflat1,*dataflat2,meandark1,sigdark;
    float lcut,hcut,meandark2,meanflat1,sigflat,meanflat2,meandarkdiff;
    float sigdarkdiff,meanflatdiff,sigflatdiff,counts,avbackdark,avbackflat;
    float *skymap,covar;

    /* Check validity of input frameset */

    if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
        cpl_msg_error(fctid,"Input framelist NULL or has no input data");
        return(-1);
    }

    /* Initialise some things */

    hawki_detector_noise_init(&ps);

    /* Get the parameters */

    p = cpl_parameterlist_find(parlist,"hawki.hawki_detector_noise.thresh");
    cs.thresh = (float)cpl_parameter_get_double(p);

    /* Sort out raw from calib frames */

    if (hawki_dfs_set_groups(framelist) != CASU_OK) {
        cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
        hawki_detector_noise_tidy(&ps,0);
        return(-1);
    }

    /* Get a list of the frame labels */ 

    if ((ps.labels = cpl_frameset_labelise(framelist,casu_compare_tags,
                                           &nlab)) == NULL) {
        cpl_msg_error(fctid,"Cannot labelise the input frames");
        hawki_detector_noise_tidy(&ps,0);
        return(-1);
    }

    /* Get the dark frames */

    if ((ps.darklist = casu_frameset_subgroup(framelist,ps.labels,nlab,
                                              HAWKI_DARK_RAW)) == NULL) {
        cpl_msg_error(fctid,"Cannot find dark frames in input frameset");
        hawki_detector_noise_tidy(&ps,0);
        return(-1);
    }
    ndarks = cpl_frameset_get_size(ps.darklist);
    if (ndarks < 2) {
        cpl_msg_error(fctid,"Only %d dark images. Need at least 2",
                      ndarks);
        hawki_detector_noise_tidy(&ps,0);
        return(-1);
    }

    /* Get the flat frames */

    if ((ps.flatlist = casu_frameset_subgroup(framelist,ps.labels,nlab,
                                              HAWKI_TWI_RAW)) == NULL) {
        cpl_msg_error(fctid,"Cannot find flat frames in input frameset");
        hawki_detector_noise_tidy(&ps,0);
        return(-1);
    }
    nflats = cpl_frameset_get_size(ps.flatlist);
    if (nflats < 2) {
        cpl_msg_error(fctid,"Only %d flat images. Need at least 2",
                      nflats);
        hawki_detector_noise_tidy(&ps,0);
        return(-1);
    }

    /* Check to see if there is a master bad pixel map. If there isn't one 
       then look for a confidence map */

    ps.master_mask = casu_mask_define(framelist,ps.labels,nlab,HAWKI_CAL_CONF,
                                      HAWKI_CAL_BPM);

    /* Now loop for all the extensions... */

    for (j = 1; j <= HAWKI_NEXTN; j++) {
        cs.readnoise = 0.0;
        cs.gain = 0.0;
        cs.covar = 0.0;

        /* Load up the images. If they won't load then signal an error,
           and save dummy results. */

        ps.darkim1 = casu_fits_load(cpl_frameset_get_position(ps.darklist,0),
                                    CPL_TYPE_FLOAT,j);
        ps.darkim2 = casu_fits_load(cpl_frameset_get_position(ps.darklist,1),
                                    CPL_TYPE_FLOAT,j);
        ps.flatim1 = casu_fits_load(cpl_frameset_get_position(ps.flatlist,0),
                                    CPL_TYPE_FLOAT,j);
        ps.flatim2 = casu_fits_load(cpl_frameset_get_position(ps.flatlist,1),
                                    CPL_TYPE_FLOAT,j);
        phu = NULL;
        if (ps.darkim1 != NULL) {
            snprintf(cs.extname,11,"%s",
                     cpl_propertylist_get_string(casu_fits_get_ehu(ps.darkim1),
                                                 "EXTNAME"));
            if (j == 1)
                phu = cpl_propertylist_duplicate(casu_fits_get_phu(ps.darkim1));
        } else if (ps.darkim2 != NULL) {
            snprintf(cs.extname,11,"%s",
                     cpl_propertylist_get_string(casu_fits_get_ehu(ps.darkim2),
                                                 "EXTNAME"));

            if (j == 1) 
                phu = cpl_propertylist_duplicate(casu_fits_get_phu(ps.darkim2));
        } else if (ps.flatim1 != NULL) {
            snprintf(cs.extname,11,"%s",
                     cpl_propertylist_get_string(casu_fits_get_ehu(ps.flatim1),
                                                 "EXTNAME"));
            if (j == 1) 
                phu = cpl_propertylist_duplicate(casu_fits_get_phu(ps.flatim1));
        } else if (ps.flatim2 != NULL) {
            snprintf(cs.extname,11,"%s",
                     cpl_propertylist_get_string(casu_fits_get_ehu(ps.flatim2),
                                                 "EXTNAME"));
            if (j == 1)
                phu = cpl_propertylist_duplicate(casu_fits_get_phu(ps.flatim2));
        } else {
            snprintf(cs.extname,11,"%s","unknown");
            if (j == 1)
                phu = cpl_propertylist_new();
        }
        if (ps.darkim1 == NULL || ps.darkim2 == NULL || ps.flatim1 == NULL ||
            ps.flatim2 == NULL) {
            cpl_error_reset();
            cpl_msg_error(fctid,
                          "Extension %" CPL_SIZE_FORMAT " image wouldn't load",
                          (cpl_size)j);
            hawki_detector_noise_save(framelist,parlist,&cs,&ps,j,phu,&prod);
            hawki_detector_noise_tidy(&ps,1);
            freepropertylist(phu);
            continue;
        }

        /* Presumably if one of these is flagged as a dead detector, then
           it holds true for all of them. So check the first dark to see if
           the detector is live */

        hawki_pfits_get_detlive(casu_fits_get_ehu(ps.darkim1),&live);
        if (! live) {
            cpl_error_reset();
            cpl_msg_info(fctid,"Detector flagged dead %" CPL_SIZE_FORMAT,
                         (cpl_size)j);
            hawki_detector_noise_save(framelist,parlist,&cs,&ps,j,phu,&prod);
            hawki_detector_noise_tidy(&ps,1);
            freepropertylist(phu);
            continue;            
        }        

        /* Load the master mask extension */

        nx = (int)cpl_image_get_size_x(casu_fits_get_image(ps.darkim1));
        ny = (int)cpl_image_get_size_y(casu_fits_get_image(ps.darkim1));
        npts = nx*ny;
        retval = casu_mask_load(ps.master_mask,j,nx,ny);
        if (retval == CASU_FATAL) {
            cpl_msg_info(fctid,
                         "Unable to load mask image %s[%" CPL_SIZE_FORMAT "]",
                         casu_mask_get_filename(ps.master_mask),(cpl_size)j);
            cpl_msg_info(fctid,"Forcing all pixels to be good from now on");
            casu_mask_force(ps.master_mask,nx,ny);
        }
        bpm = casu_mask_get_data(ps.master_mask);

        /* Do the statistical computations now... */

        cpl_msg_info(fctid,"Doing computation for extension %" CPL_SIZE_FORMAT,
                     (cpl_size)j);
        datadark1 = cpl_image_get_data_float(casu_fits_get_image(ps.darkim1));
        datadark2 = cpl_image_get_data_float(casu_fits_get_image(ps.darkim2));
        dataflat1 = cpl_image_get_data_float(casu_fits_get_image(ps.flatim1));
        dataflat2 = cpl_image_get_data_float(casu_fits_get_image(ps.flatim2));
        casu_medmad(datadark1,bpm,npts,&meandark1,&sigdark);
        sigdark *= 1.48;
        lcut = meandark1 - cs.thresh*sigdark;
        hcut = meandark1 + cs.thresh*sigdark;
        casu_medmadcut(datadark1,bpm,npts,lcut,hcut,&meandark1,&sigdark);
        casu_medmad(datadark2,bpm,npts,&meandark2,&sigdark);
        sigdark *= 1.48;
        lcut = meandark2 - cs.thresh*sigdark;
        hcut = meandark2 + cs.thresh*sigdark;
        casu_medmadcut(datadark2,bpm,npts,lcut,hcut,&meandark2,&sigdark);
        casu_medmad(dataflat1,bpm,npts,&meanflat1,&sigflat);
        sigflat *= 1.48;
        lcut = meanflat1 - cs.thresh*sigflat;
        hcut = meanflat1 + cs.thresh*sigflat;
        casu_medmadcut(dataflat1,bpm,npts,lcut,hcut,&meanflat1,&sigflat);
        casu_medmad(dataflat2,bpm,npts,&meanflat2,&sigflat);
        sigflat *= 1.48;
        lcut = meanflat2 - cs.thresh*sigflat;
        hcut = meanflat2 + cs.thresh*sigflat;
        casu_medmadcut(dataflat2,bpm,npts,lcut,hcut,&meanflat2,&sigflat);
        
        /* Form the difference images */

        cpl_image_subtract(casu_fits_get_image(ps.darkim1),
                           casu_fits_get_image(ps.darkim2));
        cpl_image_subtract(casu_fits_get_image(ps.flatim1),
                           casu_fits_get_image(ps.flatim2));

        /* Remove any residual large scale structure in both */

        bpmcpl = cpl_malloc(npts*sizeof(cpl_binary));
        for (i = 0; i < npts; i++)
            bpmcpl[i] = (cpl_binary)bpm[i];
        status = CASU_OK;
        casu_backmap(datadark1,bpmcpl,nx,ny,nbsize,&avbackdark,&skymap,&status);
        for (i = 0; i < npts; i++) 
            datadark1[i] -= skymap[i];
        freespace(skymap);
        casu_backmap(dataflat1,bpmcpl,nx,ny,nbsize,&avbackflat,&skymap,&status);
        for (i = 0; i < npts; i++) 
            dataflat1[i] -= skymap[i];
        freespace(skymap);
        freespace(bpmcpl);

        /* Do some more stats */

        casu_medmad(datadark1,bpm,npts,&meandarkdiff,&sigdarkdiff);
        sigdarkdiff *= 1.48;
        lcut = meandarkdiff - cs.thresh*sigdarkdiff;
        hcut = meandarkdiff + cs.thresh*sigdarkdiff;
        casu_medmadcut(datadark1,bpm,npts,lcut,hcut,&meandarkdiff,
                       &sigdarkdiff);
        sigdarkdiff *= 1.48;
        casu_medmad(dataflat1,bpm,npts,&meanflatdiff,&sigflatdiff);
        sigflatdiff *= 1.48;
        lcut = meanflatdiff - cs.thresh*sigflatdiff;
        hcut = meanflatdiff + cs.thresh*sigflatdiff;
        casu_medmadcut(dataflat1,bpm,npts,lcut,hcut,&meanflatdiff,
                       &sigflatdiff);
        sigflatdiff *= 1.48;

        /* Now remove the DC level in the flat difference image */

        for (i = 0; i < npts; i++)
            dataflat1[i] -= meanflatdiff;

        /* Do a autocorrelation and find out what the covariance is */
        
        hawki_detector_noise_autocorr(dataflat1,bpm,nx,ny,sigflatdiff,&covar);

        /* Now do the computation */

        counts = meanflat1 + meanflat2 - meandark1 - meandark2;
        cs.gain = counts/(sigflatdiff*sigflatdiff - sigdarkdiff*sigdarkdiff);
        cs.covar = covar;
        cs.readnoise = sigdarkdiff/sqrt(2.0);

        /* Save the result */

        hawki_detector_noise_save(framelist,parlist,&cs,&ps,j,phu,&prod);
        hawki_detector_noise_tidy(&ps,1);
        freepropertylist(phu);
    }
    hawki_detector_noise_tidy(&ps,0);
    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Do a 3x3 autocorrelation of an image
  @param    data         The input map
  @param    bpm          The input bad pixel mask
  @param    nx           The input x length of the map
  @param    ny           The input y length of the map
  @param    sig          The background sigma used for clipping
  @param    covar        The output covariance
  @return   nothing
 */
/*---------------------------------------------------------------------------*/

static void hawki_detector_noise_autocorr(float *data, unsigned char *bpm, 
                                          int nx, int ny, float sig, 
                                          float *covar) {
    int i,j,ioff,joff,n,ist,jst,ifn,jfn,ind,i1,j1,indoff;
    float clip,sum,av,acor[3][3],renorm;

    /* Set a clipping threshold */

    clip = 3.0*sig;

    /* Start by getting the average in the middle half of the array */

    ist = nx/4;
    ifn = 3*nx/4;
    jst = ny/4;
    jfn = 3*ny/4;
    n = 0;
    sum = 0.0;
    for (j = jst; j < jfn; j++) {
        for (i = ist; i < ifn; i++) {
            ind = j*ny + i;
            if (fabs(data[ind]) < clip && ! bpm[ind]) {
                n++;
                sum += data[ind];
            }
        }
    }
    av = sum/(float)n;
    
    /* Clear the auto-correlation array */

    for (i = 0; i < 3; i++)
        for (j = 0; j < 3; j++)
            acor[i][j] = 0.0;

    /* Now do the auto-correlation */

    for (j1 = 0; j1 < 3; j1++) {
        joff = -1 + j1;
        for (i1 = 0; i1 < 3; i1++) {
            ioff = -1 + i1;
            n = 0;
            sum = 0.0;
            for (j = jst; j < jfn; j++) {
                for (i = ist; i < ifn; i++) {
                    ind = j*nx + i;
                    indoff = (j + joff)*nx + (i + ioff);
                    if (fabs(data[ind]) < clip && ! bpm[ind] &&
                        fabs(data[indoff]) < clip && ! bpm[indoff]) {
                        n++;
                        sum += (data[ind] - av)*(data[indoff] - av);
                    }
                }
            }
            acor[i1][j1] = sum/(float)n;
        }
    }
    
    /* Now renormalise by the central value */

    renorm = acor[1][1];
    sum = 0.0;
    for (j = 0; j < 3; j++)
        for (i = 0; i < 3; i++)
            sum += acor[i][j]/renorm;

    /* Get out of here */

    *covar = sum;
}


/*---------------------------------------------------------------------------*/
/**
  @brief    The recipe data products are saved here
  @param    framelist    The input frame list
  @param    parlist      The input recipe parameter list
  @param    cs           The configuration structure defined in the main routine
  @param    ps           The memory structure defined in the main routine
  @param    nexten       The number of the extension
  @param    phu          The propertylist for the phu
  @param    product_frame  The frame for the output readgain product
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_detector_noise_save(cpl_frameset *framelist, 
                                     cpl_parameterlist *parlist, 
                                     configstruct *cs, memstruct *ps,
                                     int nexten, cpl_propertylist *phu,
                                     cpl_frame **product_frame) {
    const char *recipeid = "hawki_detector_noise";
    const char *fname = "readgain.fits";
    const char *fctid = "hawki_detector_noise_save";
    cpl_table *tab;
    cpl_propertylist *p;

    /* If we need to make a PHU then do that now. */

    if (nexten == 1) {

        /* Create a new product frame object and define some tags */

        *product_frame = cpl_frame_new();
        cpl_frame_set_filename(*product_frame,fname);
        cpl_frame_set_tag(*product_frame,HAWKI_PRO_READGAIN);
        cpl_frame_set_type(*product_frame,CPL_FRAME_TYPE_IMAGE);
        cpl_frame_set_group(*product_frame,CPL_FRAME_GROUP_PRODUCT);
        cpl_frame_set_level(*product_frame,CPL_FRAME_LEVEL_FINAL);

        /* Set the header for the primary */

        hawki_dfs_set_product_primary_header(phu,*product_frame,
                                             framelist,parlist,
                                             recipeid,"PRO-1.15",
                                             NULL,0);

        /* 'Save' the PHU image */                         

        if (cpl_image_save(NULL,fname,CPL_TYPE_UCHAR,phu,CPL_IO_DEFAULT) != 
            CPL_ERROR_NONE) {
            cpl_msg_error(fctid,"Cannot save product PHU");
            cpl_frame_delete(*product_frame);
            return(-1);
        }
        cpl_frameset_insert(framelist,*product_frame);

        /* Create the table */

        tab = cpl_table_new(4);
        cpl_table_new_column(tab,"EXTNAME",CPL_TYPE_STRING);
        cpl_table_new_column(tab,"READNOISE",CPL_TYPE_FLOAT);
        cpl_table_set_column_unit(tab,"READNOISE","ADUs");
        cpl_table_new_column(tab,"GAIN",CPL_TYPE_FLOAT);
        cpl_table_set_column_unit(tab,"GAIN","e-/ADU");
        cpl_table_new_column(tab,"COVAR",CPL_TYPE_FLOAT);
        cpl_table_set_column_unit(tab,"COVAR","");
        ps->tab = tab;
        
    }

    /* Fill in the most recent values */

    cpl_table_set_string(ps->tab,"EXTNAME",(cpl_size)(nexten-1),cs->extname);
    cpl_table_set_float(ps->tab,"READNOISE",(cpl_size)(nexten-1),cs->readnoise);
    cpl_table_set_float(ps->tab,"GAIN",(cpl_size)(nexten-1),cs->gain);
    cpl_table_set_float(ps->tab,"COVAR",(cpl_size)(nexten-1),cs->covar);

    /* If this is the last extension to be done, then save the table */

    if (nexten == HAWKI_NEXTN) {
        p = cpl_propertylist_new();
        cpl_propertylist_append_bool(p,"EXTNAME",0);
        cpl_table_sort(ps->tab,p);
        cpl_propertylist_delete(p);
        p = cpl_propertylist_new();

        /* Add QC parameters from the table to the header */
        if (hawki_setqc_detector_noise(ps->tab, p) != CPL_ERROR_NONE) {
            cpl_msg_error(fctid,"Cannot extract QC paramters from the table");
            cpl_propertylist_delete(p);
            return(-1);
        }

        hawki_dfs_set_product_exten_header(p,*product_frame,
                                           framelist,parlist,
                                           recipeid,
                                           "PRO-1.15",NULL);
        if (cpl_table_save(ps->tab,NULL,p,fname,CPL_IO_EXTEND) != 
            CPL_ERROR_NONE) {
            cpl_msg_error(fctid,"Cannot save product table extension");
            cpl_propertylist_delete(p);
            return(-1);
        }
        cpl_propertylist_delete(p);
    }

    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Initialise the pointers in the memory structure
  @param    ps           The memory structure defined in the main routine
 */
/*---------------------------------------------------------------------------*/

static void hawki_detector_noise_init(memstruct *ps) {
    ps->labels = NULL;
    ps->darklist = NULL;
    ps->flatlist = NULL;
    ps->darkim1 = NULL;
    ps->darkim2 = NULL;
    ps->flatim1 = NULL;
    ps->flatim2 = NULL;
    ps->master_mask = NULL;
    ps->tab = NULL;
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Free any allocated workspace in the memory structure
  @param    ps           The memory structure defined in the main routine
  @param    level        The level of tidying. 
 */
/*---------------------------------------------------------------------------*/

static void hawki_detector_noise_tidy(memstruct *ps, int level) {
    
    freefits(ps->darkim1);
    freefits(ps->darkim2);
    freefits(ps->flatim1);
    freefits(ps->flatim2);

    if (level == 1)
        return;
    freespace(ps->labels);
    freeframeset(ps->darklist);
    freeframeset(ps->flatlist);
    freemask(ps->master_mask);
    freetable(ps->tab);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Adds qc values taken from the table to the propertylist
  @param    tab    (in)       The table with the calculated values
  @param    qclist (out)      The quality control propertylist
 */
/*---------------------------------------------------------------------------*/

static cpl_error_code hawki_setqc_detector_noise(const cpl_table *tab,
                                                 cpl_propertylist * qclist) {

    cpl_error_ensure(tab != NULL, CPL_ERROR_NULL_INPUT,
                    return CPL_ERROR_NULL_INPUT, "Table is NULL");
    cpl_error_ensure(qclist != NULL, CPL_ERROR_NULL_INPUT,
                    return CPL_ERROR_NULL_INPUT, "QC list is NULL");

    float readnoise = 0.;
    float gain = 0.;
    const char * extname = NULL;
    char * qcname = NULL;

    cpl_size nrow = cpl_table_get_nrow(tab);

    if (cpl_table_has_column(tab, "EXTNAME") &&
                    cpl_table_has_column(tab, "READNOISE") &&
                    cpl_table_has_column(tab, "GAIN")){

        /* Extract information from the table and write QC parameter */
        for (cpl_size i = 0; i < nrow; i++) {
            readnoise = cpl_table_get_float(tab, "READNOISE", i, NULL);
            gain = cpl_table_get_float(tab, "GAIN", i, NULL);
            extname = cpl_table_get_string(tab, "EXTNAME", i);

            /* Replace the "." with a "-" to obay DICD rules */
            char * extname_mod = cpl_strdup(extname);
            for (size_t index = 0; index < strlen(extname_mod); index++)
            {
                if(extname_mod[index] == '.') {
                    extname_mod[index] = '-';
                }
            }

            qcname = cpl_sprintf("ESO QC RON %s", extname_mod);
            cpl_propertylist_update_float(qclist, qcname, readnoise);
            cpl_propertylist_set_comment(qclist,qcname,"Read-out noise in ADU");
            cpl_free(qcname);

            qcname = cpl_sprintf("ESO QC GAIN %s", extname_mod);
            cpl_propertylist_update_float(qclist, qcname, gain);
            cpl_propertylist_set_comment(qclist,qcname,"Gain in e-/ADU");
            cpl_free(qcname);
            cpl_free(extname_mod);

        }
    }

    return cpl_error_get_code() ;
}

/**@}*/
/*

$Log: hawki_detector_noise.c,v $
Revision 1.6  2015/09/11 09:31:20  jim
Changed parameter declarations from _value to _range where appropriate

Revision 1.5  2015/08/07 13:07:06  jim
Fixed copyright to ESO

Revision 1.4  2015/08/06 05:32:17  jim
A few fixes to get rid of compiler complaints

Revision 1.3  2015/04/30 12:09:44  jim
Added autocorrelation

Revision 1.2  2015/03/12 09:16:51  jim
Modified to remove some compiler moans

Revision 1.1  2015/01/29 12:01:28  jim
New routine


*/
