/* $Id: $
 *
 * This file is part of the HAWKI Pipeline
 * Copyright (C) 2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: $
 * $Date: $
 * $Revision: $
 * $Name:  $
 */

/* Includes */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <libgen.h>
#include <cpl.h>
#include <math.h>

#include "hawki_utils.h"
#include "hawki_pfits.h"
#include "hawki_dfs.h"
#include "hawki_var.h"
#include "casu_utils.h"
#include "casu_mask.h"
#include "casu_mods.h"
#include "casu_fits.h"
#include "casu_stats.h"

#define  freepolynomial(_p) if (_p != NULL) {cpl_polynomial_delete(_p); _p = NULL;}

#define HAWKI_RECIPENAME "hawki_linearity_analyse"

typedef struct {
    int    order;
    int    fitzero; 
    float  rchithr;
    float  satlim;
    float  reflevel;
} configstruct;

typedef struct {
    cpl_frameset   *darks;
    cpl_frameset   *domes;
    int            ndarks;
    int            ndomes;
    float          dit;
    int            ndit;
    float          mean_flat;
    float          med_flat;
} ddgrp;

typedef struct {

    /* Level 0 stuff */

    cpl_size       *labels;
    cpl_frameset   *darks;
    cpl_frameset   *flats;
    cpl_frame      *cube_product;
    cpl_frame      *map_product;
    cpl_polynomial *poly;
    cpl_polynomial *poly_lin;
    double         *expmat;
    double         *expmat2;
    double         *invals;
    double         *invals_sig;
    double         *resid;
    int            nddg;
    ddgrp          *ddg;

    /* Level 1 stuff */

    cpl_image      **curims;
    int            nproc;
    float          *res_cube;
    int            *res_im;
    double         *mean_fit;
} memstruct;

static int hawki_linearity_analyse_create(cpl_plugin *plugin);
static int hawki_linearity_analyse_exec(cpl_plugin *plugin);
static int hawki_linearity_analyse_destroy(cpl_plugin *plugin);
static int hawki_linearity_analyse(cpl_parameterlist *parlist,
                                   cpl_frameset *framelist);

static int hawki_lin_domedark_groups(memstruct *ps);
static int hawki_lin_save_cube(cpl_propertylist *phu, cpl_propertylist *ehu,
                               cpl_imagelist *imglist, cpl_frameset *framelist,
                               cpl_parameterlist *parlist, cpl_frame *template,
                               memstruct *ps, cpl_frame **product);
static int hawki_lin_save_map(cpl_propertylist *phu, cpl_propertylist *ehu,
                              cpl_image *img, cpl_frameset *framelist,
                              cpl_parameterlist *parlist, cpl_frame *template,
                              cpl_frame **product);
static int hawki_lin_save_tab(cpl_propertylist *phu, cpl_propertylist *ehu,
                              memstruct *ps, cpl_frameset *framelist,
                              cpl_parameterlist *parlist, cpl_frame *template,
                              float reflevel, int order, cpl_frame **product);
static void hawki_lin_init(memstruct *ps);
static void hawki_lin_tidy(memstruct *ps, int level);

static char hawki_linearity_analyse_description[] =
"hawki_linearity_analyse -- HAWKI detector linearity recipe.\n\n"
"Check the linearity of the HAWKI detectors on a pixel by pixel basis\n"
"The program accepts the following files in the SOF:\n\n"
"    Tag                   Description\n"
"    -----------------------------------------------------------------------\n"
"    %-21s A list of dark images \n"
"    %-21s A list of dome flat images\n"
"All of the above are required. The should be a dark frame for each DIT/NDIT"
"combination used for the dome flats\n"
"\n";

/**@{*/

/**
    \ingroup recipelist
    \defgroup hawki_linearity_analyse hawki_linearity_analyse
    \brief Look at the linearity of the HAWKI detectors

    \par Name: 
        hawki_linearity_analyse
    \par Purpose: 
        Work out the linearity of the HAWKI detectors.
    \par Description: 
        A series of flat fields with varying DITs are input and a power
        series is fit to the flux versus DIT for each pixel. 
    \par Language:
        C
    \par Parameters:
        - \b order (int): The order of the polynomial to be fit
        - \b fitzero (bool): If set, then the zeroth order term will also 
             be fit.
        - \b rchithr (float): A threshold reduced chi-squared value above which
             at fit is flagged as non-linear
        - \b satlim (float): A maximum flux in a flat above which we 
             consider the detector to be saturated
        - \b reflev (float): A reference flux level used to calculate the
             average non-linearity
    \par Input File Types:
        The following list gives the file types that can appear in the SOF
        file. The word in bold is the DO category value. There should be
        at least one dark exposure with the same DIT/NDIT combination for 
        each flat.
        - \b DARK_DETCHECK (required): A dark exposure
        - \b FLAT_LAMP_DETCHECK (required): A dome flat exposure
    \par Output Products:
        The following list gives the output data products that are generated
        by this recipe. The word in bold gives the DPR CATG keyword value for
        each product:
        - A data cube with the polynomial fit coefficients for each 
          pixel in each detector (\b COEFFS_CUBE)
        - A bad pixel mask for each detector marking where non-linear 
          pixels are located (\b BP_MAP_NL)
        - A table for each detector with mean/median linearity information
          (\b DET_LIN_INFO)
    \par Output QC Parameters:
        - \b LIN COEFi
             The mean value of the ith order coefficient over the whole
             detector
        - \b LIN COEFi ERR
             The standard deviation of the ith order coefficient over the
             whole detector
        - \b LIN EFF
             The effective non-linearity at the reference flux
        - \b LIN EFF FLUX
             The reference flux
        - \b LIN NNLPIX
             The number of non-linear pixels on the detector
        - \b LIN NNLFRAC
             The fraction of pixels on a detector that are non-linear
    \par Notes
        None.
    \par Fatal Error Conditions:
        - NULL input frameset
        - Input frameset headers incorrect meaning that RAW and CALIB frame
          can't be distinguished
        - No flat or dark frames in SOF
        - Unable to save data products
    \par Non-Fatal Error Conditions:
        - No dark frame available for flats with a given NDIT/DIT combination
    \par Conditions Leading To Dummy Products:
        None
    \par Author:
        Jim Lewis, CASU
    \par Code Reference: 
        hawki_linearity_analyse.c
*/
       

/* Function code */

/*---------------------------------------------------------------------------*/
/**
  @brief    Build the list of available plugins, for this module.
  @param    list    the plugin list
  @return   0 if everything is ok

  This function is exported.
 */
/*---------------------------------------------------------------------------*/

int cpl_plugin_get_info(cpl_pluginlist *list) {
    cpl_recipe  *recipe = cpl_calloc(1,sizeof(*recipe));
    cpl_plugin  *plugin = &recipe->interface;
    char alldesc[SZ_ALLDESC];
    (void)snprintf(alldesc,SZ_ALLDESC,
                   hawki_linearity_analyse_description,
                   HAWKI_LIN_DARK_DETCHECK,
                   HAWKI_LIN_FLAT_DETCHECK);

    cpl_plugin_init(plugin,
                    CPL_PLUGIN_API,
                    HAWKI_BINARY_VERSION,
                    CPL_PLUGIN_TYPE_RECIPE,
                    "hawki_linearity_analyse",
                    "HAWKI linearity recipe",
                    alldesc,
                    "Jim Lewis",
                    "jrl@ast.cam.ac.uk",
                    hawki_get_license(),
                    hawki_linearity_analyse_create,
                    hawki_linearity_analyse_exec,
                    hawki_linearity_analyse_destroy);

    cpl_pluginlist_append(list,plugin);

    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    Setup the recipe options
  @param    plugin  the plugin
  @return   0 if everything is ok

  Create the recipe instance and make it available to the application using the
  interface.
 */
/*---------------------------------------------------------------------------*/

static int hawki_linearity_analyse_create(cpl_plugin *plugin) {
    cpl_recipe      *recipe;
    cpl_parameter   *p;

    /* Get the recipe out of the plugin */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin;
    else 
        return(-1);

    /* Create the parameters list in the cpl_recipe object */

    recipe->parameters = cpl_parameterlist_new();

    /* Fitting order */

    p = cpl_parameter_new_value("hawki.hawki_linearity_analyse.order",
                                CPL_TYPE_INT,"Polynomial order to fit",
                                "hawki.hawki_linearity_analyse",3);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"order");
    cpl_parameterlist_append(recipe->parameters,p);

    /* Flag to fit the zeroth order term  */

    p = cpl_parameter_new_value("hawki.hawki_linearity_analyse.fitzero",
                                CPL_TYPE_BOOL,
                                "Fit zeroth order term?",
                                "hawki.hawki_linearity_analyse",FALSE);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"fitzero");
    cpl_parameterlist_append(recipe->parameters,p);

    /* A threshold reduced chi-squared value above which a pixel is 
       considered to be non-linear*/

    p = cpl_parameter_new_value("hawki.hawki_linearity_analyse.rchithr",
                                CPL_TYPE_DOUBLE,
                                "Maximum allowable value of reduced chisq?",
                                "hawki.hawki_linearity_analyse",5.0);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"rchithr");
    cpl_parameterlist_append(recipe->parameters,p);

    /* A maximum flux above which we consider the data to be saturated */

    p = cpl_parameter_new_value("hawki.hawki_linearity_analyse.satlim",
                                CPL_TYPE_DOUBLE,
                                "Saturation limit?",
                                "hawki.hawki_linearity_analyse",65535.0);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"satlim");
    cpl_parameterlist_append(recipe->parameters,p);

    /* A reference flux level where we check the mean nonlinearity */

    p = cpl_parameter_new_value("hawki.hawki_linearity_analyse.reflev",
                                CPL_TYPE_DOUBLE,
                                "Reference flux level for NL check",
                                "hawki.hawki_linearity_analyse",10000.0);
    cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"reflev");
    cpl_parameterlist_append(recipe->parameters,p);
        
    /* Get out of here */

    return(0);
}
    
/*---------------------------------------------------------------------------*/
/**
  @brief    Execute the plugin instance given by the interface
  @param    plugin  the plugin
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_linearity_analyse_exec(cpl_plugin *plugin) {
    cpl_recipe  *recipe;

    /* Get the recipe out of the plugin */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin;
    else 
        return(-1);

    return(hawki_linearity_analyse(recipe->parameters,recipe->frames));
}
                                
/*---------------------------------------------------------------------------*/
/**
  @brief    Destroy what has been created by the 'create' function
  @param    plugin  the plugin
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_linearity_analyse_destroy(cpl_plugin *plugin) {
    cpl_recipe *recipe ;

    /* Get the recipe out of the plugin */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
        recipe = (cpl_recipe *)plugin;
    else 
        return(-1);

    cpl_parameterlist_delete(recipe->parameters);
    return(0);
}

/*---------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    parlist     the parameters list
  @param    framelist   the frames list
  @return   0 if everything is ok
 */
/*---------------------------------------------------------------------------*/

static int hawki_linearity_analyse(cpl_parameterlist *parlist,
                                   cpl_frameset *framelist) {
    const char *fctid="hawki_linearity_analyse";
    configstruct cs;
    memstruct ps;
    cpl_size nlab,mindeg,maxdeg,power,maxdeg_lin=1;
    cpl_parameter *p;
    int nfail,ncol,n,i,j,k,nx,ny,npix,k1,k2,ind,isrej,ndof,ntot;
    cpl_frame *fr = NULL, *product_cube = NULL, *product_map = NULL;
    cpl_frame *product_tab = NULL;
    cpl_image *darkim,*im;
    float data,sig;
    float reads[] = {9.8,8.9,8.2,8.3}; /* ADUs */
    float gains[] = {2.1,2.2,2.1,2.5}; /* e-/ADU */
    double rchisq;
    cpl_imagelist *imglist;
    cpl_propertylist *phu,*ehu;
    cpl_vector *iv,*ivs,*ires;
    cpl_matrix *mat;

    /* Check validity of the input frameset */

    if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
        cpl_msg_error(fctid,"Input framelist NULL or has no input data");
        return(-1);
    }

    /* Initialise a few things */

    hawki_lin_init(&ps);

    /* Get the parameters */

    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_linearity_analyse.order");
    cs.order = cpl_parameter_get_int(p);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_linearity_analyse.fitzero");
    cs.fitzero = cpl_parameter_get_bool(p);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_linearity_analyse.rchithr");
    cs.rchithr = (float)cpl_parameter_get_double(p);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_linearity_analyse.satlim");
    cs.satlim = (float)cpl_parameter_get_double(p);
    p = cpl_parameterlist_find(parlist,
                               "hawki.hawki_linearity_analyse.reflev");
    cs.reflevel = (float)cpl_parameter_get_double(p);
    
    /* Sort out raw from calib frames */

    if (hawki_dfs_set_groups(framelist) != CASU_OK) {
        cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
        hawki_lin_tidy(&ps,0);
        return(-1);
    }

    /* Label the input frames */

    if ((ps.labels = cpl_frameset_labelise(framelist,casu_compare_tags,
                                           &nlab)) == NULL) {
        cpl_msg_error(fctid,"Cannot labelise the input frames");
        hawki_lin_tidy(&ps,0);
        return(-1);
    }

    /* Get the input flat frames */

    nfail = 0;
    if ((ps.flats = 
        casu_frameset_subgroup(framelist,ps.labels,nlab,
                               HAWKI_LIN_FLAT_DETCHECK)) == NULL) {
        cpl_msg_error(fctid,"No flat images to process!");
        hawki_lin_tidy(&ps,0);
        return(-1);
    }
    nfail += hawki_testfrms(ps.flats,HAWKI_NEXTN,1,0);

    /* Get the dark frames */
    
    if ((ps.darks = casu_frameset_subgroup(framelist,ps.labels,nlab,
                                           HAWKI_LIN_DARK_DETCHECK)) == NULL) {
        cpl_msg_error(fctid,"No dark images to process!");
        hawki_lin_tidy(&ps,0);
        return(-1);
    }
    nfail += hawki_testfrms(ps.darks,HAWKI_NEXTN,1,0);

    /* Ok if any of this failed, then get out of here. This makes it a bit 
       simpler later on since we now know that all of the specified files 
       have the correct number of extensions and will load properly */

    if (nfail > 0) {
        cpl_msg_error(fctid,
                      "There are %" CPL_SIZE_FORMAT " input file errors -- cannot continue",
                      (cpl_size)nfail);
        hawki_lin_tidy(&ps,0);
        return(-1);
    }

    /* Group the domes and darks by exposure time. Check to make sure there
       is a dark exposure for each NDIT/DIT available in the list of domes */

    if (hawki_lin_domedark_groups(&ps) != 0) {
        hawki_lin_tidy(&ps,0);
        return(-1);
    }
    
    /* See if the number of exposure times is too small for the order of the
       polynomial you want to fit. If it is, the adjust the order */

    if (ps.nddg < cs.order+1) {
        cpl_msg_warning(fctid,
                        "Number of exposure times is too small: %" CPL_SIZE_FORMAT ", order: %" CPL_SIZE_FORMAT "\nTaking fit down to order %" CPL_SIZE_FORMAT,
                        (cpl_size)(ps.nddg),(cpl_size)cs.order,
                        (cpl_size)(ps.nddg-1));
        cs.order = ps.nddg - 1;
    }

    /* Ok, let's get started. Set up the polynomial structures */

    ps.poly = cpl_polynomial_new(1);
    ps.poly_lin = cpl_polynomial_new(1);
    ncol = 0;
    for (i = 0; i < ps.nddg; i++)
        ncol += ps.ddg[i].ndomes;
    ps.expmat = cpl_calloc(ncol,sizeof(double));
    ps.expmat2 = cpl_calloc(ncol,sizeof(double));
    n = 0;
    for (i = 0; i < ps.nddg; i++) 
        for (j = 0; j < ps.ddg[i].ndomes; j++) 
            ps.expmat[n++] = (double)ps.ddg[i].dit;
    ps.invals = cpl_calloc(n,sizeof(double));
    ps.invals_sig = cpl_calloc(n,sizeof(double));
    ps.resid = cpl_calloc(n,sizeof(double));
    mindeg = (cs.fitzero ? 0 : 1);
    maxdeg = (cpl_size)cs.order;
    
    /* Get some space to hold the current load of images */

    ps.curims = cpl_malloc(n*sizeof(cpl_image *));
    ps.nproc = n;

    /* Loop for each extension */
    
    for (j = 1; j <= HAWKI_NEXTN; j++) {
        cpl_msg_info(fctid,"Working on extension %" CPL_SIZE_FORMAT,
                     (cpl_size)j);
        
        /* Start by loading the first dark image and all flat images from 
           each group. Dark correct the flats as you go along */

        n = 0;
        for (i = 0; i < ps.nddg; i++) {
            fr = cpl_frameset_get_position(ps.ddg[i].darks,0);
            darkim = cpl_image_load(cpl_frame_get_filename(fr),CPL_TYPE_FLOAT,
                                    0,j);
            for (k = 0; k < cpl_frameset_get_size(ps.ddg[i].domes); k++) {
                fr = cpl_frameset_get_position(ps.ddg[i].domes,k);
                ps.curims[n] = cpl_image_load(cpl_frame_get_filename(fr),
                                              CPL_TYPE_FLOAT,0,j);
                cpl_image_subtract(ps.curims[n],darkim);
                ps.ddg[i].mean_flat += cpl_image_get_mean(ps.curims[n]);
                ps.ddg[i].med_flat += cpl_image_get_median(ps.curims[n]);
                n++;
            }
            cpl_image_delete(darkim);
            ps.ddg[i].mean_flat /= (float)cpl_frameset_get_size(ps.ddg[i].domes);
            ps.ddg[i].med_flat /= (float)cpl_frameset_get_size(ps.ddg[i].domes);
        }

        /* OK, we now have a collection of dark corrected flat images
           for this extension. Start fitting for each pixel. (We're assuming
           that the user hasn't been silly enough to have images with 
           different axis lengths...) */

        nx = cpl_image_get_size_x(ps.curims[0]);
        ny = cpl_image_get_size_y(ps.curims[0]);
        npix = nx*ny;
        ps.res_cube = cpl_calloc(npix*(cs.order+1),sizeof(float));
        ps.res_im = cpl_calloc(npix,sizeof(int));
        ps.mean_fit = cpl_calloc(cs.order+1,sizeof(double));
        for (k = 0; k < npix; k++) {
            k2 = k/nx;
            k1 = k - k2*nx;
            ntot = 0;
            for (i = 0; i < n; i++) {
                data = cpl_image_get(ps.curims[i],(cpl_size)(k1+1),
                                     (cpl_size)(k2+1),&isrej);
                if (data < cs.satlim) {
                    ps.invals[ntot] = (double)data;
                    sig = sqrt(max(data/gains[j-1] + pow(reads[j-1],2.0),1.0));
                    ps.invals_sig[ntot] = (double)sig;
                    ps.expmat2[ntot] = ps.expmat[i];
                    ntot++;
                }
            }
            if (ntot >= maxdeg+1) {
                mat = cpl_matrix_wrap(1,ntot,ps.expmat2);
                iv = cpl_vector_wrap(ntot,ps.invals);
                ivs = cpl_vector_wrap(ntot,ps.invals_sig);
                ires = cpl_vector_wrap(ntot,ps.resid);
                cpl_polynomial_fit(ps.poly,mat,NULL,iv,NULL,
                                   CPL_FALSE,&mindeg,&maxdeg);
                cpl_polynomial_fit(ps.poly_lin,mat,NULL,iv,
                                   NULL,CPL_FALSE,&mindeg,&maxdeg_lin);
                cpl_vector_fill_polynomial_fit_residual(ires,iv,
                                                        NULL,ps.poly_lin,
                                                        mat,&rchisq);
                rchisq = 0.0;
                for (i = 0; i < ntot; i++) 
                    rchisq += pow(cpl_vector_get(ires,i)/cpl_vector_get(ivs,i),2.0);
                ndof = n - 1;
                if (cs.fitzero)
                    ndof--;
                rchisq /= (double)ndof;

                /* Load up the coefficients */
                
                for (i = 0; i < cs.order+1; i++) {
                    ind = i*npix + k2*nx + k1;
                    power = (cpl_size)i;
                    ps.res_cube[ind] = cpl_polynomial_get_coeff(ps.poly,&power);
                }
                cpl_matrix_unwrap(mat);
                cpl_vector_unwrap(iv);
                cpl_vector_unwrap(ivs);
                cpl_vector_unwrap(ires);

                /* If the reduced chi-squared of the linear fit surpasses the
                   threshold, then we assume it's not a linear pixel */
                
                if (rchisq > cs.rchithr)
                    ps.res_im[k2*nx+k1] = 1;
            } else {
                ps.res_im[k2*nx+k1] = 1;
            }
        }

        /* The only way to save a data cube is to make it out as an
           image list. So create that now */

        imglist = cpl_imagelist_new();
        for (i = 0; i <= cs.order; i++) {
            im = cpl_image_wrap_float((cpl_size)nx,(cpl_size)ny,
                                      ps.res_cube+i*nx*ny);
            cpl_imagelist_set(imglist,im,i);
        }

        /* Save the cube results */

        fr = cpl_frameset_get_position(ps.ddg[0].domes,0);
        if (j == 1) {
            phu = cpl_propertylist_load(cpl_frame_get_filename(fr),0);
            product_cube = NULL;
            product_map = NULL;
            product_tab = NULL;
        } else {
            phu = NULL;
        }
        ehu = cpl_propertylist_load(cpl_frame_get_filename(fr),j);
        if (hawki_lin_save_cube(phu,ehu,imglist,framelist,parlist,fr,
                                &ps,&product_cube) != CASU_OK) {
            freepropertylist(phu);
            freepropertylist(ehu);
            hawki_lin_tidy(&ps,0);
            return(-1);
        }

        /* Tidy up the image list and cube */

        for (i = 0; i <= cs.order; i++)
            cpl_image_unwrap(cpl_imagelist_get(imglist,(cpl_size)i));
        cpl_imagelist_unwrap(imglist);

        /* Save the non-linear pixel map */

        im = cpl_image_wrap_int(nx,ny,ps.res_im);
        if (hawki_lin_save_map(phu,ehu,im,framelist,parlist,fr,
                               &product_map) != CASU_OK) {
            freepropertylist(phu);
            freepropertylist(ehu);
            hawki_lin_tidy(&ps,0);
            return(-1);
        }
        cpl_image_unwrap(im);

        /* Create the output table */

        if (hawki_lin_save_tab(phu,ehu,&ps,framelist,parlist,fr,
                               cs.reflevel,cs.order,&product_tab) != CASU_OK) {
            freepropertylist(phu);
            freepropertylist(ehu);
            hawki_lin_tidy(&ps,0);
            return(-1);
        }        
        freepropertylist(phu);
        freepropertylist(ehu);

        /* Free up some space */

        hawki_lin_tidy(&ps,1);
    }
    
    /* Tidy up the rest and get out of here */

    hawki_lin_tidy(&ps,0);
    return(0);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_lin_save_tab
    \par Purpose:
        Save the mean linearity table
    \par Description:
        A table with mean linearity information is created for each
        detector.
    \par Language:
        C
    \param phu
        The primary header or NULL if this isn't the first extension
    \param ehu
        The extension header
    \param ps
        The memory structure with the results
    \param framelist
        The input file frameset
    \param parlist
        The input command line parameter list
    \param template
        A template header
    \param reflevel
        The reference level for the non-linearity estimate
    \param order
        The order of the polynomials that were fit
    \param product
        The output product frame
    \return
        A status value flagging the success or failure of the routine
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static int hawki_lin_save_tab(cpl_propertylist *phu, cpl_propertylist *ehu,
                              memstruct *ps, cpl_frameset *framelist,
                              cpl_parameterlist *parlist, cpl_frame *template,
                              float reflevel, int order, cpl_frame **product) {
    cpl_propertylist *plist;
    const char *fname = "lintab.fits";
    const char *fctid = "hawki_lin_save_cube";
    int i,j;
    cpl_size ii;
    cpl_table *tab;
    cpl_polynomial *poly;
    double nonlin,val;

    /* Start by creating the primary (if it doesn't already exist). Delete
       old versions. */
    
    if (phu != NULL) {
        if (access(fname,F_OK))
            remove(fname);

        /* Create the product frame object and define some tags */

        *product = cpl_frame_new();
        cpl_frame_set_filename(*product,fname);
        cpl_frame_set_tag(*product,HAWKI_LIN_TAB);
        cpl_frame_set_type(*product,CPL_FRAME_TYPE_IMAGE);
        cpl_frame_set_group(*product,CPL_FRAME_GROUP_PRODUCT);
        cpl_frame_set_level(*product,CPL_FRAME_LEVEL_FINAL);

        /* Set up the PHU header */

        plist = cpl_propertylist_duplicate(phu);
        hawki_dfs_set_product_primary_header(plist,*product,framelist,
                                             parlist,HAWKI_RECIPENAME,
                                             "PRO-1.15",template,1);

        /* Save the PHU image */

        if (cpl_image_save(NULL,fname,CPL_TYPE_UCHAR,plist,CPL_IO_DEFAULT) !=
            CPL_ERROR_NONE) {
            cpl_msg_error(fctid,"Cannot save table product PHU");
            cpl_frame_delete(*product);
            cpl_propertylist_delete(plist);
            return(CASU_FATAL);
        }
        cpl_propertylist_delete(plist);
        cpl_frameset_insert(framelist,*product);
    }

    /* Create the table from the info in the data structure */

    tab = cpl_table_new(ps->nddg);
    cpl_table_new_column(tab,"DIT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab,"MED",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab,"MEAN",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab,"MED_DIT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab,"MEAN_DIT",CPL_TYPE_DOUBLE);
    cpl_table_new_column(tab,"ADL",CPL_TYPE_DOUBLE);

    /* The DIT column */

    for (i = 0; i < ps->nddg; i++)
        cpl_table_set_double(tab,"DIT",(cpl_size)i,(double)ps->ddg[i].dit);

    /* The MED column */

    for (i = 0; i < ps->nddg; i++) 
        cpl_table_set_double(tab,"MED",(cpl_size)i,(double)ps->ddg[i].med_flat);

    /* The MEAN column */

    for (i = 0; i < ps->nddg; i++) 
        cpl_table_set_double(tab,"MEAN",(cpl_size)i,
                             (double)ps->ddg[i].mean_flat);
    
    /* The MED_DIT column */

    for (i = 0; i < ps->nddg; i++) 
        cpl_table_set_double(tab,"MED_DIT",(cpl_size)i,
                             (double)(ps->ddg[i].med_flat/ps->ddg[i].dit));

    /* The MEAN_DIT column */

    for (i = 0; i < ps->nddg; i++) 
        cpl_table_set_double(tab,"MEAN_DIT",(cpl_size)i,
                             (double)(ps->ddg[i].mean_flat/ps->ddg[i].dit));

    /* The ADL column (mean of MED_DIT column times DIT) */

    for (i = 0; i < ps->nddg; i++) 
        cpl_table_set_double(tab,"ADL",(cpl_size)i,
                             cpl_table_get_column_mean(tab,"MED_DIT")*ps->ddg[i].dit);

    /* Get the extension propertylist */

    plist = cpl_propertylist_duplicate(ehu);
    hawki_dfs_set_product_exten_header(plist,*product,framelist,parlist,
                                       HAWKI_RECIPENAME,"PRO-1.15",template);

    /* Work out the mean non-linearity */

    ps->mean_fit[0] -= reflevel;
    for (i = 2; i <= order; i++) {
        for (j = 0; j < i; j++)
            ps->mean_fit[i] /= ps->mean_fit[1];
    }
    ps->mean_fit[1] = 1.0;
    poly = cpl_polynomial_new(1);
    for (ii = 0; ii <= order; ii++)
        cpl_polynomial_set_coeff(poly,&ii,ps->mean_fit[ii]);
    cpl_polynomial_solve_1d(poly,reflevel,&val,1);
    cpl_polynomial_delete(poly);
    nonlin = (val - (double)reflevel)/(double)reflevel;
    cpl_propertylist_update_double(plist,"ESO QC LIN EFF",nonlin);
    cpl_propertylist_set_comment(plist,"ESO QC LIN EFF","Mean non-linearity");
    cpl_propertylist_update_double(plist,"ESO QC LIN EFF FLUX",reflevel);
    cpl_propertylist_set_comment(plist,"ESO QC LIN EFF FLUX",
                                 "Reference level for non-linearity");
    
    /* Now save the table */

    if (cpl_table_save(tab,NULL,plist,fname,CPL_IO_EXTEND) !=
        CPL_ERROR_NONE) {
        cpl_msg_error(fctid,"Cannot save table product extension");
        cpl_propertylist_delete(plist);
        return(CASU_FATAL);
    }
    
    /* Get out of here */

    cpl_propertylist_delete(plist);
    cpl_table_delete(tab);
    return(CASU_OK);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_lin_save_cube
    \par Purpose:
        Save the linearity coefficient cube
    \par Description:
        A file with the cube of linearity coefficients for each pixel in 
        each detector is saved.
    \par Language:
        C
    \param phu
        The primary header or NULL if this isn't the first extension
    \param ehu
        The extension header
    \param imglist
        The image list representation of the data cube
    \param framelist
        The input file frameset
    \param parlist
        The input command line parameter list
    \param template
        A template header
    \param ps
        The memory structure with dark/dome entries
    \param product
        The output product frame
    \return
        A status value flagging the success or failure of the routine
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static int hawki_lin_save_cube(cpl_propertylist *phu, cpl_propertylist *ehu,
                               cpl_imagelist *imglist, cpl_frameset *framelist,
                               cpl_parameterlist *parlist, cpl_frame *template,
                               memstruct *ps, cpl_frame **product) {
    cpl_propertylist *plist;
    const char *fname = "lincube.fits";
    const char *fctid = "hawki_lin_save_cube";
    char label[64];
    int i;
    double val;

    /* Start by creating the primary (if it doesn't already exist). Delete
       old versions. */
    
    if (phu != NULL) {
        if (access(fname,F_OK))
            remove(fname);

        /* Create the product frame object and define some tags */

        *product = cpl_frame_new();
        cpl_frame_set_filename(*product,fname);
        cpl_frame_set_tag(*product,HAWKI_LIN_COEFF_CUBE);
        cpl_frame_set_type(*product,CPL_FRAME_TYPE_IMAGE);
        cpl_frame_set_group(*product,CPL_FRAME_GROUP_PRODUCT);
        cpl_frame_set_level(*product,CPL_FRAME_LEVEL_FINAL);

        /* Set up the PHU header */

        plist = cpl_propertylist_duplicate(phu);
        hawki_dfs_set_product_primary_header(plist,*product,framelist,
                                             parlist,HAWKI_RECIPENAME,
                                             "PRO-1.15",template,1);

        /* Save the PHU image */

        if (cpl_image_save(NULL,fname,CPL_TYPE_UCHAR,plist,CPL_IO_DEFAULT) !=
            CPL_ERROR_NONE) {
            cpl_msg_error(fctid,"Cannot save cube product PHU");
            cpl_frame_delete(*product);
            cpl_propertylist_delete(plist);
            return(CASU_FATAL);
        }
        cpl_propertylist_delete(plist);
        cpl_frameset_insert(framelist,*product);
    }

    /* Get the extension propertylist */

    plist = cpl_propertylist_duplicate(ehu);
    hawki_dfs_set_product_exten_header(plist,*product,framelist,parlist,
                                       HAWKI_RECIPENAME,"PRO-1.15",template);

    /* Create some QC parameters */

    for (i = 0; i < cpl_imagelist_get_size(imglist); i++) {
        (void)snprintf(label,64,"ESO QC LIN COEF%d",i);
        val = cpl_image_get_median(cpl_imagelist_get(imglist,i));
        cpl_propertylist_update_double(plist,label,val);
        cpl_propertylist_set_comment(plist,label,"Median linearity coefficient");
        ps->mean_fit[i] = val;
        (void)snprintf(label,64,"ESO QC LIN COEF%d ERR",i);
        val = cpl_image_get_stdev(cpl_imagelist_get(imglist,i));
        cpl_propertylist_update_double(plist,label,val);
        cpl_propertylist_set_comment(plist,label,"Stddev linearity coefficient");
    }   
    
    /* Now save the cube */

    if (cpl_imagelist_save(imglist,fname,CPL_TYPE_FLOAT,plist,CPL_IO_EXTEND) !=
        CPL_ERROR_NONE) {
        cpl_msg_error(fctid,"Cannot save cube product extension");
        cpl_propertylist_delete(plist);
        return(CASU_FATAL);
    }
    
    /* Get out of here */

    cpl_propertylist_delete(plist);
    return(CASU_OK);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_lin_save_map
    \par Purpose:
        Save the non-linearity bad pixel map
    \par Description:
        An image for each detector is saved which marks out which detectors
        are non-linear
    \par Language:
        C
    \param phu
        The primary header or NULL if this isn't the first extension
    \param ehu
        The extension header
    \param img
        The image to be saved
    \param framelist
        The input file frameset
    \param parlist
        The input command line parameter list
    \param template
        A template header
    \param product
        The output product frame
    \return
        A status value flagging the success or failure of the routine
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static int hawki_lin_save_map(cpl_propertylist *phu, cpl_propertylist *ehu,
                              cpl_image *img, cpl_frameset *framelist,
                              cpl_parameterlist *parlist, cpl_frame *template,
                              cpl_frame **product) {
    cpl_propertylist *plist;
    const char *fname = "nonlinmap.fits";
    const char *fctid = "hawki_lin_save_map";
    int ntot;
    float frac;

    /* Start by creating the primary (if it doesn't already exist). Delete
       old versions. */
    
    if (phu != NULL) {
        if (access(fname,F_OK))
            remove(fname);

        /* Create the product frame object and define some tags */

        *product = cpl_frame_new();
        cpl_frame_set_filename(*product,fname);
        cpl_frame_set_tag(*product,HAWKI_LIN_NL_MAP);
        cpl_frame_set_type(*product,CPL_FRAME_TYPE_IMAGE);
        cpl_frame_set_group(*product,CPL_FRAME_GROUP_PRODUCT);
        cpl_frame_set_level(*product,CPL_FRAME_LEVEL_FINAL);

        /* Set up the PHU header */

        plist = cpl_propertylist_duplicate(phu);
        hawki_dfs_set_product_primary_header(plist,*product,framelist,
                                             parlist,HAWKI_RECIPENAME,
                                             "PRO-1.15",template,1);

        /* Save the PHU image */

        if (cpl_image_save(NULL,fname,CPL_TYPE_UCHAR,plist,CPL_IO_DEFAULT) !=
            CPL_ERROR_NONE) {
            cpl_msg_error(fctid,"Cannot save cube product PHU");
            cpl_frame_delete(*product);
            cpl_propertylist_delete(plist);
            return(CASU_FATAL);
        }
        cpl_propertylist_delete(plist);
        cpl_frameset_insert(framelist,*product);
    }

    /* Get the extension propertylist */

    plist = cpl_propertylist_duplicate(ehu);
    hawki_dfs_set_product_exten_header(plist,*product,framelist,parlist,
                                       HAWKI_RECIPENAME,"PRO-1.15",template);

    /* Set a few QC variables */

    ntot = (int)cpl_image_get_flux(img);
    frac = (float)ntot/(float)(cpl_image_get_size_x(img)*cpl_image_get_size_y(img));
    cpl_propertylist_update_int(plist,"ESO QC LIN NNLPIX",ntot);
    cpl_propertylist_set_comment(plist,"ESO QC LIN NNLPIX",
                                 "Number of non-linear pixels");
    cpl_propertylist_update_float(plist,"ESO QC LIN NNLFRAC",frac);
    cpl_propertylist_set_comment(plist,"ESO QC LIN NNLFRAC",
                                 "Fraction of non-linear pixels");
    
    /* Now save the cube */

    if (cpl_image_save(img,fname,CPL_TYPE_UCHAR,plist,CPL_IO_EXTEND) !=
        CPL_ERROR_NONE) {
        cpl_msg_error(fctid,"Cannot save cube product extension");
        cpl_propertylist_delete(plist);
        return(CASU_FATAL);
    }
    
    /* Get out of here */

    cpl_propertylist_delete(plist);
    return(CASU_OK);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_lin_domedark_groups
    \par Purpose:
        Set up groups of dome/dark exposures by NDIT/DIT
    \par Description:
        Groups of domes and darks with the same values of DIT/NDIT are
        collected together.
    \par Language:
        C
    \param ps
        The memory structure where the dome/dark information will be stored
    \return
        A status value flagging the success or failure of the routine
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static int hawki_lin_domedark_groups(memstruct *ps) {
    int i,j,found,ndit,nflats,ndarks;
    float dit;
    cpl_frame *frame;
    cpl_propertylist *plist;
    const char *fctid = "hawki_lin_domedark_groups";

    /* Start by getting the memory for the domedark groups */

    nflats = cpl_frameset_get_size(ps->flats);
    ndarks = cpl_frameset_get_size(ps->darks);
    ps->ddg = cpl_calloc(nflats,sizeof(ddgrp));
    ps->nddg = 0;

    /* Loop for each of the dome frames and get its exposure time. If this
       doesn't exist, then signal an error and go on */

    for (i = 0; i < nflats; i++) {
        frame = cpl_frameset_get_position(ps->flats,i);
        plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
        if (hawki_pfits_get_dit(plist,&dit) != CASU_OK) {
            cpl_msg_warning(fctid,"No dit time found in %s",
                            cpl_frame_get_filename(frame));
            cpl_propertylist_delete(plist);
            continue;
        }
        if (hawki_pfits_get_ndit(plist,&ndit) != CASU_OK) {
            cpl_msg_warning(fctid,"No ndit found in %s",
                            cpl_frame_get_filename(frame));
            cpl_propertylist_delete(plist);
            continue;
        }
        cpl_propertylist_delete(plist);

        /* Search the domedark groups to see if this DIT/NDIT combination has 
           already been used. If not, then create a new group. If it has 
           then just add this frame to the correct group */

        found = 0;
        for (j = 0; j < ps->nddg; j++) {
            if (ps->ddg[j].dit == dit && ps->ddg[j].ndit == ndit) {
                found = 1;
                break;
            }
        }
        if (found) {
            cpl_frameset_insert(ps->ddg[j].domes,cpl_frame_duplicate(frame));
            ps->ddg[j].ndomes += 1;
        } else {
            ps->ddg[ps->nddg].dit = dit;
            ps->ddg[ps->nddg].ndit = ndit;
            ps->ddg[ps->nddg].darks = cpl_frameset_new();
            ps->ddg[ps->nddg].domes = cpl_frameset_new();
            ps->ddg[ps->nddg].ndarks = 0;
            ps->ddg[ps->nddg].ndomes = 1;
            cpl_frameset_insert(ps->ddg[ps->nddg].domes,
                               cpl_frame_duplicate(frame));
            ps->ddg[ps->nddg].mean_flat = 0.0;
            ps->ddg[ps->nddg].med_flat = 0.0;
            ps->nddg += 1;
        }
    }
            
    /* Right, now loop through all the darks and get their exposure times */

    for (i = 0; i < ndarks; i++) {
        frame = cpl_frameset_get_position(ps->darks,i);
        plist = cpl_propertylist_load(cpl_frame_get_filename(frame),0);
        if (hawki_pfits_get_dit(plist,&dit) != CASU_OK) {
            cpl_msg_warning(fctid,"No dit time found in %s",
                            cpl_frame_get_filename(frame));
            cpl_propertylist_delete(plist);
            continue;
        }
        if (hawki_pfits_get_ndit(plist,&ndit) != CASU_OK) {
            cpl_msg_warning(fctid,"No ndit found in %s",
                            cpl_frame_get_filename(frame));
            cpl_propertylist_delete(plist);
            continue;
        }
        cpl_propertylist_delete(plist);

        /* Search the domedark groups to see if this dark fits into one of
           the defined groups. If not, then ignore it. If it does, then
           add it into the dark frameset */

        found = 0;
        for (j = 0; j < ps->nddg; j++) {
            if (ps->ddg[j].dit == dit && ps->ddg[j].ndit == ndit) {
                found = 1;
                break;
            }
        }
        if (found) {
            cpl_frameset_insert(ps->ddg[j].darks,cpl_frame_duplicate(frame));
            ps->ddg[j].ndarks += 1;
        }
    }

    /* Now go through the domedark groups and ditch any that don't have any
       dark frames */

    i = 0;
    while (i < ps->nddg) {
        if (ps->ddg[i].ndarks == 0) {
            cpl_msg_warning(fctid,
                            "No dark frames exist for %g/%d\nThrowing these away",
                            ps->ddg[i].dit,ps->ddg[i].ndit);
            freeframeset(ps->ddg[i].darks);
            freeframeset(ps->ddg[i].domes);
            for (j = i+1; j < ps->nddg; j++)
                ps->ddg[j-1] = ps->ddg[j];
            ps->nddg -= 1;
        } else
            i++;
    }

    /* Resize the output array and return so long as there is anything
       left. If there isn't then signal a major error */

    if (ps->nddg > 0) {
        ps->ddg = cpl_realloc(ps->ddg,ps->nddg*sizeof(ddgrp));
        return(0);
    } else {
        cpl_msg_error(fctid,"There are no darks defined for input domes");
        return(-1);
    }
}

/**@}*/

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_lin_init
    \par Purpose:
        Initialise pointers in the recipe memory structure.
    \par Description:
        Initialise pointers in the recipe memory structure.
    \par Language:
        C
    \param ps
        The memory structure
    \return
        Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static void hawki_lin_init(memstruct *ps) {
    
    /* Level 0 stuff */

    ps->labels = NULL;
    ps->darks = NULL;
    ps->flats = NULL;
    ps->cube_product = NULL;
    ps->map_product = NULL;
    ps->ddg = NULL;
    ps->nddg = 0;
    ps->poly = NULL;
    ps->poly_lin = NULL;
    ps->expmat = NULL;
    ps->invals = NULL;
    ps->invals_sig = NULL;
    ps->resid = NULL;

    /* Level 1 stuff */

    ps->curims = NULL;
    ps->nproc = 0;
    ps->res_cube = NULL;
    ps->res_im = NULL;
    ps->mean_fit = NULL;
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        hawki_lin_tidy
    \par Purpose:
        Free allocated workspace in the recipe memory structure
    \par Description:
        Free allocated workspace in the recipe memory structure. The tidy works
        on two levels. Level 1 is for items that are usually cleared up after
        each extension is processed. Level 0 is for cleaning up the whole
        recipe
    \par Language:
        C
    \param ps
        The memory structure
    \param level
        The level of the tidy to be done. 1: Tidy up after finishing an 
        extension, 0: Tidy up after finishing the recipe.
    \return
        Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static void hawki_lin_tidy(memstruct *ps, int level) {
    int i;

    /* Level 1 stuff */

    for (i = 0; i < ps->nproc; i++)
        freeimage(ps->curims[i]);
    freespace(ps->res_im);
    freespace(ps->res_cube);
    freespace(ps->mean_fit);

    if (level == 1)
        return;

    /* Level 0 stuff */

    freespace(ps->labels);
    freeframeset(ps->darks);
    freeframeset(ps->flats);
    if (ps->ddg != NULL) {
        for (i = 0; i < ps->nddg; i++) {
            freeframeset(ps->ddg[i].darks);
            freeframeset(ps->ddg[i].domes);
        }
        freespace(ps->ddg);
    }  
    freespace(ps->expmat);
    freespace(ps->invals);
    freespace(ps->invals_sig);
    freespace(ps->resid);
    freepolynomial(ps->poly);
    freepolynomial(ps->poly_lin);
    freespace(ps->curims);
}

/*

$Log: $

*/
