/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2014 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*----------------------------------------------------------------------------*
 *                              Includes                                      *
 *----------------------------------------------------------------------------*/
#include <stdio.h>
#include <float.h>
#include <math.h>
#include <string.h>
#include <cpl.h>
#include <muse.h>
#include "muse_flat_z.h"

/*---------------------------------------------------------------------------*
 *                              Functions code                               *
 *---------------------------------------------------------------------------*/

/*---------------------------------------------------------------------------*/
/**
  @brief    Add the QC parameters to the combined flat
  @param    aImage   the output master image
  @param    aList    the image list of input files
 */
/*---------------------------------------------------------------------------*/
static void
muse_flat_qc_header(muse_image *aImage, muse_imagelist *aList)
{
  cpl_msg_debug(__func__, "Adding QC keywords");

  unsigned stats = CPL_STATS_MEDIAN | CPL_STATS_MEAN | CPL_STATS_STDEV
                 | CPL_STATS_MIN | CPL_STATS_MAX;

  /* write the image statistics for input flat field exposures */
  unsigned int k;
  for (k = 0; k < muse_imagelist_get_size(aList); k++) {
    char *keyword = cpl_sprintf(QC_FLAT_PREFIXi, k+1);
    muse_basicproc_stats_append_header(muse_imagelist_get(aList, k)->data,
                                       aImage->header, keyword, stats);
    cpl_free(keyword);
    keyword = cpl_sprintf(QC_FLAT_PREFIXi" "QC_BASIC_NSATURATED, k+1);
    int nsaturated = cpl_propertylist_get_int(muse_imagelist_get(aList, k)->header,
                                              MUSE_HDR_TMP_NSAT);
    cpl_propertylist_update_int(aImage->header, keyword, nsaturated);
    cpl_free(keyword);
  } /* for k (all images in list) */

  /* properties of the resulting master frame */
  stats |= CPL_STATS_FLUX;
  muse_basicproc_stats_append_header(aImage->data, aImage->header,
                                     QC_FLAT_MASTER_PREFIX, stats);
} /* muse_flat_qc_header() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Add the QC parameters to the trace table
  @param    aHeader   the FITS header of the output image
  @param    aTrace    the computed trace table
 */
/*---------------------------------------------------------------------------*/
static void
muse_flat_qc_trace_header(cpl_propertylist *aHeader, const cpl_table *aTrace)
{
  if (!aHeader || !aTrace) {
    return;
  }
  unsigned char ifu = muse_utils_get_ifu(aHeader);
  cpl_msg_debug(__func__, "Adding tracing QC keywords for IFU %hhu", ifu);

  /* create vector for gap statistics */
  cpl_array *gaps = cpl_array_new(kMuseSlicesPerCCD - 1, CPL_TYPE_DOUBLE);
  double redge = -1; /* keep right edge of slice */
  int nslice;
  for (nslice = 1; nslice <= kMuseSlicesPerCCD; nslice++) {
    cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
                                                                   nslice);
    if (!ptrace) {
      cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
                      "missing!", nslice, ifu);
      continue;
    }

    /* ask for the center of the slice at the vertical center of the CCD */
    float xpos = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
                                        kMuseOutputYTop/2, NULL);
    if (xpos < 1 || xpos > kMuseOutputXRight || !isnormal(xpos)) {
      cpl_msg_warning(__func__, "slice %2d of IFU %hhu: faulty polynomial "
                      "detected at y=%d (center: %f)", nslice, ifu,
                      kMuseOutputYTop/2, xpos);
      muse_trace_polys_delete(ptrace);
      continue; /* next slice */
    }
    /* also may need the centers of the trace  *
     * at bottom and top for tilt computation: */
    double x1 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER], 1, NULL),
           x2 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
                                       kMuseOutputYTop, NULL);
    float tilt = atan((x2 - x1) / (kMuseOutputYTop - 1)) * CPL_MATH_DEG_RAD;
    if (nslice == 1) {
      cpl_propertylist_append_float(aHeader, QC_TRACE_L_XPOS, xpos);
      cpl_propertylist_append_float(aHeader, QC_TRACE_L_TILT, tilt);
    } else if (nslice == kMuseSlicesPerCCD) {
      cpl_propertylist_append_float(aHeader, QC_TRACE_R_XPOS, xpos);
      cpl_propertylist_append_float(aHeader, QC_TRACE_R_TILT, tilt);
    }
    if (redge > 0) {
      /* the gap is the difference between the left edge of the *
       * current slice and the right edge of the previous slice */
      double gap = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
                                          kMuseOutputYTop/2, NULL)
                 - redge;
      cpl_array_set_double(gaps, nslice - 2, gap);
    }
    /* compute right edge for gap computation */
    redge = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], kMuseOutputYTop/2,
                                   NULL);

    /* Compute derivatives of all three tracing polynominals to  *
     * derive the maximum slope of the solutions within the CCD. */
    double maxslope = 0.;
    int ipoly, j, jmax = -1, ipolymax = -1;
    for (ipoly = MUSE_TRACE_CENTER; ipoly <= MUSE_TRACE_RIGHT; ipoly++) {
      cpl_polynomial_derivative(ptrace[ipoly], 0);
      /* have to evaluate the polynomial at all positions to  *
       * get the maximum within the typical CCD boundaries... */
      for (j = 1; j <= kMuseOutputYTop; j++) {
        double slope = cpl_polynomial_eval_1d(ptrace[ipoly], j, NULL);
        if (fabs(slope) > fabs(maxslope)) {
          maxslope = slope;
          jmax = j;
          ipolymax = ipoly;
        } /* if */
      } /* for j (all positions in vertical direction) */
    } /* for ipoly (all tracing polynomials) */
#define MAX_NORMAL_SLOPE 0.015
    if (fabs(maxslope) > MAX_NORMAL_SLOPE) {
      cpl_msg_warning(__func__, "Slope in slice %2d of IFU %hhu is unusually "
                      "high (maximum in polynomial %d at y=%d is |%f| > %f)",
                      nslice, ifu, ipolymax, jmax, maxslope, MAX_NORMAL_SLOPE);
    }
    char *kw = cpl_sprintf(QC_TRACE_SLICEj_MXSLOP, nslice);
    cpl_propertylist_append_float(aHeader, kw, maxslope);
    cpl_free(kw);
    muse_trace_polys_delete(ptrace);

    /* add widths of corner slices as QC parameter to detect clipped edges */
    if (nslice == 10 || nslice == 46 || nslice == 3 || nslice == 39) {
      /* get entry for the slice (table should be ordered, but to be sure) */
      int irow, err, nrow = cpl_table_get_nrow(aTrace);
      for (irow = 0; irow < nrow; irow++) {
        int slice = cpl_table_get_int(aTrace, MUSE_TRACE_TABLE_COL_SLICE_NO,
                                      irow, &err);
        if (slice == nslice && !err) {
          break;
        }
      } /* for irow */
      if (irow >= nrow) { /* slice not found! */
        cpl_msg_warning(__func__, "Slice %d not found in trace table!", nslice);
        continue;
      }
      float width = cpl_table_get(aTrace, MUSE_TRACE_TABLE_COL_WIDTH, irow, &err);
      kw = cpl_sprintf(QC_TRACE_SLICEj_WIDTH, nslice);
      cpl_propertylist_append_float(aHeader, kw, width);
      cpl_free(kw);
    } /* if corner slice */
  } /* for nslice */

  float median, mean, stdev, min, max;
  /* now add the gap statistics as QC parameters */
  median = cpl_array_get_median(gaps);
  mean = cpl_array_get_mean(gaps);
  stdev = cpl_array_get_stdev(gaps);
  min = cpl_array_get_min(gaps);
  max = cpl_array_get_max(gaps);
  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MEDIAN, median);
  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MEAN, mean);
  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_STDEV, stdev);
  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MIN, min);
  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MAX, max);
  cpl_array_delete(gaps);

  /* add the slice widths also as QC paramters */
  median = cpl_table_get_column_median(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
  mean = cpl_table_get_column_mean(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
  stdev = cpl_table_get_column_stdev(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
  min = cpl_table_get_column_min(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
  max = cpl_table_get_column_max(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MEDIAN, median);
  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MEAN, mean);
  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_STDEV, stdev);
  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MIN, min);
  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MAX, max);
} /* muse_flat_qc_trace_header() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Add per-slice QC statistics to the master flat image
  @param    aImage   the combined master flat, before normalization
  @param    aTrace   the computed trace table
 */
/*---------------------------------------------------------------------------*/
static void
muse_flat_qc_per_slice(muse_image *aImage, const cpl_table *aTrace)
{
  if (!aImage->header || !aTrace) {
    return;
  }
  unsigned char ifu = muse_utils_get_ifu(aImage->header);
  cpl_msg_debug(__func__, "Adding per-slice QC statistics for IFU %hhu", ifu);

  /* common vertical region in each slice, around the slice center */
  const int y1 = kMuseOutputYTop/2 - 100,
            y2 = kMuseOutputYTop/2 + 100;
  int nslice;
  for (nslice = 1; nslice <= kMuseSlicesPerCCD; nslice++) {
    cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
                                                                   nslice);
    if (!ptrace) {
      cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
                      "missing!", nslice, ifu);
      continue;
    }
    char *kwmean = cpl_sprintf(QC_FLAT_MASTER_SLICEj_MEAN, nslice),
         *kwstdev = cpl_sprintf(QC_FLAT_MASTER_SLICEj_STDEV, nslice);
    /* horizontal region for this slice */
    int x1 = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
                                           kMuseOutputYTop/2, NULL)) + 1,
        x2 = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT],
                                           kMuseOutputYTop/2, NULL)) - 1;
    if (x1 < 1 || x2 > kMuseOutputXRight || x1 > x2) {
      cpl_msg_warning(__func__, "slice %2d of IFU %hhu: faulty polynomial "
                      "detected at y=%d (borders: %d ... %d)", nslice, ifu,
                      kMuseOutputYTop/2, x1, x2);
      muse_trace_polys_delete(ptrace);
      continue; /* next slice */
    }
    double mean = cpl_image_get_mean_window(aImage->data, x1, y1, x2, y2),
           stdev = cpl_image_get_stdev_window(aImage->data, x1, y1, x2, y2);
    cpl_propertylist_update_float(aImage->header, kwmean, mean);
    cpl_propertylist_update_float(aImage->header, kwstdev, stdev);

    muse_trace_polys_delete(ptrace);
    cpl_free(kwmean);
    cpl_free(kwstdev);
  } /* for nslice */
} /* muse_flat_qc_per_slice() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Kick of tracing and bad pixel search on a master flat-field exposure
  @param    aProcessing   the processing structure
  @param    aParams       the recipe parameters
  @param    aImage        the master flat-field image
  @return   CPL_ERROR_NONE if everything is ok, a CPL error otherwise
 */
/*---------------------------------------------------------------------------*/
static cpl_error_code
muse_flat_trace_badpix(muse_processing *aProcessing,
                       muse_flat_params_t *aParams, muse_image *aImage)
{
  cpl_table *samples = NULL;
  cpl_table *tracetable = muse_trace(aImage, aParams->nsum, aParams->edgefrac,
                                     aParams->order,
                                     aParams->samples ? &samples : NULL);
  if (!tracetable) {
    cpl_table_delete(samples); /* just in case... */
    return cpl_error_get_code();
  }

  /* create table header by copying most of the image header */
  cpl_propertylist *header = cpl_propertylist_duplicate(aImage->header);
  cpl_propertylist_erase_regexp(header,
                                "^SIMPLE$|^BITPIX$|^NAXIS|^EXTEND$|^XTENSION$|"
                                "^DATASUM$|^DATAMIN$|^DATAMAX$|^DATAMD5$|"
                                "^PCOUNT$|^GCOUNT$|^HDUVERS$|^BLANK$|"
                                "^BZERO$|^BSCALE$|^BUNIT$|^CHECKSUM$|^INHERIT$|"
                                "^PIPEFILE$|^ESO QC |^ESO PRO ", 0);

  muse_flat_qc_trace_header(header, tracetable);
  cpl_error_code rc = muse_processing_save_table(aProcessing, aParams->nifu,
                                                 tracetable, header,
                                                 MUSE_TAG_TRACE_TABLE,
                                                 MUSE_TABLE_TYPE_CPL);
  if (samples) {
    /* don't want to save QC headers with the samples table: */
    cpl_propertylist_erase_regexp(header, QC_TRACE_PREFIX, 0);
    muse_processing_save_table(aProcessing, aParams->nifu, samples, header,
                               MUSE_TAG_TRACE_SAMPLES, MUSE_TABLE_TYPE_CPL);
    cpl_table_delete(samples);
  }
  cpl_propertylist_delete(header);
  if (rc != CPL_ERROR_NONE) {
    cpl_table_delete(tracetable);
    return rc;
  }

  /* if we have successfully traced, we can also search for dark pixels */
  int nbad = muse_quality_flat_badpix(aImage, tracetable,
                                      aParams->losigmabadpix,
                                      aParams->hisigmabadpix);
  cpl_msg_info(__func__, "Found %d bad pixels in the flat-field image of IFU %d",
               nbad, aParams->nifu);

  /* add image statistics as QC now that we know the slice locations */
  muse_flat_qc_per_slice(aImage, tracetable);

  cpl_table_delete(tracetable);
  return rc;
} /* muse_flat_trace_badpix() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Interpret the command line options and execute the data processing
  @param    aProcessing   the processing structure
  @param    aParams       the parameters list
  @return   0 if everything is ok, -1 something went wrong
 */
/*---------------------------------------------------------------------------*/
int
muse_flat_compute(muse_processing *aProcessing, muse_flat_params_t *aParams)
{
  muse_basicproc_params *bpars = muse_basicproc_params_new(aProcessing->parameters,
                                                           "muse.muse_flat");
  muse_imagelist *images = muse_basicproc_load(aProcessing, aParams->nifu, bpars);
  muse_basicproc_params_delete(bpars);
  cpl_ensure(images, cpl_error_get_code(), -1);

  muse_combinepar *cpars = muse_combinepar_new(aProcessing->parameters,
                                               "muse.muse_flat");
  muse_image *masterimage = muse_combine_images(cpars, images);
  muse_combinepar_delete(cpars);
  if (!masterimage) {
    cpl_msg_error(__func__, "Combining input frames failed for IFU %d!",
                  aParams->nifu);
    muse_imagelist_delete(images);
    return -1;
  }
  cpl_propertylist_erase_regexp(masterimage->header, MUSE_WCS_KEYS, 0);

  muse_flat_qc_header(masterimage, images);
  muse_imagelist_delete(images);

  cpl_error_code rc1 = CPL_ERROR_NONE;
  if (aParams->trace) {
    /* trace and search for bad pixels; do this before saving the     *
     * masterimage, so that we can add bad pixels to its DQ extension */
    rc1 = muse_flat_trace_badpix(aProcessing, aParams, masterimage);
    if (rc1 != CPL_ERROR_NONE) {
      /* warn but don't return yet; tracing is important but the user should *
       * still be able to get the masterimage saved to take a look at it     */
      cpl_msg_error(__func__, "Tracing/bad pixel search failed in IFU %d",
                    aParams->nifu);
    }
  }

  /* if requested, normalize the output flat-field to 1 using the average value */
  if (aParams->normalize) {
    double mean = cpl_propertylist_get_float(masterimage->header,
                                             QC_FLAT_MASTER_PREFIX" MEAN");
    muse_image_scale(masterimage, 1. / mean);
  }

  muse_basicproc_qc_saturated(masterimage, QC_FLAT_MASTER_PREFIX);
  cpl_error_code rc2 = muse_processing_save_image(aProcessing, aParams->nifu,
                                                  masterimage,
                                                  MUSE_TAG_MASTER_FLAT);
  muse_image_delete(masterimage);
  return rc1 == CPL_ERROR_NONE && rc2 == CPL_ERROR_NONE ? 0 : -1;
} /* muse_flat_compute() */
