/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2014 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*---------------------------------------------------------------------------*
 *                             Includes                                      *
 *---------------------------------------------------------------------------*/
#include <stdio.h>
#include <float.h>
#include <math.h>
#include <string.h>
#include <cpl.h>
#include <muse.h>
#include "muse_ampl_z.h"

/*---------------------------------------------------------------------------*
 *                             Functions code                                *
 *---------------------------------------------------------------------------*/

/*---------------------------------------------------------------------------*/
/**
  @brief    Add the QC parameters to the combined ampl file
  @param    aImage   the output master image
  @param    aList    the image list of input files
 */
/*---------------------------------------------------------------------------*/
static void
muse_ampl_qc_header(muse_image *aImage, muse_imagelist *aList)
{
  /* write the image statistics for input ampl exposures */
  unsigned int k;
  for (k = 0; k < muse_imagelist_get_size(aList); k++) {
    char *keyword = cpl_sprintf("ESO QC AMPL INPUT%d "QC_BASIC_NSATURATED, k+1);
    int nsaturated = cpl_propertylist_get_int(muse_imagelist_get(aList, k)->header,
                                              MUSE_HDR_TMP_NSAT);
    cpl_propertylist_update_int(aImage->header, keyword, nsaturated);
    cpl_free(keyword);
  } /* for k (all images in list) */
} /* muse_ampl_qc_header() */

/*----------------------------------------------------------------------------*/
/**
  @private
  @brief    Create filter-like response table from double[][] array
  @param    aAmpl   the input double[][] array, has to be zero-terminated
  @return   cpl_table * on success, NULL on failure
 */
/*----------------------------------------------------------------------------*/
static cpl_table *
muse_ampl_table_new(const double aAmpl[][2])
{
  cpl_ensure(aAmpl, CPL_ERROR_NULL_INPUT, NULL);
  int i = -1, n = 0;
  while (aAmpl[++i][0] > 0.) n++;
  cpl_msg_debug(__func__, "creating table with %d entries", n);
  cpl_table *tampl = cpl_table_new(n);
  cpl_table_new_column(tampl, "lambda", CPL_TYPE_DOUBLE);
  cpl_table_new_column(tampl, "throughput", CPL_TYPE_DOUBLE);
  for (i = 0; i < n; i++) {
    cpl_table_set_double(tampl, "lambda", i, aAmpl[i][0]);
    cpl_table_set_double(tampl, "throughput", i, aAmpl[i][1]);
  } /* for i */
#if 0
  cpl_table_dump(tampl, 0, 100000, stdout);
  fflush(stdout);
#endif
  return tampl;
} /* muse_ampl_table_new() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Interpret the command line options and execute the data processing
  @param    aProcessing   the processing structure
  @param    aParams       the parameters list
  @return   0 if everything is ok, -1 something went wrong
 */
/*----------------------------------------------------------------------------*/
int
muse_ampl_compute(muse_processing *aProcessing, muse_ampl_params_t *aParams)
{
  /* search and load the files that we really need */
  cpl_table *trace = muse_processing_load_ctable(aProcessing, MUSE_TAG_TRACE_TABLE,
                                                 aParams->nifu);
  cpl_table *wave = muse_processing_load_ctable(aProcessing, MUSE_TAG_WAVECAL_TABLE,
                                                aParams->nifu);
  if (!trace || !wave) {
    cpl_msg_error(__func__, "Calibration could not be loaded:%s%s",
                  !trace ? " "MUSE_TAG_TRACE_TABLE : "",
                  !wave ? " "MUSE_TAG_WAVECAL_TABLE : "");
    /* try to clean up in case some files were successfully loaded */
    cpl_table_delete(trace);
    cpl_table_delete(wave);
    return -1;
  }

  /* now that we know that we have everything, start the real processing */
  muse_basicproc_params *bpars = muse_basicproc_params_new(aProcessing->parameters,
                                                           "muse.muse_ampl");
  muse_imagelist *images = muse_basicproc_load(aProcessing, aParams->nifu, bpars);
  muse_basicproc_params_delete(bpars);
  if (!images) {
    cpl_error_set_message(__func__, cpl_error_get_code(),
                          "Loading and processing raw data failed!");
    cpl_table_delete(trace);
    cpl_table_delete(wave);
    return -1;
  }
  muse_combinepar *cpars = muse_combinepar_new(aProcessing->parameters,
                                               "muse.muse_ampl");
  cpars->scale = CPL_TRUE; /* always scale to relative exposure time */
  muse_image *image = muse_combine_images(cpars, images);
  muse_combinepar_delete(cpars);
  if (!image) {
    cpl_msg_error(__func__, "Combining input frames failed!");
    muse_imagelist_delete(images);
    cpl_table_delete(trace);
    cpl_table_delete(wave);
    return -1;
  }

  cpl_propertylist_erase_regexp(image->header, MUSE_WCS_KEYS, 0);
  muse_ampl_qc_header(image, images);
  muse_basicproc_qc_saturated(image, "ESO QC AMPL MASTER");
  muse_imagelist_delete(images);
  if (aParams->savemaster) {
    muse_processing_save_image(aProcessing, aParams->nifu, image, "MASTER_AMPL");
  }

  /* get the filter name from the header and try to load its response */
  const char *fname = muse_pfits_get_pam2_filter(image->header);
  muse_table *filter = muse_table_load_filter(aProcessing, fname);
  if (!filter) {
    cpl_msg_error(__func__, "Filter \"%s\" could not be loaded from FILTER_LIST"
                  " file!", fname);
    return -1;
  }

  /* create the pixel table to work on */
  muse_pixtable *pt = muse_pixtable_create(image, trace, wave, NULL);
  cpl_table_delete(trace);
  cpl_table_delete(wave);
  if (!pt) {
    cpl_msg_error(__func__, "pixel table was not created: %s",
                  cpl_error_get_message());
    muse_image_delete(image);
    muse_table_delete(filter);
    return -1;
  }
  cpl_table *geotable = muse_processing_load_ctable(aProcessing,
                                                    MUSE_TAG_GEOMETRY_TABLE, 0);
  /* start with the approximate (constant) area per IFU [cm**2] */
  double fpsize = 60. / muse_pfits_get_focu_scale(pt->header),
         ifuarea = pow(fpsize / 10., 2) / kMuseNumIFUs;
  const char *modestring = muse_pfits_get_mode(pt->header) < MUSE_MODE_NFM_AO_N
                         ? "WFM" : "NFM";
  if (!geotable) {
    if (muse_pfits_get_mode(pt->header) >= MUSE_MODE_NFM_AO_N) {
      /* in narrow-field mode MUSE has a much smaller size in the focal plane */
      ifuarea /= pow(kMuseSpaxelSizeX_WFM / kMuseSpaxelSizeX_NFM, 2);
    } /* if NFM */
    cpl_msg_warning(__func__, "assuming constant per-IFU area of %f cm**2 "
                    "(mode %s)", ifuarea, modestring);
  } else {
    /* sum up the area from the geometry table */
    ifuarea = muse_geo_table_ifu_area(geotable, aParams->nifu, fpsize / 10.);
    cpl_msg_info(__func__, "computed area of IFU %d in focal plane: %.3f cm**2 "
                 "(mode %s)", aParams->nifu, ifuarea, modestring);
    cpl_table_delete(geotable);
  } /* else */

  /*
   * the actual processing of interest happens next:
   * - input data is in electrons, ignoring the CCD response it is in photons
   * - convolve the flux of each CCD pixel with the filter response
   * - convert data to W and W/cm**2
   * - convert photo diode current from A to W/cm**2
   * - compute throughput relative to both photo diodes from the total detected
   *   power per area
   * - compute the throughput relative to the 2nd photo diode from the power
   *   per area detected in each slice
   */
  /* data in photon units: */
  cpl_table_set_column_unit(pt->table, MUSE_PIXTABLE_DATA, "ph");
  /* collect energy [J] in stat column: */
  cpl_table_set_column_unit(pt->table, MUSE_PIXTABLE_STAT, "J");
  /* cut off and sum data values and energies */
  const float *lbda = cpl_table_get_data_float_const(pt->table, MUSE_PIXTABLE_LAMBDA);
  float *data = cpl_table_get_data_float(pt->table, MUSE_PIXTABLE_DATA),
        *stat = cpl_table_get_data_float(pt->table, MUSE_PIXTABLE_STAT);
  cpl_table_unselect_all(pt->table);
  cpl_msg_debug(__func__, "Using black body correction with T = %.1f K",
                aParams->temp);
  /* summed values: data and the respective photon energy, as well as *
   * the weighted mean wavelength (and the weight sum used for that)  */
  double dsum = 0., esum = 0., lwmean = 0., wsum = 0.;
  cpl_size irow, nrow = cpl_table_get_nrow(pt->table);
  for (irow = 0; irow < nrow; irow++) {
    double resp = muse_flux_response_interpolate(filter->table, lbda[irow],
                                                 NULL, MUSE_FLUX_RESP_FILTER);
    if (resp <= 0.) {
      cpl_table_select_row(pt->table, irow);
      continue;
    }
    /* fold the remaining datapoints with the filter response */
    data[irow] *= resp;
    /* misuse the stat column to save photon energy: *
     * Eph = h nu = h * c / lambda * nph             *
     * units: [Js] * [m/s] / [m] * [1] = [J]         */
    stat[irow] = CPL_PHYS_H * CPL_PHYS_C / lbda[irow] * 1e10 * data[irow];
    dsum += data[irow]; /* sum the number of photons */
    esum += stat[irow]; /* some the photon energy */
    /* compute the normalized (unitless) Planck curve Bn(T,lambda) *
     * for the assumed  flat-field (halogen) lamp temperature      */
    double lbda_m = lbda[irow] / 1e10, /* wavelength [m] */
           fb = (CPL_PHYS_H * CPL_PHYS_C) / (CPL_PHYS_K * aParams->temp * lbda_m),
           Bn = 15. / lbda_m * pow(fb / CPL_MATH_PI, 4) / (exp(fb) - 1.);
    wsum += data[irow] / Bn;
    lwmean += lbda[irow] * data[irow] / Bn;
  } /* for irow (pixel table rows) */
  lwmean /= wsum;
  cpl_msg_info(__func__, "%"CPL_SIZE_FORMAT" of %"CPL_SIZE_FORMAT" pixel table "
               "rows (%.2f%%) unused", cpl_table_count_selected(pt->table), nrow,
               100. * cpl_table_count_selected(pt->table) / nrow);
  cpl_table_erase_selected(pt->table);
  muse_pixtable_compute_limits(pt);
  double exptime = muse_pfits_get_exptime(pt->header),
         power = esum / exptime; /* power [W] */
  /* ... to compute the corresponding power density */
  double pdensity = power / ifuarea; /* power density [W/cm**2] */
  cpl_msg_info(__func__, "Summed flux: %e ph, energy: %e J, power: %e W, power "
               "density: %e W/cm**2", dsum, esum, power, pdensity);

  /* create tables from kMuseAmpl1[][] and kMuseAmpl2[][] and use   *
   * them to compute mean photo diode sensitivities in [A/W*cm**2]; *
   * take into account the size of the photo diodes                 */
  cpl_table *tampl1 = muse_ampl_table_new(kMuseAmpl1),
            *tampl2 = muse_ampl_table_new(kMuseAmpl2);
  double sens1 = muse_flux_response_interpolate(tampl1, lwmean, NULL,
                                                MUSE_FLUX_RESP_FILTER)
               * kMuseAmplPhysicalSize,
         sens2 = muse_flux_response_interpolate(tampl2, lwmean, NULL,
                                                MUSE_FLUX_RESP_FILTER)
               * kMuseAmplPhysicalSize;
  cpl_table_delete(tampl1);
  cpl_table_delete(tampl2);
  cpl_msg_info(__func__, "mean photo diode sensitivities at weighted mean "
               "wavelength %.2f Angstrom: %f A/W*cm**2, %f A/W*cm**2", lwmean,
               sens1, sens2);
  /* take into account beam widening for photo diode 2 */
  cpl_msg_debug(__func__, "using beam widening factor %.3f for photo diode 2",
                aParams->fbeam);
  sens2 /= aParams->fbeam;

  /* now convert the AMPL measurements from the FITS *
   * header to the same power density units          */
  cpl_errorstate state = cpl_errorstate_get();
  double pam1 = muse_pfits_get_pam_intensity(image->header, 1), /* [A] */
         pam2 = muse_pfits_get_pam_intensity(image->header, 2); /* [A] */
  if (!cpl_errorstate_is_equal(state)) {
    cpl_msg_error(__func__, "Pico-amplifier measurement(s) not found in header: %s",
                  cpl_error_get_message());
    muse_image_delete(image);
    muse_table_delete(filter);
    muse_pixtable_delete(pt);
    return -1;
  }
  double pam1stdev = muse_pfits_get_pam_stdev(image->header, 1),
         pam2stdev = muse_pfits_get_pam_stdev(image->header, 2),
         p1density = pam1 / sens1, /* now [W/cm**2] */
         p2density = pam2 / sens2; /* now [W/cm**2] */
  cpl_msg_info(__func__, "amplifiers: AMPL1 %.3e +/- %.3e A, %.2e W/cm**2, "
               "AMPL2 %.3e +/- %.3e A, %.2e W/cm**2", pam1, pam1stdev,
               p1density, pam2, pam2stdev, p2density);
  /* now compute the instrument throughput: compare the power density         *
   * detected on the CCD to the same value measured at each photo diode,      *
   * (unitless throughput values); the throughput errors are estimated as in  *
   * VLT-TRE-MUS-14670-0682 v0.90, as 3% error due to equal IFU area and the  *
   * filter error either 1 or 2%; additionally the value computed for the     *
   * fbeam parameter is around 1.03 +/- 0.04, so 4% error there.              */
  double thru1 = pdensity / p1density * 100.,
         thru2 = pdensity / p2density * 100.,
         areaerr = 0.03,
         filterr = lwmean > 5000 && lwmean < 8000 ? 0.01 : 0.02,
         beamerr = 0.04, /* approximate error on fbeam */
         thru2err = sqrt(areaerr*areaerr + beamerr*beamerr + filterr*filterr);
  cpl_msg_info(__func__, "throughput values: to AMPL1 %.3f %%, to "
               "AMPL2 %.3f +/- %.3f %%", thru1, thru2, thru2err * thru2);

  /* write some of this as QC parameters */
  cpl_propertylist_erase_regexp(pt->header, "ESO QC", 0);
  cpl_propertylist_append_float(pt->header, "ESO QC AMPL PHOTONS", dsum);
  cpl_propertylist_append_float(pt->header, "ESO QC AMPL POWER", power);
  cpl_propertylist_append_float(pt->header, "ESO QC AMPL THRU1", thru1);
  cpl_propertylist_append_float(pt->header, "ESO QC AMPL THRU2", thru2);
  cpl_propertylist_append_float(pt->header, "ESO QC AMPL THRU2ERR", thru2 * thru2err);

  /* sum the power per slice and record as QC parameter */
  muse_pixtable **pts = muse_pixtable_extracted_get_slices(pt);
  double slicearea = ifuarea / kMuseSlicesPerCCD;
  cpl_msg_warning(__func__, "assuming constant per-slice area of %f cm**2",
                  slicearea);
  int ipt;
  for (ipt = 0; ipt < muse_pixtable_extracted_get_size(pts); ipt++) {
    uint32_t origin = cpl_table_get_int(pts[ipt]->table, MUSE_PIXTABLE_ORIGIN,
                                        0, NULL);
    unsigned short slice = muse_pixtable_origin_get_slice(origin);
    /* the stat column contains the photon energy now, see *
     * above; use its sum to compute the per-slice power   */
    double spower = cpl_table_get_column_mean(pts[ipt]->table, MUSE_PIXTABLE_STAT)
                  * cpl_table_get_nrow(pts[ipt]->table) / exptime, /* [W] */
           spdensity = spower / slicearea, /* [W/cm**2] */
           sthru2 = spdensity / p2density * 100.;
    char *keyword = cpl_sprintf("ESO QC AMPL SLICE%d THRU2", slice);
    cpl_propertylist_append_float(pt->header, keyword, sthru2);
    cpl_free(keyword);
    cpl_msg_info(__func__, "slice %2d: power %.3e W, throughput %.3f %%",
                 slice, spower, sthru2);
  } /* for ipt (all slice pixel tables) */
  muse_pixtable_extracted_delete(pts);
  if (aParams->savetable) {
    /* Since this is not a real pixel table any more, since we messed with the *
     * columns heavily, a user should not interpret it as a pixel table. So    *
     * this needs to be saved as a normal FITS table, if requested at all.     */
    muse_processing_save_table(aProcessing, aParams->nifu, pt->table,
                               pt->header, "TABLE_AMPL", MUSE_TABLE_TYPE_CPL);
  }

  /* To see if the filter convolution has worked: project it back to an    *
   * image. As this is then the primary output product, copy the computed  *
   * QC parameters from both other outputs to its header before saving it. */
  muse_imagelist *list = muse_pixtable_to_imagelist(pt);
  muse_image *outimage = muse_imagelist_get(list, 0);
  cpl_propertylist_copy_property_regexp(outimage->header, image->header,
                                        "^ESO QC", 0);
  cpl_propertylist_copy_property_regexp(outimage->header, pt->header,
                                        "^ESO QC", 0);
  /* we don't need the SLICEi.CENTER keywords here, remove them */
  cpl_propertylist_erase_regexp(outimage->header, "^ESO DRS MUSE SLICE.* CENTER", 0);
  /* Here we have to use a hack to save this image, since it's not    *
   * a normal muse_image but has different contents because we messed *
   * with the pixel table before creating it.                         */
  muse_processing_save_header(aProcessing, aParams->nifu, outimage->header,
                              "AMPL_CONVOLVED");
  /* Get the newly created frame and its filename *
   * to extend that file with the image data.     */
  cpl_frame *outframe = muse_frameset_find_master(aProcessing->outframes,
                                                  "AMPL_CONVOLVED", aParams->nifu);
  const char *fn = cpl_frame_get_filename(outframe);
  cpl_propertylist *hext = cpl_propertylist_new();
  cpl_propertylist_append_string(hext, "EXTNAME", "PHOTONS");
  cpl_propertylist_set_comment(hext, "EXTNAME", "This extension contains photon counts");
  cpl_propertylist_append_string(hext, "BUNIT", cpl_table_get_column_unit(pt->table,
                                                                          MUSE_PIXTABLE_DATA));
  cpl_image_save(outimage->data, fn, CPL_TYPE_UNSPECIFIED, hext, CPL_IO_EXTEND);
  cpl_propertylist_update_string(hext, "EXTNAME", "ENERGY");
  cpl_propertylist_set_comment(hext, "EXTNAME", "This extension contains per-pixel energy");
  cpl_propertylist_update_string(hext, "BUNIT", cpl_table_get_column_unit(pt->table,
                                                                          MUSE_PIXTABLE_STAT));
  cpl_image_save(outimage->stat, fn, CPL_TYPE_UNSPECIFIED, hext, CPL_IO_EXTEND);
#if 0 /* ignore the dq components, it's not necessary here */
  cpl_propertylist_update_string(hext, "EXTNAME", EXTNAME_DQ);
  cpl_propertylist_set_comment(hext, "EXTNAME", EXTNAME_DQ_COMMENT);
  cpl_propertylist_erase(hext, "BUNIT"); /* no unit for data quality */
  cpl_image_save(outimage->dq, fn, CPL_TYPE_UNSPECIFIED, hext, CPL_IO_EXTEND);
#endif
  cpl_frame_delete(outframe);
  cpl_propertylist_delete(hext);

  /* clean up */
  muse_imagelist_delete(list);
  muse_pixtable_delete(pt);
  muse_image_delete(image);
  muse_table_delete(filter);

  return 0;
} /* muse_ampl_compute() */
