/*
 * This file is part of the ESO SINFONI Pipeline
 * Copyright (C) 2004-2009 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

/*
 * $Author: amodigli $
 * $Date: 2013-08-15 12:35:10 $
 * $Revision: 1.12 $
 */

#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif

#include <stdio.h>
#include <cpl.h>
#include <cpl_test.h>
#include <sinfo_atmo_disp.h>
#include <sinfo_utils_wrappers.h>
#include <sinfo_utilities.h>
#include <sinfo_resampling.h>
#include <sinfo_msg.h>

const char* SINFO_STAR_BUG_REPORT = "amodigli@eso.org";
const double EPSILON = 1E-8;
const char* COL_NAME_ID = "id";
const char* COL_NAME_CENTRE_X = "center_x";
const char* COL_NAME_CENTRE_Y = "center_y";
const char* COL_NAME_AIRMASS = "airmass";
const char* COL_NAME_WAVELENGTH = "wavelength";
const char* COL_NAME_ANGLE = "angle";
const char* COL_NAME_OFFSET_X = "offset_x";
const char* COL_NAME_OFFSET_Y = "offset_y";
static void atmo_process_fits_data_file(cpl_table** ptable, const char* filename);
static void atmo_save_gauss_info(cpl_table * ptable, cpl_imagelist* pCube);
static cpl_error_code atmo_disp_load_params(		const char* fits_file,
		int * centpix,
		double* lambda0,
		double *Tc,
		double* rh,
		double* airm,
		double* p,
		double*	parallactic,
		double* pixelscale,
		double* pixelsz);
static void atmo_test_dispersion_cube( const char* fits_file, const char* out_file);
static double atmo_get_general_lambda0(const char* fits_file);
static void atmo_init_offset_table(cpl_table** big_table, int big_table_size);
static void atmo_rotate_offset(cpl_table* ptable, double rot_angle);
static void atmo_rotate_point(double* x_value, double* y_value, double rot_angle);
static void atmo_prepare_offset_table(const char* cfg_filename,
		const char* output_filename);
static cpl_polynomial*  atmo_prepare_polyfit(const char* input_filename);
static void atmo_filter_resultset(double value_min, double value_max,
		const char* column_name,
		const char* fits_file_input,
		const char* fits_file_output);
static void atmo_prepare_fit_025_HK(const char* input_filename,
		const char* output_filename);
static void atmo_prepare_poly_net(cpl_polynomial* poly, const char* filename);
static void atmo_apply_cube_polynomial_shift(cpl_polynomial* poly,
		const char* fits_file,
		const char* out_file);
static void atmo_apply_cube_polynomial_shift_list(
		cpl_polynomial* poly,
		const char* list_file);
static void atmo_save_polynom(cpl_polynomial* poly,
		const char* filename,
		cpl_propertylist* plist);
static cpl_polynomial* atmo_load_polynom(const char* filename);
/*------------------------------------------------------------*/

static cpl_error_code atmo_disp_load_params(
		const char* fits_file,
		int * centpix,
		double* lambda0,
		double *Tc,
		double* rh,
		double* airm,
		double* p,
		double*	parallactic,
		double* pixelscale,
		double* pixelsz)
{
	double * VALS_DOUBLE[10];
	const char* NAME_DOUBLE[10] = {
			"CRVAL3",
			"ESO TEL AMBI TEMP",
			"ESO TEL AMBI RHUM",
			"ESO TEL AIRM START",
			"ESO TEL AIRM END",
			"ESO TEL AMBI PRES START",
			"ESO TEL AMBI PRES END",
			"ESO ADA ABSROT START",
			"ESO ADA ABSROT END",
//			"ESO TEL PARANG START",
//			"ESO TEL PARANG END",
			"CDELT3"
	};
	double airmass_start;
	double airmass_end;
	double pres_start,	pres_end;
	double parang_start, parang_end;
	cpl_propertylist* plist = cpl_propertylist_load(fits_file, 0);
	cpl_error_code err = CPL_ERROR_NONE;
	int sz = 0;
	int i ;


	VALS_DOUBLE[0] = lambda0;
	VALS_DOUBLE[1] = Tc;
	VALS_DOUBLE[2] = rh;
	VALS_DOUBLE[3] = &airmass_start;
	VALS_DOUBLE[4] = &airmass_end;
	VALS_DOUBLE[5] = &pres_start;
	VALS_DOUBLE[6] = &pres_end;

	VALS_DOUBLE[7] =  &parang_start;
	VALS_DOUBLE[8] =  &parang_end;
	VALS_DOUBLE[9] =  pixelsz;


	cpl_test(plist);
	if (plist == 0)
	{
		cpl_msg_warning (cpl_func, "could not open the file[%s]", fits_file);
		cpl_error_reset();
		return CPL_ERROR_FILE_IO;
	}
	sz = sizeof(NAME_DOUBLE) / sizeof(NAME_DOUBLE[0]);
	for (i = 0; i < sz; i++)
	{
		*(VALS_DOUBLE[i]) = cpl_propertylist_get_double(plist, NAME_DOUBLE[i]);
		err = cpl_error_get_code();
		if(err != CPL_ERROR_NONE)
		{
			cpl_error_reset();
			return err;
		}
	}

	*centpix = cpl_propertylist_get_int(plist, "CRPIX3");
	{
		const char* optiname = cpl_propertylist_get_string(plist, "ESO INS OPTI1 NAME");

		float f = 0;
		sscanf(optiname, "%f", &f);
		*pixelscale = f;
	}

	err = cpl_error_get_code();
	if (err != CPL_ERROR_NONE)
	{
		cpl_error_reset();
		return err;
	}
	*airm = (airmass_start + airmass_end) / 2;
	*p = (pres_start + pres_end) / (2 * 0.750064); // 0.75 for conversion from mm to mbar
	*parallactic = (parang_start + parang_end) / 2;
	cpl_propertylist_delete(plist);
	return err;
}


static void
sinfo_fit_poly(cpl_table* ptable,
               const char* col_x,
               const char* col_y,
               const int order)
{
   /*
    * The "dummy" table is just a tool for eliminating invalid
    * points from the vectors to be fitted.
    */
   cpl_polynomial *py=NULL;
   int nrows=0;
   cpl_vector* vx=NULL;
   cpl_vector* vy=NULL;
   int i=0;
   cpl_error_code err = CPL_ERROR_NONE;

   nrows=cpl_table_get_nrow(ptable);
   vx = cpl_vector_new(nrows);
   vy = cpl_vector_new(nrows);
   for (i = 0; i < nrows; i++)
   {
	   int inull = 0;
	   double dy = cpl_table_get_double(ptable, col_y, i, &inull);
	   double dx = cpl_table_get_int(ptable, col_x, i, &inull);
	   cpl_vector_set(vy, i, dy);
	   cpl_vector_set(vx, i, dx);
   }
//   cpl_table_save(dummy,NULL,NULL,"pippo.fits",CPL_IO_DEFAULT);
//   cpl_table_save(ptable,NULL,NULL,"or_pippo.fits",CPL_IO_DEFAULT);

   if (nrows < 2 * order)
   {
	  cpl_msg_warning(cpl_func, "number of points is not enough for fit");
      goto cleanup;
   }
//   cpl_table_fill_invalid_double(dummy, "x", 0);
//   cpl_table_fill_invalid_double(dummy, "y", 0);
   err = cpl_error_get_code();
   if (err == CPL_ERROR_NONE)
   {
	   py = sinfo_polynomial_fit_1d_create(vx, vy, order, NULL);
	   cpl_polynomial_dump(py,stdout);
/*	   coef=cpl_vector_new(order);
	   pows[0]=0;

	   for(i=0;i<order;i++)
	   {
		  pows[0]=i;
		  cpl_vector_set(coef,i,cpl_polynomial_get_coeff(py,pows));
		  sinfo_msg("coef=%g",cpl_polynomial_get_coeff(py,pows));
	   }
	   cpl_vector_dump(coef,stdout);*/
   }
   else
   {
	   cpl_msg_warning(cpl_func, "cpl error occured [2]");
   }
  cleanup:
//   sinfoni_free_vector(&coef);
//   sinfo_free_table(&dummy);
   cpl_vector_delete(vx);
   cpl_vector_delete(vy);
   cpl_polynomial_delete(py);
   return;
}
static void atmo_save_gauss_info(cpl_table * ptable, cpl_imagelist* pCube)
{
	int sz = cpl_imagelist_get_size(pCube);
	int i;
	cpl_table_new_column(ptable, COL_NAME_ID, CPL_TYPE_INT);
	cpl_table_new_column(ptable, COL_NAME_CENTRE_X, CPL_TYPE_DOUBLE);
	cpl_table_new_column(ptable, COL_NAME_CENTRE_Y, CPL_TYPE_DOUBLE);
	for (i = 0; i < sz; i++)
	{
		cpl_image * pimage = cpl_imagelist_get(pCube, i);
		double    	 norm;
		double   	xcen;
		double   	ycen;
		double   	sig_x;
		double   	sig_y;
		double   	fwhm_x;
		double   	fwhm_y;
		cpl_error_code err = cpl_image_fit_gaussian(pimage, 32, 32, 15,
				&norm,
				&xcen,
				&ycen,
				&sig_x,
				&sig_y,
				&fwhm_x,
				&fwhm_y);
		if (err == CPL_ERROR_NONE && (!isnan(xcen)) && (!isnan(ycen)))
		{
			// write to the table
			cpl_table_set_int(ptable, COL_NAME_ID, i, i);
			cpl_table_set(ptable, COL_NAME_CENTRE_X, i, xcen);
			cpl_table_set(ptable, COL_NAME_CENTRE_Y, i, ycen);
		}
		else
		{
			//cpl_msg_warning(cpl_func, "fit gaussian failed on the image #%d", i);
			cpl_error_reset();
		}
	}

//	sinfo_fit_poly(ptable,COL_NAME_ID, COL_NAME_CENTRE_X, 3);
//	sinfo_fit_poly(ptable,COL_NAME_ID, COL_NAME_CENTRE_Y, 3);


}
static double atmo_get_general_lambda0(const char* fits_file)
{
	int centpix 	= 0;
	double lambda0	= 0;
    double Tc 		= 0;
    double rh 		= 0;
    double airm 	= 0;
    double p 		= 0;
    double parallactic = 0;
    double pixelscale = 0;
    double pixelsz = 0;
    /*cpl_error_code err = */atmo_disp_load_params(fits_file, &centpix, &lambda0, &Tc, &rh, &airm, &p, &parallactic, &pixelscale, &pixelsz);
    return lambda0;
}
static void atmo_test_dispersion_cube( const char* fits_file, const char* out_file)
{
	int centpix 	= 0;
	double lambda0	= 0;
    double Tc 		= 0;
    double rh 		= 0;
    double airm 	= 0;
    double p 		= 0;
    double parallactic = 0;
    double pixelscale = 0;
    double pixelsz = 0;
    cpl_table* ptable = 0;
    cpl_table* ptableresult = 0;
    cpl_imagelist* pCube = 0;
    cpl_propertylist* plist = 0;
    cpl_error_code err = atmo_disp_load_params(fits_file, &centpix, &lambda0, &Tc, &rh, &airm, &p, &parallactic, &pixelscale, &pixelsz);
    cpl_test(err == CPL_ERROR_NONE);
    // read the cube
    pCube = cpl_imagelist_load(fits_file, CPL_TYPE_FLOAT, 0);
    ptable = cpl_table_new(cpl_imagelist_get_size(pCube));
    atmo_save_gauss_info(ptable, pCube);
    cpl_msg_warning(cpl_func, "atmo_test_dispersion_cube parallactic[%f] pixescale[%f]", parallactic, pixelscale);
    err = sinfo_atm_dispersion_cube(
    		 pCube,
       		centpix, // central plane in the cube
       		lambda0, // wavelength of the central plane
    		Tc, // temperature in Celsius TEL.AMBI.TEMP
    		rh, // relative humidity in % TEL.AMBI.RHUM
    		airm, // airmass for the moment of observation TEL.AMBI.PRES
    		p, // atmospheric pressure TEL.AMBI.PRES
    		parallactic, // TEL.PARANG
    		pixelscale, // couble be for SINFONI 0.025, 0.100, 0.250
    		pixelsz);
    ptableresult = cpl_table_new(cpl_imagelist_get_size(pCube));
    atmo_save_gauss_info(ptableresult, pCube);
    cpl_test(err == CPL_ERROR_NONE);
    plist = cpl_propertylist_load(fits_file, 0);
 //   cpl_imagelist_save(pCube, out_file, CPL_BPP_IEEE_DOUBLE, plist, CPL_IO_CREATE);
    cpl_table_save(ptable, NULL, NULL, "origin.fits", CPL_IO_CREATE);
    cpl_table_save(ptableresult, NULL, NULL, out_file, CPL_IO_CREATE);
    cpl_propertylist_delete(plist);
    cpl_imagelist_delete(pCube);
    cpl_table_delete(ptable);
    cpl_table_delete(ptableresult);
}
static void atmo_apply_cube_polynomial_shift(cpl_polynomial* poly, const char* fits_file, const char* out_file)
{
	cpl_error_code 	err = CPL_ERROR_NONE;
    cpl_imagelist* pCube = 0;
    double lambda0 = 0;
    double Tc = 0;
    double rh = 0;
	double airmass = 0;
	double p = 0;
	double parallactic = 0;
	double pixelscale = 0;
	double pixelsz = 0;
	cpl_vector* vparams = 0;
	int centpix = 0;
	double * kernel = sinfo_generate_interpolation_kernel("default");


	err = atmo_disp_load_params(fits_file, &centpix, &lambda0, &Tc, &rh, &airmass, &p, &parallactic, &pixelscale, &pixelsz);
	cpl_msg_warning(cpl_func, "centpix[%d] lambda0[%f] angle[%f]", centpix, lambda0, parallactic);
	parallactic = parallactic * PI_NUMB / 180; // convert to radian
    pCube = cpl_imagelist_load(fits_file, CPL_TYPE_FLOAT, 0);

    if (pCube && (err == CPL_ERROR_NONE))
    {
    	int cubesize = 0;
    	int i = 0;
    	double l0_shift_x = 0; // shift for the central point by X
    	double l0_shift_y = 0; // shift for the central point by Y

    	vparams = cpl_vector_new(2);
    	cpl_vector_set(vparams, 0, airmass);
		cpl_vector_set(vparams, 1, lambda0);

		l0_shift_y = cpl_polynomial_eval(poly, vparams); // North - South
		l0_shift_x = 0; // (EAST-WEST direction)

		// rotate the shift
   		atmo_rotate_point(&l0_shift_x, &l0_shift_y, parallactic);


    	cpl_test(err == CPL_ERROR_NONE);
        cubesize = cpl_imagelist_get_size(pCube);
    	for (i = 0; i < cubesize; i++)
    	{
    		cpl_image* plane = cpl_imagelist_get(pCube, i);
    		cpl_image* pimresult = 0;
    		// calculate the wavelength
    		double lambda = lambda0 - (centpix - i) * pixelsz;
    		double shift_y = 0;
    		double shift_x = 0;

    		cpl_vector_set(vparams, 1, lambda);
    		// calc the shift
    		shift_y = cpl_polynomial_eval(poly, vparams); // North - South
    		shift_x = 0; // (EAST-WEST direction)

    		// rotate the shift
    		atmo_rotate_point(&shift_x, &shift_y, parallactic);
    		// apply shift
    		pimresult = sinfo_new_shift_image(
    		    plane,
    		    -(shift_x - l0_shift_x),
    		    -(shift_y - l0_shift_y),
    		    kernel);
    		err = cpl_imagelist_set(pCube, pimresult, i);
    		if (err != CPL_ERROR_NONE)
    			break;
    	}
    }
    else
    {
    	cpl_msg_warning(cpl_func, "Cannot load the cube [%s]", fits_file);
    }
	if (err != CPL_ERROR_NONE)
	{
		cpl_msg_warning(cpl_func, "An error during shifting image");
	}
	else
	{
		cpl_propertylist* plist = 0;
		plist = cpl_propertylist_load(fits_file, 0);
		sinfo_new_convert_0_to_ZERO_for_cubes(pCube);
		cpl_imagelist_save(pCube, out_file, CPL_BPP_IEEE_DOUBLE, plist, CPL_IO_CREATE);
		cpl_propertylist_delete(plist);
	}
	cpl_imagelist_delete(pCube);
    cpl_vector_delete(vparams);
	cpl_free(kernel);
}

static void atmo_init_offset_table(cpl_table** big_table, int big_table_size)
{
	*big_table = cpl_table_new(big_table_size);
	cpl_table_new_column(*big_table, COL_NAME_CENTRE_X, CPL_TYPE_DOUBLE);
	cpl_table_new_column(*big_table, COL_NAME_CENTRE_Y, CPL_TYPE_DOUBLE);
	cpl_table_new_column(*big_table, COL_NAME_AIRMASS, CPL_TYPE_DOUBLE);
	cpl_table_new_column(*big_table, COL_NAME_WAVELENGTH, CPL_TYPE_DOUBLE);
	cpl_table_new_column(*big_table, COL_NAME_ANGLE, CPL_TYPE_DOUBLE);
	cpl_table_new_column(*big_table, COL_NAME_OFFSET_X, CPL_TYPE_DOUBLE);
	cpl_table_new_column(*big_table, COL_NAME_OFFSET_Y, CPL_TYPE_DOUBLE);
}
static void atmo_process_fits_data_file(cpl_table** ptable, const char* filename)
{
	cpl_error_code err = CPL_ERROR_NONE;
	// open the file
	const int CUTOFF_PLANES = 80;
	int centpix 	= 0;
	double lambda0	= 0;
	double Tc 		= 0;
	double rh 		= 0;
	double airm 	= 0;
	double p 		= 0;
	double parallactic = 0;
	double pixelscale = 0;
	double pixelsz = 0;
	int row_lambda0 = -1;
	int i = 0;
	int nrow = 0;
	int cubesize = 0;
//	cpl_table* ptableresult = 0;
	cpl_imagelist* pCube = 0;
	// read the airmass from the file

	err = atmo_disp_load_params(filename, &centpix, &lambda0, &Tc, &rh, &airm, &p, &parallactic, &pixelscale, &pixelsz);
	if(err != CPL_ERROR_NONE)
	{
		cpl_msg_warning(cpl_func, "error [%s]", filename);
		cpl_error_reset();
		return;
	}
    pCube = cpl_imagelist_load(filename, CPL_TYPE_FLOAT, 0);
    cubesize = cpl_imagelist_get_size(pCube);
    atmo_init_offset_table(ptable, cubesize);
	for (i = CUTOFF_PLANES; i < cubesize - CUTOFF_PLANES; i++)
	{
		cpl_image* plane = cpl_imagelist_get(pCube, i);
		// calculate the wavelength
		double lambda = lambda0 - (centpix - i) * pixelsz;
		// for each wavelength
		// Gaussian fit for x and y
		// write row to the table:
		// wavelength, airmass, x_offset, y_offset, sourcefilename

		double    	norm;
		double   	xcen;
		double   	ycen;
		double   	sig_x;
		double   	sig_y;
		double   	fwhm_x;
		double   	fwhm_y;

		err = cpl_image_fit_gaussian(plane, 32, 32, 32,
				&norm,
				&xcen,
				&ycen,
				&sig_x,
				&sig_y,
				&fwhm_x,
				&fwhm_y);
//		cpl_msg_warning(cpl_func, "xcen[%f] ycen[%f] lambda[%f] airm[%f] ", xcen,ycen,lambda,airm );
		if (err == CPL_ERROR_NONE && (!isnan(xcen)) && (!isnan(ycen)))
		{
			// write to the table
			cpl_table_set(*ptable, COL_NAME_CENTRE_X, i, xcen -1);
			cpl_table_set(*ptable, COL_NAME_CENTRE_Y, i, ycen -1);
			cpl_table_set(*ptable, COL_NAME_OFFSET_X, i, xcen -1);
			cpl_table_set(*ptable, COL_NAME_OFFSET_Y, i, ycen -1);
			cpl_table_set(*ptable, COL_NAME_WAVELENGTH, i, lambda);
			cpl_table_set(*ptable, COL_NAME_AIRMASS, i, airm);
			cpl_table_set(*ptable, COL_NAME_ANGLE, i, parallactic);
			if (fabs(lambda - lambda0) < EPSILON)
			{
				row_lambda0 = i;
			}
			nrow++;
		}
		else
		{
//			cpl_msg_warning(cpl_func, "fit gaussian failed on the file[%s]image #%d", filename, i);
			cpl_error_reset();
		}

		/*if (i % 10 == 0)
			cpl_msg_warning(cpl_func, " shift image #%d, size[%d:%d] dx[%f] dy[%f]", i, szx, szy, shiftx, shifty);*/

	}
	if (row_lambda0 > 0 && (nrow > (cubesize / 3)))
	{
		double x_median = 0;
		double y_median = 0;
		int isnull = 0;
		atmo_rotate_offset(*ptable, -parallactic * PI_NUMB / 180);
		x_median = cpl_table_get(*ptable, COL_NAME_OFFSET_X, row_lambda0, &isnull);
		y_median = cpl_table_get(*ptable, COL_NAME_OFFSET_Y, row_lambda0, &isnull);
		cpl_table_erase_invalid_rows(*ptable);
		cpl_table_add_scalar(*ptable, COL_NAME_OFFSET_X, -x_median);
		cpl_table_add_scalar(*ptable, COL_NAME_OFFSET_Y, -y_median);

	}
	else
	{
		// the values for lambda0 was not computed - remove the table
		cpl_msg_warning(cpl_func, "---[%s] is not taken",filename );
		cpl_table_select_all(*ptable);
		cpl_table_erase_selected(*ptable);
	}
	cpl_imagelist_delete(pCube);
}
static void atmo_rotate_offset(cpl_table* ptable, double rot_angle)
{
	int i = 0;
	int nrow = cpl_table_get_nrow(ptable);

	for (i = 0; i < nrow; i++)
	{
		int isnull = 0;
		double x_value = cpl_table_get(ptable,COL_NAME_CENTRE_X, i, &isnull );
		double y_value = cpl_table_get(ptable,COL_NAME_CENTRE_Y, i, &isnull );
		if (isnull == 0)
		{
			atmo_rotate_point(&x_value, &y_value, rot_angle);
			cpl_table_set(ptable,COL_NAME_OFFSET_X, i, x_value);
			cpl_table_set(ptable,COL_NAME_OFFSET_Y, i, y_value);
		}
	}
}

static void atmo_rotate_point(double* x_value, double* y_value, double rot_angle)
{
	double newx = *x_value * cos(rot_angle) - *y_value * sin(rot_angle);
	double newy = *x_value * sin(rot_angle) + *y_value * cos(rot_angle);
	*x_value = newx;
	*y_value = newy;
}
static void atmo_prepare_offset_table(const char* cfg_filename, const char* output_filename)
{
	FILE * pFile;
    char fitsfilename [1024];
    cpl_table* tables[1500];
    cpl_table* big_table = 0;
    int next_table = 0;
    int big_table_size = 0;
    int next_row = 0;
    pFile = fopen (cfg_filename , "r");
    memset(&tables[0], 0, sizeof(cpl_table*) * 200);
	if (pFile == NULL)
	{
		perror ("Error opening file");
		return;
	}
	else
	{
		double lambda0 = 0;
		int isLambda0 = 0;
		// prepare table
		while (fgets (fitsfilename , 1024 , pFile))
		{
			if(fitsfilename[0] != '#')
			{
				cpl_table** pptable = 0;
				if (fitsfilename[strlen(fitsfilename)-1] != 's') // s means "fits"
				{
					// remove end-of-line
					fitsfilename[strlen(fitsfilename)-1] = 0;
				}
				cpl_msg_warning(cpl_func, "Process file: [%s]", fitsfilename);
				if (isLambda0 == 0)
				{
					isLambda0 = 1;
					lambda0 = atmo_get_general_lambda0(fitsfilename);
				}
				pptable = &(tables[next_table++]);
				atmo_process_fits_data_file(pptable, fitsfilename);
				if (*pptable)
				{
					big_table_size += cpl_table_get_nrow(*pptable);
				}
				else
				{
					next_table--;
				}
//				cpl_msg_warning(cpl_func, "table size[%d]", big_table_size);
			}

		}
		fclose (pFile);
	}

	// integrate and save the tables
	cpl_msg_warning(cpl_func, "------------integrate and save the tables");
	atmo_init_offset_table(&big_table, big_table_size);
	next_row = 0;
	for (int i = 0; i < next_table; i++)
	{
		int nrow = 0;
		int j = 0;
		cpl_table* ptable = tables[i];
		if (ptable == 0)
		{
			cpl_msg_warning(cpl_func, "end of file reached");
			break;
		}
		nrow = cpl_table_get_nrow(ptable);

		for ( j = 0; j < nrow; j++)
		{
			int inull;
			// read from the table
			double lambda = cpl_table_get(ptable,COL_NAME_WAVELENGTH , j, &inull);
			double airmas = cpl_table_get(ptable,COL_NAME_AIRMASS , j, &inull);
			double x_center = cpl_table_get(ptable,COL_NAME_CENTRE_X , j, &inull);
			double y_center = cpl_table_get(ptable,COL_NAME_CENTRE_Y , j, &inull);
			double x_offset = cpl_table_get(ptable,COL_NAME_OFFSET_X , j, &inull);
			double y_offset = cpl_table_get(ptable,COL_NAME_OFFSET_Y , j, &inull);
			double angle = cpl_table_get(ptable,COL_NAME_ANGLE , j, &inull);
			// write to the table
			cpl_table_set(big_table, COL_NAME_OFFSET_X, next_row, x_offset);
			cpl_table_set(big_table, COL_NAME_OFFSET_Y, next_row, y_offset);
			cpl_table_set(big_table, COL_NAME_CENTRE_X, next_row, x_center);
			cpl_table_set(big_table, COL_NAME_CENTRE_Y, next_row, y_center);
			cpl_table_set(big_table, COL_NAME_WAVELENGTH, next_row, lambda);
			cpl_table_set(big_table, COL_NAME_AIRMASS, next_row, airmas);
			cpl_table_set(big_table, COL_NAME_ANGLE, next_row, angle);
			next_row++;
		}
		cpl_table_delete(ptable);
	}
	// save the big table
	cpl_table_save(big_table, NULL, NULL, output_filename, CPL_IO_CREATE);
	cpl_table_delete(big_table);
	// release the tables
}
static void atmo_filter_resultset(double value_min, double value_max,
		const char* column_name,
		const char* fits_file_input,
		const char* fits_file_output)
{
	const char* COLS[] = {
			COL_NAME_CENTRE_X,
			COL_NAME_CENTRE_Y,
			COL_NAME_AIRMASS,
			COL_NAME_WAVELENGTH,
			COL_NAME_ANGLE,
			COL_NAME_OFFSET_X,
			COL_NAME_OFFSET_Y,
	};
	cpl_table* ptable = 0;
	int nrow = 0;
	cpl_table* ptableresult = 0;
	static int ncols = sizeof(COLS) / sizeof(COLS[0]);
	// open the file
	ptable = cpl_table_load(fits_file_input, 1,0);
	if (ptable == 0)
	{
		cpl_msg_warning(cpl_func, "cannot load table from file[%s]", fits_file_input);
		return;
	}
	nrow = cpl_table_get_nrow(ptable);
	atmo_init_offset_table(&ptableresult, nrow);
	if (ptableresult)
	{
		int i;
		int result_pos = 0;
		for (i = 0; i < nrow; i++)
		{
			int inull = 0;
			double value = cpl_table_get(ptable, column_name, i, &inull);
			if (inull == 0 && value <= value_max && value >= value_min)
			{
				// copy values to the new table
				int j;
				for (j = 0; j < ncols; j++)
				{
					double dvalue = cpl_table_get(ptable, COLS[j], i, &inull);
					cpl_table_set(ptableresult, COLS[j], result_pos, dvalue);
				}
				result_pos++;
			}
		}
		cpl_table_erase_invalid_rows(ptableresult);
		cpl_table_save(ptableresult, NULL, NULL,fits_file_output, CPL_IO_CREATE);
	}
	else
	{
		cpl_msg_warning(cpl_func, "cannot create result table");
	}
	cpl_table_delete(ptable);
	cpl_table_delete(ptableresult);
}
static void atmo_prepare_fit_025_HK(const char* input_filename, const char* output_filename)
{
	const double AIRMASS_MIN = 1.07;
	const double AIRMASS_MAX = 2.25;
	const double WAVELENGTH_MIN = 1.45;
	const double WAVELENGTH_MAX = 2.5;
	const double OFFSET_Y_MIN = -7;
	const double OFFSET_Y_MAX = 5;
	atmo_filter_resultset(AIRMASS_MIN,AIRMASS_MAX, COL_NAME_AIRMASS, input_filename, output_filename);
	atmo_filter_resultset(WAVELENGTH_MIN,WAVELENGTH_MAX, COL_NAME_WAVELENGTH, output_filename, output_filename);
	atmo_filter_resultset(OFFSET_Y_MIN,OFFSET_Y_MAX, COL_NAME_OFFSET_Y, output_filename, output_filename);
}
static void atmo_prepare_fit_025_J(const char* input_filename, const char* output_filename)
{
	const double OFFSET_Y_MIN = -6;
	const double OFFSET_Y_MAX = 4;
	atmo_filter_resultset(OFFSET_Y_MIN,OFFSET_Y_MAX, COL_NAME_OFFSET_Y, input_filename, output_filename);
}
static void atmo_filter_angle(const char* input_filename, const char* output_filename, double min, double max)
{
	atmo_filter_resultset(min,max, COL_NAME_ANGLE, input_filename, output_filename);
}

static cpl_polynomial* atmo_prepare_polyfit(const char* input_filename)
{
	cpl_polynomial  * fit2d = 0;
	cpl_matrix      * points = 0;
	cpl_vector      * fitvals = 0;
	cpl_error_code    err = CPL_ERROR_NONE;
	cpl_size maxdeg2d = 3;
	cpl_table* ptable;

	ptable = cpl_table_load(input_filename, 1,0);
	if (ptable)
	{
		int nrow = 0;
		int i = 0;
		int inull = 0;
		nrow = cpl_table_get_nrow(ptable);
		points = cpl_matrix_new(2, nrow);
		fitvals = cpl_vector_new(nrow);
		for (i = 0; i < nrow; i++)
		{
			double airmass = cpl_table_get(ptable, COL_NAME_AIRMASS, i, &inull);
			double wavelength= cpl_table_get(ptable, COL_NAME_WAVELENGTH, i, &inull);
			double offsety 	= cpl_table_get(ptable, COL_NAME_OFFSET_Y, i, &inull);
			if (inull == 0)
			{
				cpl_matrix_set(points, 0, i,airmass );
				cpl_matrix_set(points, 1, i,wavelength );
				cpl_vector_set(fitvals, i, offsety);
			}
			else
			{
				cpl_msg_warning(cpl_func, "invalid element #%d", i);
			}
		}
		err = cpl_error_get_code();
		if (err == CPL_ERROR_NONE)
		{
			fit2d = cpl_polynomial_new(2);
			err = cpl_polynomial_fit(fit2d, points, NULL, fitvals, NULL, CPL_FALSE,
										 NULL, &maxdeg2d);
			if (err == CPL_ERROR_NONE)
			{
				cpl_polynomial_dump(fit2d, stdout);
			}
			else
			{
				cpl_msg_error(cpl_func, "error during polyfit");
				cpl_polynomial_delete(fit2d);
				fit2d = 0;
			}
		}
		else
		{
			cpl_msg_error(cpl_func, "error before polyfit");
		}
	}
	cpl_vector_delete(fitvals);
	cpl_matrix_delete(points);
	cpl_table_delete(ptable);
	return fit2d;
}
static void atmo_prepare_poly_net(cpl_polynomial* poly, const char* filename)
{
	const double AIRMASS_MIN = 1.0;
	const double AIRMASS_MAX = 2.4;
	const double WAVELENGTH_MIN = 1.4;
	const double WAVELENGTH_MAX = 2.5;
	const int N_AIRMASS = 100;
	const int N_WAVELENGTH = 1000;
	cpl_table* ptable = 0;
	int i, j;
	cpl_vector* vargs = 0;

	vargs = cpl_vector_new(2);
	ptable = cpl_table_new(N_AIRMASS * N_WAVELENGTH);
	cpl_table_new_column(ptable, COL_NAME_AIRMASS, CPL_TYPE_DOUBLE);
	cpl_table_new_column(ptable, COL_NAME_WAVELENGTH, CPL_TYPE_DOUBLE);
	cpl_table_new_column(ptable, COL_NAME_OFFSET_Y, CPL_TYPE_DOUBLE);
	for (i = 0; i < N_AIRMASS; i++)
	{
		double airmass =
			1.0* AIRMASS_MIN + i * (AIRMASS_MAX - AIRMASS_MIN) / N_AIRMASS;
		for (j = 0; j < N_WAVELENGTH; j++)
		{
			double wavelength =
				1.0 * WAVELENGTH_MIN +  j *(WAVELENGTH_MAX - WAVELENGTH_MIN) / N_WAVELENGTH;
			double result = 0;
			cpl_vector_set(vargs, 0, airmass);
			cpl_vector_set(vargs, 1, wavelength);
			result = cpl_polynomial_eval(poly, vargs);
			cpl_table_set(ptable, COL_NAME_AIRMASS, i * N_WAVELENGTH + j, airmass);
			cpl_table_set(ptable, COL_NAME_WAVELENGTH, i * N_WAVELENGTH + j, wavelength);
			cpl_table_set(ptable, COL_NAME_OFFSET_Y, i * N_WAVELENGTH + j, result);
		}
	}
	cpl_table_save(ptable, NULL, NULL,filename, CPL_IO_CREATE);
	cpl_table_delete(ptable);
	cpl_vector_delete(vargs);
}
static void atmo_apply_cube_polynomial_shift_list(cpl_polynomial* poly, const char* list_file)
{
	FILE * pFile;
    char fitsfilename [1024];
    pFile = fopen (list_file , "r");

    if (pFile == NULL)
 	{
 		perror ("Error opening file");
 		return;
 	}
 	else
 	{
 		// prepare table
 		while (fgets (fitsfilename , 1024 , pFile))
 		{
 			if(fitsfilename[0] != '#')
 			{
 				char output_file_name[1024];
 				memset(&output_file_name[0], 0, 1024);
 				if (fitsfilename[strlen(fitsfilename)-1] != 's') // s means "fits"
 				{
 					// remove end-of-line
 					fitsfilename[strlen(fitsfilename)-1] = 0;
 				}
 				// prepare outputfilename
 				sprintf(output_file_name, "output/%s.corrected.fits", fitsfilename);
 				cpl_msg_warning(cpl_func, "Process file: [%s]", fitsfilename);
 				atmo_apply_cube_polynomial_shift(poly, fitsfilename, output_file_name);
 			}

 		}
 		fclose (pFile);
 	}

}
static void atmo_get_poly_coeff(
		cpl_polynomial* poly,
		int dim ,
		int degree,
		cpl_size * exp,
		int step,
		cpl_table* ptable,
		int* curr_row)
{
	int i = 0;
	for (i = 0; i <= degree; i++)
	{
		exp[step] = i;
		if(step == dim - 1)
		{
			double coeff = cpl_polynomial_get_coeff(poly, exp);
			if (fabs(coeff) > EPSILON)
			{
				int j = 0;
				for (j = 0; j < dim; j++)
				{
					char col_name[255];
					sprintf(col_name, "col_%d", j);
					cpl_table_set_int(ptable, col_name, *curr_row, exp[j]);
				}
				cpl_table_set(ptable, "value", *curr_row, coeff);
				(*curr_row)++;
			}
		}
		else
		{
			atmo_get_poly_coeff(poly, dim, degree - i, exp, step + 1,
					ptable, curr_row);
		}
	}
}
static void atmo_save_polynom(cpl_polynomial* poly,
		const char* filename,
		cpl_propertylist* plist)
{
	cpl_table* ptable = 0;
	int dim = 0;
	int degree = 0;
	cpl_size* exp = 0;
	int i = 0;
	int curr_row = 0;
	if (!poly)
		return;
	if (!filename)
		return;

	dim = cpl_polynomial_get_dimension(poly);
	degree = cpl_polynomial_get_degree(poly);
	exp = cpl_malloc(dim * sizeof(exp[0]));
	memset(&exp[0], 0, dim * sizeof(exp[0]));

	ptable = cpl_table_new(pow(degree + 1, dim));
	for (i = 0; i < dim; i++)
	{
		char col_name[255];
		sprintf(col_name, "col_%d", i);
		cpl_table_new_column(ptable, col_name, CPL_TYPE_INT);
	}
	cpl_table_new_column(ptable, "value", CPL_TYPE_DOUBLE);
	atmo_get_poly_coeff(poly, dim, degree, exp,0, ptable, &curr_row);
	cpl_table_erase_invalid_rows(ptable);
	cpl_table_save(ptable, plist, NULL, filename, CPL_IO_CREATE);
	cpl_table_delete(ptable);
	cpl_free(exp);
}

static cpl_polynomial* atmo_load_polynom(const char* filename)
{
	cpl_polynomial* poly = 0;
	cpl_table* ptable = 0;

	ptable = cpl_table_load(filename, 1, 0);
	if (ptable)
	{
		int dim = 0;
		int nrows = 0;
		//int i = 0;
		cpl_size* exp = 0;

		dim = cpl_table_get_ncol(ptable) - 1;
		poly = cpl_polynomial_new(dim );
		nrows = cpl_table_get_nrow(ptable);
		exp = cpl_malloc(dim * sizeof(exp[0]));
		memset(&exp[0], 0, dim * sizeof(exp[0]));
		for (int i = 0; i < nrows; i++)
		{
			int j = 0;
			int inull = 0;
			double value = 0;
			for (j = 0; j < dim; j++)
			{
				char col_name[255];
				sprintf(col_name, "col_%d", j);
				exp[j] = cpl_table_get_int(ptable, col_name, i, &inull);
			}
			value = cpl_table_get(ptable, "value", i, &inull);
			cpl_polynomial_set_coeff(poly, exp, value);
		}
		cpl_free(exp);
	}
	cpl_table_delete(ptable);
	return poly;
}
int frame_cmp(cpl_frame* frame1, cpl_frame* frame2)
{
	return 1; //(frame1 == frame2) ? 1 : 0;
}
int main(int argc, char* argv[])
{
	cpl_polynomial * poly = 0;
	cpl_test_init(SINFO_STAR_BUG_REPORT, CPL_MSG_WARNING);
	/*********************************
	 */
	/*{
		int ncount = 10;
		int i = 0;
		int nb_labels = 0;
		int* selection = 0;
		cpl_frameset* p = cpl_frameset_new();
		cpl_frameset* p_extracted = 0;
		for (i = 0; i < ncount; i++)
		{
			cpl_frame* pframe = cpl_frame_new();
			cpl_frameset_insert(p, pframe);
		}
		selection = cpl_frameset_labelise(p, frame_cmp, &nb_labels);
		for (i = -10; i < ncount + 10; i++)
		{
			p_extracted = cpl_frameset_extract(p, selection, i);
			cpl_frameset_delete(p_extracted);
		}
		cpl_frameset_delete(p);
		cpl_free(selection);
		return 0;
	}*/

	 /***************************/
	if (argc<3)
	{
		return 0;
	}
//	atmo_prepare_offset_table(argv[1], argv[2]);
//	atmo_prepare_fit_025_J(argv[1], argv[2]);
//	atmo_filter_angle(argv[1], argv[2], -150, 20);
	poly = atmo_prepare_polyfit(argv[1]);
	if (poly)
	{
		cpl_propertylist* plist = cpl_propertylist_new();
		cpl_propertylist_append_string(plist, "HIERARCH ESO INS OPTI1 NAME", "0.025");
		atmo_save_polynom(poly, argv[2], plist);
/*		cpl_polynomial* poly_saved = atmo_load_polynom(argv[2]);
		if (cpl_polynomial_compare(poly, poly_saved, EPSILON) == 0)
		{
			cpl_msg_warning(cpl_func, "Polynomials are the same");
		}
		else
		{
			cpl_msg_warning(cpl_func, "Polynomials are different");
		}*/
		atmo_prepare_poly_net(poly, argv[3]);
//		atmo_apply_cube_polynomial_shift_list(poly, argv[2]);
//		atmo_apply_cube_polynomial_shift(poly, argv[2], argv[3]);*/
		cpl_polynomial_delete(poly);
//		cpl_polynomial_delete(poly_saved);
		cpl_propertylist_delete(plist);
	}

//	atmo_prepare_offset_table("/diskb/kmirny/sinfoni/atm_disp/250/250.input", "/diskb/kmirny/sinfoni/atm_disp/250/250.result.fits");
//	atmo_prepare_offset_table(argv[1], argv[2]);
//	atmo_test_dispersion_cube("/diskb/kmirny/sinfoni/sinfo_demo/pro/sinfo_rec_jitter_stdstar/out_0013.fits",
//			"out_0013a.fits");
//	atmo_test_dispersion_cube("/diskb/kmirny/sinfoni/atm_dispersion/SI_SCDJ_291811_2008-02-07T03:36:17.586_5_H+K_025.fits",
//			"/diskb/kmirny/sinfoni/atm_dispersion/SSI_SCDJ_291811_2008-02-07T03:36:17.586_5_H+K_025_result.fits");
//		atmo_test_dispersion_cube("/diskb/kmirny/sinfoni/atm_dispersion/SI_PCST_080124A_H+K_025.fits",
//				"/diskb/kmirny/sinfoni/atm_dispersion/SI_PCST_080124A_H+K_025_result.fits");
//	atmo_test_dispersion_cube("/diskb/kmirny/sinfoni/atm_dispersion/diffract_std/SI_PCST_050512A_H+K_025.fits",
//			"/diskb/kmirny/sinfoni/atm_dispersion/diffract_std/SI_PCST_050512A_H+K_025_result.fits");

	return cpl_test_end(0);
}
