/* $Id: imcore_background.c,v 1.4 2015/08/12 11:16:55 jim Exp $
 *
 * This file is part of the CASU Pipeline utilities
 * Copyright (C) 2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jim $
 * $Date: 2015/08/12 11:16:55 $
 * $Revision: 1.4 $
 * $Name:  $
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>

#include <cpl.h>

#include "floatmath.h"
#include "util.h"
#include "imcore.h"

static int **hist = NULL;
static int *nnp = NULL;
static int npvx;
static int npvy;
static void tidy(void);
static void sortit (float [], int);

/**@{*/

/*---------------------------------------------------------------------------*/
/**
    \ingroup cataloguemodules
    \brief Model and create background map
  
    \par Name:
        imcore_background
    \par Purpose:
        Model and create background map
    \par Description:
        The image data array is split into cells. In each cell a robust 
        background estimate is obtained. The cell raster is gently smoothed
        and then used to create a full background map with a bi-linear
        interpolation scheme.
    \par Language:
        C
    \param ap
        The current ap structure
    \param nbsize
        The size of the cells in pixels
    \param nullval
        A null value used to flag bad pixels
    \retval CASU_OK
        If all went well. This is currently the only value.
    \par QC headers:
        None
    \par DRS headers:
        None
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int imcore_background(ap_t *ap, int nbsize, float nullval) {
    float fracx,fracy,skymed,sigma,skymedc,sigmac,avsky,fnbsize,dely,delx;
    float t1,t2,dsky,*map,**bvals,*work;
    int ifracx,ifracy,nbsizx,nbsizy,nbx,nby,npixstripe,l,i,ll;
    int isquare,ilev,j,iclip,mcpix,iloop,irej,nbsizo2,kk,k,iby,ibyp1,ibx,ibxp1;
    int *shist;
    unsigned char *mflag,*bbad;
    long nx,ny;

    /* Set up some variables */

    map = ap->indata;
    mflag = ap->mflag;
    nx = ap->lsiz;
    ny = ap->csiz;

    /* check to see if nbsize is close to exact divisor */

    fracx = ((float)nx)/((float)nbsize);
    fracy = ((float)ny)/((float)nbsize);
    ifracx = (int)(fracx + 0.1);
    ifracy = (int)(fracy + 0.1);
    nbsizx = nx/ifracx;
    nbsizy = ny/ifracy;
    nbsize = MAX(NINT(0.9*nbsize), MIN(nbsize, MIN(nbsizx,nbsizy)));
    nbsize = MIN(nx,MIN(ny,nbsize)); /* trap for small maps */

    /* Divide the map into partitions */

    nbx = nx/nbsize;
    nby = ny/nbsize;
    npixstripe = nbsize*nx;
    npvx = nbx;
    npvy = nby;

    /* Get histogram workspace if you can */

    hist = cpl_malloc(nbx*sizeof(int *));
    for (l = 0; l < nbx; l++) 
        hist[l] = cpl_malloc(MAXHIST*sizeof(int));

    /* Same for background values array */

    bvals = cpl_malloc(nby*sizeof(float *));
    for (l = 0; l < nby; l++) 
        bvals[l] = cpl_malloc(nbx*sizeof(float));
    bbad = cpl_calloc(nbx*nby,sizeof(unsigned char));

    /* Store some of this away for use later */

    ap->backmap.nbx = nbx;
    ap->backmap.nby = nby;
    ap->backmap.nbsize = nbsize;
    ap->backmap.bvals = bvals;

    /* Finally a counter array */

    nnp = cpl_malloc(nbx*sizeof(int));

    /* Loop for each row of background squares. Start by initialising
       the accumulators and histograms */

    for (l = 0; l < nby; l++) {
        memset((char *)nnp,0,nbx*sizeof(*nnp));
        for (i = 0; i < nbx; i++) 
            memset((char *)hist[i],0,MAXHIST*sizeof(int));

        /* Skim through the data in this stripe. Find out which square each
           belongs to and add it it to the relevant histogram */

        ll = l*npixstripe;
        for (i = 0; i < npixstripe; i++) {
            if (map[ll+i] != nullval && mflag[ll+i] != MF_ZEROCONF &&
                mflag[ll+i] != MF_STUPID_VALUE) {
                isquare = (int)((float)(i % nx)/(float)nbsize);
                isquare = MIN(nbx-1,MAX(0,isquare));
                ilev = MIN(MAXHISTVAL,MAX(MINHISTVAL,NINT(map[i+ll])));
                hist[isquare][ilev-MINHISTVAL] += 1;
                nnp[isquare] += 1;
            }
        }
  
        /* but only do background estimation if enough pixels ----------- */

        for (j = 0; j < nbx; j++) {
            if (nnp[j] > 0.25*nbsize*nbsize){
                shist = hist[j];
                imcore_medsig(shist,MAXHIST,MINHISTVAL-1,nnp[j],&skymed,
                              &sigma);

                /* do an iterative 3-sigma upper clip to give a more robust
                   estimator */

                iclip = MAXHISTVAL;
                mcpix = nnp[j];
                skymedc = skymed;
                sigmac = sigma;
                for (iloop = 0; iloop < 3; iloop++) {
                    irej = 0;
                    for(i = NINT(skymedc+3.0*sigmac); i <= iclip; i++)
                        irej += shist[i-MINHISTVAL];
                    if (irej == 0)
                        break;
                    iclip = NINT(skymedc+3.0*sigmac) - 1;
                    mcpix = mcpix - irej;
                    imcore_medsig(shist,MAXHIST,MINHISTVAL-1,mcpix,
                                  &skymedc,&sigmac);
                }
                bvals[l][j] = skymedc;
            } else {
                bvals[l][j] = -1000.0;
                bbad[j+nbx*l] = 1;
            }
        }
    }

    /* filter raw background values */

    imcore_bfilt(bvals,nbx,nby);

    /* compute average sky level */

    work = cpl_malloc(nbx*nby*sizeof(*work));
    k = 0;
    for(l = 0; l < nby; l++)
        for(j = 0; j < nbx; j++) 
            if (! bbad[l*nbx+j])
                work[k++] = bvals[l][j];
    sortit(work,k);
    avsky = work[(k)/2];
    freespace(work);
    freespace(bbad);

    /* ok now correct map for background variations and put avsky back on */

    nbsizo2 = nbsize/2;
    fnbsize = 1.0/((float)nbsize);
    for (k = 0; k < ny; k++) {
        kk = k*nx;

        /* Nearest background pixel vertically */

        iby = (k + 1 + nbsizo2)/nbsize;
        ibyp1 = iby + 1;
        iby = MIN(nby,MAX(1,iby));
        ibyp1 = MIN(nby,ibyp1);
        dely = (k + 1 - nbsize*iby + nbsizo2)*fnbsize;

        for (j = 0; j < nx; j++) {
            if (map[kk+j] == nullval) 
                continue;

            /* nearest background pixel across */

            ibx = (j + 1 + nbsizo2)/nbsize;
            ibxp1 = ibx + 1;
            ibx = MIN(nbx,MAX(1,ibx));
            ibxp1 = MIN(nbx,ibxp1);
            delx = (j + 1 - nbsize*ibx + nbsizo2)*fnbsize;

            /* bilinear interpolation to find background */
            
            t1 = (1.0 - dely)*bvals[iby-1][ibx-1] + dely*bvals[ibyp1-1][ibx-1];
            t2 = (1.0 - dely)*bvals[iby-1][ibxp1-1] + dely*bvals[ibyp1-1][ibxp1-1];
            dsky = avsky - (1.0 - delx)*t1 - delx*t2;
            map[kk+j] += dsky;
        }
    }

    /* Free some workspace */

    tidy();
    return(CASU_OK);
}

/*---------------------------------------------------------------------------*/
/**
    \ingroup cataloguemodules
    \brief Work out robust background estimate over a whole input image
  
    \par Name:
        imcore_backstats
    \par Purpose:
        Work out a robust background estimate over a whole input image
    \par Description:
        The image is analysed to work out a robust estimate of the background
        median, sigma and saturation level.
    \par Language:
        C
    \param ap
        The current ap structure
    \param nullval
        A null value used to flag bad pixels
    \param satonly
        If set, then only the saturation level will be computed.
    \param skymed
        Output sky median
    \param skysig
        Output sky noise
    \param sat
        Output saturation level
    \retval CASU_OK
        If all went well. 
    \retval CASU_WARN
        If there aren't enough good values to do the calculation
    \par QC headers:
        None
    \par DRS headers:
        None
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern int imcore_backstats(ap_t *ap, float nullval, int satonly, 
                            float *skymed, float *skysig, float *sat) {
    int ilev,iclip,iloop,i,*ihist,isat,iter;
    long mpix,npix,k,mcpix,irej,lpix,nx,ny;
    float skymedc,sigmac,*map,sata,fac,skyref;
    unsigned char *mflag;

    /* Get some info from the ap structure */

    map = ap->indata;
    nx = ap->lsiz;
    ny = ap->csiz;
    mflag = ap->mflag;

    /* Check to make sure there are some non-zero values here */

    ilev = 1;
    for (i = 0; i < nx*ny; i++) {
        if (map[i] != nullval && mflag[i] != MF_ZEROCONF &&
            mflag[i] != MF_STUPID_VALUE) {
            ilev = 0;
            break;
        }
    }
    if (ilev == 1) {
        *skymed = 0.0;
        *skysig = 0.0;
        *sat = 0.0;
        return(CASU_WARN);
    }

    /* First, get some workspace for the background histogram */

    ihist = cpl_calloc(MAXHIST,sizeof(*ihist));

    /* Loop for up to 10 iterations. For each iteration we multiply the 
       input data by a successively higher power of 2 in order to 
       try and deal with data that has very small noise estimates */

    fac = 0.5;
    skyref = 0.0;
    for (iter = 0; iter <= 9; iter++) {
        fac *= 2.0;
        if (iter == 1)
            skyref = skymedc;
        for (k = 0; k < MAXHIST; k++)
            ihist[k] = 0;

        /* Now form the histogram of all pixel intensities */

        mpix = 0;
        isat = 0;
        npix = nx*ny;
        for (k = 0; k < npix; k++) {
            if (map[k] != nullval && mflag[k] != MF_ZEROCONF &&
                mflag[k] != MF_STUPID_VALUE) {
                ilev = MIN(MAXHISTVAL,MAX(MINHISTVAL,NINT(fac*(map[k]-skyref))));
                ihist[ilev - MINHISTVAL] += 1;
                isat = MAX(isat,ilev);
                mpix++;
            }
        }
        sata = MIN(MAXHISTVAL,MAX(MINSATURATE,0.9*((float)isat))/fac);
        lpix = ihist[isat - MINHISTVAL];
        while (lpix < mpix/1000 && isat > MINHISTVAL) {
            isat--;
            lpix += ihist[isat - MINHISTVAL];
        }
        *sat = ((float)isat)/fac + skyref;
        *sat = MIN(MAXHISTVAL,MAX(MINSATURATE,MAX(0.95*(*sat),sata)));

        /* If all you want is the saturation level, then get out of here...*/

        if (satonly) {
            freespace(ihist);
            return(CASU_OK);
        }

        /* Now find the median and sigma */

        imcore_medsig(ihist,MAXHIST,MINHISTVAL-1,mpix,skymed,skysig);

        /* Do an iterative 3-sigma upper clip to give a more robust 
           estimator */

        iclip = MAXHISTVAL;
        mcpix = mpix;
        skymedc = *skymed;
        sigmac = *skysig;
        for (iloop = 0; iloop < 3; iloop++) {
            irej = 0;
            for (i = NINT(skymedc+3.0*sigmac); i <= iclip; i++)
                irej += ihist[i - MINHISTVAL];
            if (irej == 0)
                break;
            iclip = NINT(skymedc+3.0*sigmac)-1;
            mcpix = mcpix-irej;
            imcore_medsig(ihist,MAXHIST,MINHISTVAL-1,mcpix,&skymedc,
                          &sigmac);
        }
        if (sigmac > 2.5)
            break;
    }

    /* Set the final answer */

    *skymed = skymedc/fac + skyref;
    *skysig = sigmac/fac;
    freespace(ihist);
    return(CASU_OK);
}   

/*---------------------------------------------------------------------------*/
/**
    \ingroup cataloguemodules
    \brief Work out estimated sky for a pixel position
  
    \par Name:
        imcore_backest
    \par Purpose:
        Work out estimated sky for a pixel position
    \par Description:
        Given the coarse background grid, calculate the background at a 
        given image pixel position by doing a bi-linear interpolation of
        it's position within the grid.
    \par Language:
        C
    \param ap
        The current ap structure
    \param x
        The X position in question
    \param y
        The Y position in question
    \param skylev
        Output sky level at x,y
    \param skyrms
        Output sky noise at x,y
    \returns
        Nothing
    \par QC headers:
        None
    \par DRS headers:
        None
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void imcore_backest(ap_t *ap, float x, float y, float *skylev, 
                           float *skyrms) {
    int i,j,nbx,nby,nbsize,nbsizo2,iby,ibyp1,ibx,ibxp1;
    float **bvals,fnbsize,dely,delx,t1,t2;

    /* Define some local variables */

    nbx = ap->backmap.nbx;
    nby = ap->backmap.nby;
    nbsize = ap->backmap.nbsize;
    bvals = ap->backmap.bvals;

    /* Get closest pixel to the input location */

    i = NINT(x);
    j = NINT(y);

    /* Now, work out where in the map to do the interpolation */

    nbsizo2 = nbsize/2;
    fnbsize = 1.0/((float)nbsize);
    iby = (j + nbsizo2)/nbsize;
    ibyp1 = iby + 1;
    iby = MIN(nby,MAX(1,iby));
    ibyp1 = MIN(nby,ibyp1);
    dely = (j  - nbsize*iby + nbsizo2)*fnbsize;
    ibx = (i + nbsizo2)/nbsize;
    ibxp1 = ibx + 1;
    ibx = MIN(nbx,MAX(1,ibx));
    ibxp1 = MIN(nbx,ibxp1);
    delx = (i - nbsize*ibx + nbsizo2)*fnbsize;

    /* Now do a linear interpolation to find the background. Calculate MAD of
       the four adjacent background cells as an estimate of the RMS */

    t1 = (1.0 - dely)*bvals[iby-1][ibx-1] + dely*bvals[ibyp1-1][ibx-1];
    t2 = (1.0 - dely)*bvals[iby-1][ibxp1-1] + dely*bvals[ibyp1-1][ibxp1-1];
    *skylev = (1.0 - delx)*t1 + delx*t2;
    *skyrms = 0.25*(fabsf(bvals[iby-1][ibx-1] - *skylev) +
                    fabsf(bvals[ibyp1-1][ibx-1] - *skylev) +
                    fabsf(bvals[iby-1][ibxp1-1] - *skylev) +
                    fabsf(bvals[ibyp1-1][ibxp1-1] - *skylev));
}

/*---------------------------------------------------------------------------*/
/**
    \ingroup cataloguemodules
    \brief Analyse histogram to work out median and sigma
  
    \par Name:
        imcore_medsig
    \par Purpose:
        Analyse histogram to work out median and sigma
    \par Description:
        Given a histogram work out the median and sigma of a distribution.
        A starting point in the histogram can be given, which allows you
        to ignore bins at the lower end. A target value defines at what point
        we stop summing the histogram. 
    \par Language:
        C
    \param shist
        The input histogram
    \param nh
        The number of bins in the histogram
    \param ist
        The first bin position that we will look at
    \param itarg
        The target value for the summation. 
    \param med
        Output median
    \param sig
        Output sigma from the first quartile.
    \returns
        Nothing
    \par QC headers:
        None
    \par DRS headers:
        None
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

extern void imcore_medsig(int *shist, int nh, int ist, int itarg, 
                          float *med, float *sig) {
    int isum, medata;
    float ffrac,sigmed;
 
    /* median */ 

    isum = 0;
    medata = ist;
    while (isum <= (itarg+1)/2 && (medata-MINHISTVAL) < nh) {
        medata++;
        isum += shist[medata-MINHISTVAL];
    }
    if (shist[medata-MINHISTVAL] == 0) {
        ffrac = 0.0;
    } else {
        ffrac = (float)(isum - (itarg+1)/2)/(float)shist[medata-MINHISTVAL];
    }
    *med = (float)medata - ffrac + 0.5;

    /* sigma */

    isum = 0;
    medata = ist;
    while (isum <= (itarg+3)/4 && (medata-MINHISTVAL) < nh) {
        medata++;
        isum += shist[medata-MINHISTVAL];
    }
    if (shist[medata-MINHISTVAL] == 0) {
        ffrac = 0.0;
    } else {
        ffrac = (float)(isum - (itarg+3)/4)/(float)shist[medata-MINHISTVAL];
    }
    sigmed = (float)medata - ffrac + 0.5;
    *sig = 1.48*(*med - sigmed);
    *sig = MAX(0.5,*sig);
}

/*---------------------------------------------------------------------------*/
/**
    \par Name:
        sortit
    \par Purpose:
        Sort a single array into ascending order
    \par Description:
        A single array is sorted into ascending order
    \par Language:
        C
    \param ia
        The input data array
    \param n
        The number of values in the data array
    \returns
        Nothing
    \author
        Jim Lewis, CASU
 */
/*---------------------------------------------------------------------------*/

static void sortit (float ia[], int n) {
    int i, j, ii, jj, ifin;
    float it;
 
    jj = 4;
    while (jj < n) 
        jj = 2 * jj;
    jj = MIN(n,(3 * jj)/4 - 1);
    while (jj > 1) {
        jj = jj/2;
        ifin = n - jj;
        for (ii = 0; ii < ifin; ii++) {
            i = ii;
            j = i + jj;
            if (ia[i] <= ia[j]) 
                continue;
            it = ia[j];
            do {
                ia[j] = ia[i];
                j = i;
                i = i - jj;
                if (i < 0) 
                    break;
            } while (ia[i] > it);
            ia[j] = it;
        }
    }
    return;
}


static void tidy(void) {
    int i;
 
    freespace(nnp);
    if (hist != NULL) {
        for (i = 0; i < npvx; i++) 
            freespace(hist[i]);
    }
    freespace(hist);
    return;
}

/**@}*/

/*

$Log: imcore_background.c,v $
Revision 1.4  2015/08/12 11:16:55  jim
Modified procedure names to protect namespace

Revision 1.3  2015/08/07 13:06:54  jim
Fixed copyright to ESO

Revision 1.2  2015/08/06 05:33:31  jim
Small modification to help with backgrounds where the vast majority of
the pixels are bad

Revision 1.1.1.1  2015/06/12 10:44:32  jim
Initial import

Revision 1.4  2014/04/09 11:08:21  jim
Get rid of a couple of compiler moans

Revision 1.3  2014/04/09 09:09:51  jim
Detabbed

Revision 1.2  2014/03/26 15:25:19  jim
Modified for floating point confidence maps

Revision 1.1.1.1  2013/08/27 12:07:48  jim
Imported


*/

