/*
 * © Copyright 1996-2017 ECMWF.
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 * In applying this licence, ECMWF does not waive the privileges and immunities
 * granted to it by virtue of its status as an intergovernmental organisation nor
 * does it submit to any jurisdiction.
 */

#include <ctype.h>
#include <errno.h>
#include <math.h>

#include <algorithm>
#include <cstring>
#include <fstream>
#include <memory>
#include <string>

#include "pproc.h"

#include "eckit/config/LibEcKit.h"
#include "eckit/exception/Exceptions.h"
#include "eckit/geometry/Point2.h"
#include "eckit/geometry/Sphere.h"
#include "eckit/linalg/LinearAlgebraDense.h"
#include "eckit/linalg/LinearAlgebraSparse.h"
#include "eckit/log/Log.h"
#include "eckit/runtime/Main.h"
#include "eckit/thread/AutoLock.h"
#include "eckit/thread/Mutex.h"

#include "mir/api/MIRJob.h"
#include "mir/api/mir_version.h"
#include "mir/config/LibMir.h"
#include "mir/input/GribMemoryInput.h"
#include "mir/input/VectorInput.h"
#include "mir/output/GribMemoryOutput.h"
#include "mir/output/VectorOutput.h"


namespace mir {
namespace {

static std::shared_ptr<api::MIRJob> job;

static std::string tidy(const char* in, bool to_lowercase = true, bool no_quotes = false) {
    std::string r(in);

    if (to_lowercase) {
        for (char& c : r) {
            c = (c == ' ' ? '-' : static_cast<char>(tolower(c)));
        }
    }

    if (no_quotes) {
        if (!r.empty() && r.front() == '"')
            r.erase(0, 1);
        if (!r.empty() && r.back() == '"')
            r.pop_back();
    }

    return r;
}


}  // anonymous namespace


int intf2(const void* grib_in,
          const int& length_in,
          void* grib_out,
          int& length_out) {

    eckit::Log::debug<LibMir>() << "++++++ intf2" << std::endl;

    try {

        input::GribMemoryInput input(grib_in, length_in);
        output::GribMemoryOutput output(grib_out, length_out);

        static const char* capture = getenv("MIR_CAPTURE_CALLS");
        if (capture) {
            std::ofstream out(capture);
            out << "mars<<EOF" << std::endl;
            out << "retrieve,target=in.grib,";
            input.marsRequest(out);
            out << std::endl;
            out << "EOF" << std::endl;
            job->mirToolCall(out);
            out << std::endl;
        }

        job->execute(input, output);

        ASSERT(output.interpolated() + output.saved() == 1);

        if (output.saved() == 1) {
            length_out = 0;  // Not interpolation performed
        }
        else {
            length_out = output.length();
        }
    }
    catch (std::exception& e) {
        eckit::Log::error() << "MIR: " << e.what() << std::endl;
        return -2;
    }

    return 0;
}


int intuvp3(const void* vort_grib_in,
            const void* div_grib_in,
            const int& vort_length_in,
            const int& div_length_in,
            void* u_grib_out,
            void* v_grib_out,
            int& length_out) {

    eckit::Log::debug<LibMir>() << "++++++ intuvp3" << std::endl;

    try {

        // Second order packing may return different sizes
        auto len_out = size_t(length_out);

        ::memset(u_grib_out, 0, len_out);
        ::memset(v_grib_out, 0, len_out);

        input::GribMemoryInput vort_input(vort_grib_in, size_t(vort_length_in));
        input::GribMemoryInput div_input(div_grib_in, size_t(div_length_in));

        output::GribMemoryOutput u_output(u_grib_out, len_out);
        output::GribMemoryOutput v_output(v_grib_out, len_out);

        input::VectorInput input(vort_input, div_input);
        output::VectorOutput output(u_output, v_output);

        job->set("vod2uv", true);


        static const char* capture = getenv("MIR_CAPTURE_CALLS");
        if (capture) {
            std::ofstream out(capture);
            out << "mars<<EOF" << std::endl;
            out << "retrieve,target=in.grib,";
            vort_input.marsRequest(out);
            out << std::endl;
            out << "retrieve,target=in.grib,";
            div_input.marsRequest(out);
            out << std::endl;
            out << "EOF" << std::endl;
            job->mirToolCall(out);
            out << std::endl;
        }

        job->execute(input, output);

        job->clear("vod2uv");

        ASSERT(u_output.interpolated() + u_output.saved() == 1);
        ASSERT(v_output.interpolated() + v_output.saved() == 1);

        // If packing=so, u and v will have different sizes
        // ASSERT(u_output.length() == v_output.length());
        length_out = int(std::max(u_output.length(), v_output.length()));

        // {
        //     eckit::StdFile f("debug.u", "w");
        //     fwrite(u_grib_out, 1, *length_out, f);
        // }
        // {
        //     eckit::StdFile f("debug.v", "w");
        //     fwrite(v_grib_out, 1, *length_out, f);
        // }
    }
    catch (std::exception& e) {
        eckit::Log::error() << "MIR: " << e.what() << std::endl;
        return -2;
    }

    return 0;
}


int intvect3(const void* u_grib_in,
             const void* v_grib_in,
             const int& u_length_in,
             const int& v_length_in,
             void* u_grib_out,
             void* v_grib_out,
             int& length_out) {
    eckit::Log::debug<LibMir>() << "++++++ intvect3" << std::endl;

    try {
        input::GribMemoryInput u_input(u_grib_in, size_t(u_length_in));
        input::GribMemoryInput v_input(v_grib_in, size_t(v_length_in));

        auto len_out = size_t(length_out);
        output::GribMemoryOutput u_output(u_grib_out, len_out);
        output::GribMemoryOutput v_output(v_grib_out, len_out);

        input::VectorInput input(u_input, v_input);
        output::VectorOutput output(u_output, v_output);

        job->set("uv2uv", true);

        static const char* capture = getenv("MIR_CAPTURE_CALLS");
        if (capture) {
            std::ofstream out(capture);
            out << "mars<<EOF" << std::endl;
            out << "retrieve,target=in.grib,";
            u_input.marsRequest(out);
            out << std::endl;
            out << "retrieve,target=in.grib,";
            v_input.marsRequest(out);
            out << std::endl;
            out << "EOF" << std::endl;
            job->mirToolCall(out);
            out << std::endl;
        }

        job->execute(input, output);

        job->clear("uv2uv");

        ASSERT(u_output.interpolated() + u_output.saved() == 1);
        ASSERT(v_output.interpolated() + v_output.saved() == 1);

        // In case u and v have different sizes
        // ASSERT(u_output.length() == v_output.length());
        length_out = int(std::max(u_output.length(), v_output.length()));
    }
    catch (std::exception& e) {
        eckit::Log::error() << "MIR: " << e.what() << std::endl;
        return -2;
    }

    return 0;
}


int intuvp2(const void* vort_grib_in,
            const void* div_grib_in,
            const int& length_in,
            void* u_grib_out,
            void* v_grib_out,
            int& length_out) {

    eckit::Log::debug<LibMir>() << "++++++ intuvp2" << std::endl;
    return intuvp3(vort_grib_in, div_grib_in, length_in, length_in, u_grib_out, v_grib_out, length_out);
}


int intvect2(const void* u_grib_in,
             const void* v_grib_in,
             const int& length_in,
             void* u_grib_out,
             void* v_grib_out,
             int& length_out) {
    eckit::Log::debug<LibMir>() << "++++++ intvect2" << std::endl;
    return intvect3(u_grib_in, v_grib_in, length_in, length_in, u_grib_out, v_grib_out, length_out);
}


#ifdef OBSOLETE
/* For duplicates .. */
#define KEY_SIZE 32

typedef struct node {
    char key[KEY_SIZE];
    struct node* left;
    struct node* right;
} node;

/* Binary tree to find duplicates */

static int find_key(node* n, char* key, node** which, int* where) {
    *which = NULL;

    while (n) {
        *which = n;
        *where = memcmp(key, n->key, KEY_SIZE);

        if (*where == 0) {
            return 1;
        }

        if (*where > 0)
            n = n->right;
        else
            n = n->left;
    }
    return 0;
}

static int add_key(node** top, char* k) {
    node* which;
    node* p;
    int where;

    if (find_key(*top, k, &which, &where))
        return 1;

    p = NEW(node);

    memcpy(p->key, k, KEY_SIZE);

    p->left = p->right = NULL;
    if (which)
        if (where > 0)
            which->right = p;
        else
            which->left = p;
    else
        *top = p;
    return 0;
}

static void delete_node(node* n) {
    if (n) {
        delete_node(n->left);
        delete_node(n->right);
        FREE(n);
    }
}

#endif


static struct {

    int busy;
    int quiet;

#ifdef OBSOLETE
    int no_duplicates;
    node* top;
    int dup_count;
#endif

    int in_count;
    int out_count;
    int res_count;
    int bad_count;

    int area_cnt;
    int north;
    int south;
    int east;
    int west;
    int inter_cnt;

    int type_cnt;
    int* types;

    int block_cnt;
    int* blocks;

    int interval_cnt;
    time_interval* intervals;

    boolean is_bufrfile; /* unused */
    boolean original_grib;

    int ident_cnt;
    int* idents;
    int instrument_cnt;
    int* instruments;

    int restricted_cnt;

    long estimate;
    long edition;
    long derive_uv;

    char* odb_sql;

} ppdata = {
    0,
};

static timer* pptimer     = NULL;
static timer* memcpytimer = NULL;

static err no_scalar_postproc(char* buff, long inlen, long* outlen);
static err no_postproc(ppbuffer_t* pp, long* nbuffer);
static err grib_vector_postproc(ppbuffer_t* ppin, long* nbuffer);
static err grib_scalar_postproc(char* buffer, long inlen, long* outlen);
static err grib_postproc(ppbuffer_t* pp, long* nbuffer);
static err track_postproc(ppbuffer_t* pp, long* nbuffer);
static err track_scalar_postproc(char* buffer, long inlen, long* outlen);
static err restricted_postproc(ppbuffer_t* pp, long* nbuffer);
static err obs_postproc(ppbuffer_t* pp, long* nbuffer);
static err obs_scalar_postproc(char* buffer, long inlen, long* outlen);
static err odb_postproc(ppbuffer_t* pp, long* nbuffer);
static err odb_scalar_postproc(char* buffer, long inlen, long* outlen);

static void ppinfo() {
    int i;
    if (ppdata.area_cnt == 4)
        printf("AREA %d-%d-%d-%d\n",
               ppdata.north, ppdata.west, ppdata.south, ppdata.east);
    if (ppdata.type_cnt) {
        printf("OBSTYPE ");
        for (i = 0; i < ppdata.type_cnt; i++)
            printf("%d ", ppdata.types[i]);
        putchar('\n');
    }
    if (ppdata.interval_cnt) {
        printf("INTERVALS ");
        for (i = 0; i < ppdata.interval_cnt; i++)
            printf("[%lld,%lld] ", ppdata.intervals[i].begin,
                   ppdata.intervals[i].end);
        putchar('\n');
    }
    if (ppdata.block_cnt) {
        printf("BLOCK ");
        for (i = 0; i < ppdata.block_cnt; i++)
            printf("%d ", ppdata.blocks[i]);
        putchar('\n');
    }
    if (ppdata.ident_cnt) {
        printf("IDENT ");
        for (i = 0; i < ppdata.ident_cnt; i++)
            printf("%d ", ppdata.idents[i]);
        putchar('\n');
    }
    if (ppdata.instrument_cnt) {
        printf("INSTRUMENT ");
        for (i = 0; i < ppdata.instrument_cnt; i++)
            printf("%d ", ppdata.instruments[i]);
        putchar('\n');
    }
}


static err no_scalar_postproc(char* buff, long inlen, long* outlen) {
    *outlen = inlen;

    if (ppdata.original_grib) {
        int len = *outlen;
        err ret = original_grib(buff, &len);
        *outlen = len;
        return ret;
    }

    return NOERR;
}

static err no_postproc(ppbuffer_t* pp, long* nbuffer) {
    *nbuffer = 1;
    return no_scalar_postproc(pp[0].buffer, pp[0].inlen, &pp[0].outlen);
}

static int get_parameter(void* buffer, long length, err* ret);

static err ppintf(char* inbuf, long inlen, char* outbuf, long* outlen, boolean copy) {
    int out = *outlen;
    int in  = inlen;
    int ret = 0;

    if (mars.pseudogrib) {
        marslog(LOG_WARN, "Pseudo GRIB not interpolated");
        *outlen = 0;
    }
    else {
        if (mars.use_intuvp && (ppdata.derive_uv > 0) && is_wind(get_parameter(inbuf, inlen, &ret))) {
            ASSERT(ret == NOERR);

            marslog(LOG_DBUG, "Avoid calling intf2 when intuvp set and U/V");
            *outlen = out = 0;
            ret           = 0;
            ppdata.derive_uv--;
        }
        else {
            timer_start(pptimer);

            marslog(LOG_DBUG, "-> %s", "intf2");
            ret = intf2(inbuf, in, outbuf, out);
            marslog(LOG_DBUG, "<- %s", "intf2");

            timer_stop(pptimer, 0);
            *outlen = out;
            marslog(LOG_DBUG, "intf2 returns %d", ret);
        }
    }

    if (ret) {
        marslog(LOG_EROR, "Interpolation failed (%d)", ret);
        return ret;
    }

    if (ret == 0 && *outlen == 0) {
        /* marslog(LOG_INFO,"No interpolation done"); */
        if (copy) {
            timer_start(memcpytimer);
            memcpy(outbuf, inbuf, inlen);
            timer_stop(memcpytimer, inlen);
            *outlen = inlen;
        }
    }

    return ret;
}

/**************************/
/* Vector Post-Processing */
/**************************/

typedef struct pairs_t {
    int param;
    char* buffer;
    int len;
} pairs_t;

#define NPARAMS 257000
pairs_t pair[NPARAMS];

static void set_pair(int p, int q) {
    pair[p].param  = q;
    pair[p].buffer = NULL;
    pair[p].len    = 0;

    pair[q].param  = p;
    pair[q].buffer = NULL;
    pair[q].len    = 0;
}

static void init_pairs() {
    int i                = 0;
    static boolean first = true;
    if (!first)
        return;
    first = false;

    for (i = 0; i < NPARAMS; ++i) {
        pair[i].param  = 0;
        pair[i].buffer = NULL;
        pair[i].len    = 0;
    }
    /* Need to consider table as well: COME BACK HERE */
    set_pair(131, 132);
    set_pair(129131, 129132);
    set_pair(200131, 200132);
    set_pair(165, 166);
}


static boolean is_vector_parameter(int p) {
    init_pairs();

    /* For parameter 3.228, we allow syntax 228003 */
    if (p > 257)
        p %= 1000;

    if (pair[p].param)
        return true;

    return false;
}

static boolean vector_parameter_requested(const request* r) {
    int i             = 0;
    const char* param = NULL;

    init_pairs();

    param = r ? get_value(r, "PARAM", i++) : NULL;
    while (param) {
        int p = atoi(param);
        if (is_vector_parameter(p)) {
            marslog(LOG_DBUG, "Vector parameter %d requested", p);
            return true;
        }
        else {
            marslog(LOG_DBUG, "Param %d is not vector", p);
        }
        param = get_value(r, "PARAM", i++);
    }
    return false;
}

static int vector_pair(int p) {
    if (is_vector_parameter(p))
        return pair[p].param;

    return -1;
}


static int get_parameter(void* buffer, long length, err* ret) {
    grib_handle* g = grib_handle_new_from_message(0, buffer, length);

    long n = 0;
    if ((*ret = grib_get_long(g, "paramId", &n)) == NOERR) {
        ASSERT(n < INT_MAX);
    }

    grib_handle_delete(g);
    return n;
}


err vector_postproc(ppbuffer_t* pp, long* nbuffer) {
    int ret = 0;
    int p   = get_parameter(pp[0].buffer, pp[0].inlen, &ret);
    int q;

    if (ret != 0) {
        marslog(LOG_WARN, "Error %d in vector_postproc while getting parameter number", ret);
        marslog(LOG_WARN, "Interpolation not done");
        nbuffer = 0;
        return ret;
    }

    marslog(LOG_DBUG, "vector_postproc called for parameter %d", p);
    if (!is_vector_parameter(p)) {
        marslog(LOG_DBUG, "Parameter %d is not vector. Calling scalar post-processing", p);
        *nbuffer = 1;
        return grib_scalar_postproc(pp[0].buffer, pp[0].inlen, &pp[0].outlen);
    }


    if ((q = vector_pair(p)) == 0) {
        marslog(LOG_WARN, "Vector pair for parameter %d not found", p);
        marslog(LOG_WARN, "Perform scalar post-processing");
        marslog(LOG_WARN, "Please, inform MARS analyst");

        *nbuffer = 1;
        return grib_scalar_postproc(pp[0].buffer, pp[0].inlen, &pp[0].outlen);
    }

    /* Copy input buffer */
    pair[p].len    = pp[0].inlen;
    pair[p].buffer = static_cast<char*>(reserve_mem(pair[p].len));
    memcpy(pair[p].buffer, pp[0].buffer, pp[0].inlen);

    if (pair[q].buffer) {
        err ret       = 0;
        char* pufield = (p < q) ? pair[p].buffer : pair[q].buffer;
        char* pvfield = (p < q) ? pair[q].buffer : pair[p].buffer;

        int u_len = (p < q) ? pair[p].len : pair[q].len;
        int v_len = (p < q) ? pair[q].len : pair[p].len;

        int out = pp[0].buflen;

        marslog(LOG_DBUG, "Got parameters %d and %d. Calling vector post-processing", p, q);

        if (mars.debug) {
            request* r = empty_request("WIND");

            marslog(LOG_DBUG, "Buffer for %d at address %x, length %d", p, pair[p].buffer, pair[p].len, pair[p].len);
            grib_to_request(r, pair[p].buffer, pair[p].len);
            print_all_requests(r);
            free_all_requests(r);

            r = empty_request("WIND");
            marslog(LOG_DBUG, "Buffer for %d at address %x, length %d", q, pair[q].buffer, pair[q].len);
            grib_to_request(r, pair[q].buffer, pair[q].len);
            print_all_requests(r);
            free_all_requests(r);
        }

        /* If MARS_USE_INTUVP set and deriving U/V, conversion + interpolation
        has already been done
        */
        if (mars.use_intuvp && is_wind(p) && (ppdata.derive_uv > 0)) {
            marslog(LOG_DBUG, "MARS_USE_INTUVP set and parameters are U/V. Avoid calling intvect_");
            memcpy(pp[0].buffer, pufield, size_t(u_len));
            pp[0].inlen = u_len;
            memcpy(pp[1].buffer, pvfield, size_t(v_len));
            pp[1].inlen = v_len;
            out         = 0;
            /* ppdata.inter_cnt+=2; */
            ret = 0;
            ppdata.derive_uv -= 2;
        }
        else {
            /* Call vector interpolation */
            timer_start(pptimer);

            marslog(LOG_DBUG, "-> %s", "intvect3");
            ret = intvect3(pufield, pvfield, u_len, v_len, pp[0].buffer, pp[1].buffer, out);
            marslog(LOG_DBUG, "<- %s", "intvect3");

            timer_stop(pptimer, 0);
        }
        marslog(LOG_DBUG, "intvect3 returns %d", ret);


        /* If 0, no interpolation as been done.. */

        if ((out != 0) && (mars.grib_postproc == 0)) {
            marslog(LOG_EROR, "Env variable MARS_GRIB_POSTPROC has been set to 0");
            marslog(LOG_EROR, "and some field(s) need interpolation");
            return POSTPROC_ERROR;
        }

        if ((out != 0) && (mars.gridded_observations_postproc == 0)) {
            marslog(LOG_EROR, "Gridded observations need interpolation. In order to force");
            marslog(LOG_EROR, "interpolation, please, set MARS_GRIDDED_OBSERVATIONS_INTERP to 1");
            return POSTPROC_ERROR;
        }

        if (out == 0) {
            pp[0].outlen = pp[0].inlen;
            pp[1].outlen = pp[1].inlen;
        }
        else {
            pp[0].outlen = pp[1].outlen = out;
            ppdata.inter_cnt++;
            ppdata.inter_cnt++;
        }

        if (pp[0].outlen > pp[0].buflen) {
            marslog(LOG_EROR, "intvect3 output is %d bytes", pp[0].outlen);
            marslog(LOG_EXIT, "Buffer is only %d bytes", pp[0].buflen);
            return BUF_TO_SMALL;
        }

        if (ret < 0) {
            marslog(LOG_EROR, "Vector interpolation failed (%d)", ret);
            out     = 0;
            nbuffer = 0;
            return ret;
        }
        else {

            if (ret) {
                marslog(LOG_EROR, "Vector interpolation failed (%d)", ret);
                return ret;
            }

            /* Inform 2 buffers are ready */
            *nbuffer = 2;

            /* Release input buffers */
            release_mem(pair[p].buffer);
            pair[p].buffer = NULL;
            pair[p].len    = 0;

            release_mem(pair[q].buffer);
            pair[q].buffer = NULL;
            pair[q].len    = 0;
        }
    }
    else {
        /* We don't have q. Keep p and wait for q */
        *nbuffer = 0;
        marslog(LOG_DBUG, "Vector parameter %d kept. Waiting for parameter %d", p, q);
        ret = 0;
    }

    return ret;
}


static err grib_vector_postproc(ppbuffer_t* ppin, long* nbuffer) {
    if (mars.can_do_vector_postproc)
        return vector_postproc(ppin, nbuffer);

    *nbuffer = 1;
    return grib_scalar_postproc(ppin[0].buffer, ppin[0].inlen, &ppin[0].outlen);
}

static err grib_scalar_postproc(char* buffer, long inlen, long* outlen) {
    long ret            = 0;
    static char* result = NULL;
    static long length  = 0;
    long size           = MAX(*outlen, ppestimate());

    if (length < size) {
        if (result)
            release_mem(result);
        length = size;
        result = (char*)reserve_mem(length);
    }

    ret = ppintf(buffer, inlen, result, &size, false);

    /* If 0, no interpolation as been done.. */

    if ((size != 0) && (mars.grib_postproc == 0)) {
        marslog(LOG_EROR, "Env variable MARS_GRIB_POSTPROC has been set to 0");
        marslog(LOG_EROR, "and some field(s) need interpolation");
        return POSTPROC_ERROR;
    }

    if ((size != 0) && (mars.gridded_observations_postproc == 0)) {
        marslog(LOG_EROR, "Gridded observations need interpolation. In order to force");
        marslog(LOG_EROR, "interpolation, please, set MARS_GRIDDED_OBSERVATIONS_INTERP to 1");
        return POSTPROC_ERROR;
    }

    if (size == 0) {
        size = *outlen = inlen;
    }
    else {
        if (ret == 0) {
            if (*outlen < size) {
                *outlen = size;
                return BUF_TO_SMALL;
            }

            *outlen = size;
            ppdata.inter_cnt++;
            memcpy(buffer, result, size);
        }
    }

    if (ppdata.original_grib) {
        int len = size;
        ret     = original_grib(buffer, &len);
        *outlen = len;
    }

    return ret;
}

static err grib_postproc(ppbuffer_t* pp, long* nbuffer) {
    *nbuffer = 1;
    return grib_scalar_postproc(pp[0].buffer, pp[0].inlen, &pp[0].outlen);
}

static err odb_postproc(ppbuffer_t* pp, long* nbuffer) {
    *nbuffer = 1;
    return odb_scalar_postproc(pp[0].buffer, pp[0].inlen, &pp[0].outlen);
}

static err odb_scalar_postproc(char* buffer, long inlen, long* outlen) {
#if 0
    /* */
    marslog(LOG_DBUG,"in odb_scalar_postproc....inlen: %ld, outlen: %ld",inlen,*outlen);
    marslog(LOG_DBUG,"in odb_scalar_postproc, filter %s",ppdata.odb_sql);
    *outlen = inlen;
    if(ppdata.odb_sql != NULL)
    {
        //marslog(LOG_INFO|LOG_ONCE, "Apply ODB filter '%s'", ppdata.odb_sql);
        marslog(LOG_INFO, "Apply ODB filter '%s'", ppdata.odb_sql);

        size_t filteredLength = 0;
        odb_start();
        marslog(LOG_INFO, "in odb_scalar_postproc, calling filter_in_place: inlen=%ld", inlen);
        int rc = filter_in_place(buffer, inlen, &filteredLength, no_quotes(ppdata.odb_sql));
        if (rc) return rc;
        *outlen = filteredLength;
    }
#endif

    return NOERR;
}

static err track_postproc(ppbuffer_t* pp, long* nbuffer) {
    *nbuffer = 1;
    return track_scalar_postproc(pp[0].buffer, pp[0].inlen, &pp[0].outlen);
}

static err track_scalar_postproc(char* buffer, long inlen, long* outlen) {
    packed_key key;
    packed_key* keyptr = &key;

    char* p = buffer;
    char* q = buffer;

    unsigned long klength = 0;
    unsigned long count   = 0;

    *outlen = 0;


    while (inlen > 0) {

        int i;
        int ok = 1;

        /* Move to next BUFR */
        while ((p[0] != 'B' || p[1] != 'U' || p[2] != 'F' || p[3] != 'R') && inlen > 0) {
            p++;
            inlen--;
        }

        if (inlen <= 0)
            break;

        if (!get_packed_key(p, keyptr)) {
            ok = 0;
            set_key_length(&key, 1);
        }

        /* Types */
        if (ok && ppdata.type_cnt > 0) {
            ok = 0;
            for (i = 0; i < ppdata.type_cnt; i++)
                if (ppdata.types[i] == KEY_SUBTYPE(keyptr)) {
                    ok = 1;
                    break;
                }
            marslog(LOG_DBUG, "=> subtype %d %s match", KEY_SUBTYPE(keyptr), (ok ? "" : "DONT"));
        }

        /* Ident ... */
        if (ok && ppdata.ident_cnt > 0) {
            unsigned char* buf = (unsigned char*)KEY_IDENT(keyptr);
            int ident          = 0;
            ok                 = 0;

            for (i = 0; i < 3; i++)
                ident = ident * 10 + (buf[i] - '0');

            for (i = 0; i < ppdata.ident_cnt; i++) {
                marslog(LOG_DBUG, "=> ident %d: %d, (KEY_IDENT %s => %d)", i, ppdata.idents[i],
                        KEY_IDENT(keyptr), ident);
                if (ppdata.idents[i] == ident) {
                    ok = 1;
                    marslog(LOG_DBUG, "      MATCH");
                    break;
                }
                else
                    marslog(LOG_DBUG, " DONT MATCH");
            }
        }

        /* Time */
        if (ok && ppdata.interval_cnt > 0) {
            double obs_date;

            ok       = 0;
            obs_date = key_2_datetime(keyptr);
            for (i = 0; i < ppdata.interval_cnt; i++) {
                if (obs_date >= ppdata.intervals[i].begin && obs_date <= ppdata.intervals[i].end) {
                    ok = 1;
                    break;
                }
            }
            marslog(LOG_DBUG, "=> obs_date_time %lf %s match", obs_date, (ok ? "" : "DONT"));
        }

        /* Check for area */
        if (ok && ppdata.area_cnt == 4) {
            if (KEY_LATITUDE1(keyptr) > ppdata.north || KEY_LATITUDE1(keyptr) < ppdata.south)
                ok = 0;
            if (ppdata.west < ppdata.east) {
                if (KEY_LONGITUDE1(keyptr) < ppdata.west || KEY_LONGITUDE1(keyptr) > ppdata.east)
                    ok = 0;
            }
            else {
                if (!(KEY_LONGITUDE1(keyptr) > ppdata.west || KEY_LONGITUDE1(keyptr) < ppdata.east))
                    ok = 0;
            }
            marslog(LOG_DBUG, "=> lat: %ld, lon: %lf %s match", KEY_LATITUDE1(keyptr), KEY_LONGITUDE1(keyptr), (ok ? "" : "DONT"));
        }

        klength = key_length(p, keyptr);
        if (ok) {
            if (p != q) {
                memmove(q, p, klength);
            }
            q += klength;
            (*outlen) += klength;
            ppdata.out_count++;
        }

        p += klength;
        inlen -= klength;
        ppdata.in_count++;
        count += 1;
        marslog(LOG_DBUG, "track_scalar_postproc count %ld klength %ld", count,
                klength);
    }

    return 0;
}

static err restricted_postproc(ppbuffer_t* pp, long* nbuffer) {
    *nbuffer = 1;
    packed_key key;
    packed_key* keyptr = &key;
    char* p            = pp[0].buffer;
    char* q            = pp[0].buffer;
    long inlen         = pp[0].inlen;
    long* outlen       = &pp[0].outlen;
    long size          = inlen;

    unsigned long count   = 0;
    unsigned long klength = 0;
    *outlen               = 0;
    while (inlen > 0) {
        int i;
        int ok                 = 1;
        unsigned int keylength = 0;
        int version;
        /* Move to next BUFR */
        while ((p[0] != 'B' || p[1] != 'U' || p[2] != 'F' || p[3] != 'R') && inlen > 0) {
            p++;
            inlen--;
        }

        if (inlen <= 0)
            break;

        version = p[7];

        if (!get_packed_key(p, keyptr)) {
            ok = 0;
            set_key_length(&key, 1);
        }

        if (!mars.privileged && KEY_RESTRICTED(keyptr)) {
            ppdata.restricted_cnt++;
            // marslog(LOG_DBUG, "Report restricted was filtered");
            ok = 0;
        }
        keylength = key_length(p, keyptr);
        set_key_length(&key, ((keylength + sizeof(long) - 1) / sizeof(long)) * sizeof(long));

        if (ok && keylength > inlen) {
            ok = 0;
            set_key_length(&key, sizeof(long));
            marslog(LOG_EROR, "Report to large found at position %d, skipped", ppdata.in_count + 1);
        }

        klength = key_length(p, keyptr);
        if (ok) {
            if (p != q) {
                memmove(q, p, klength);
            }
            q += klength;
            (*outlen) += klength;
            ppdata.out_count++;
        }

        p += klength;
        inlen -= klength;
        ppdata.in_count++;
        count += 1;
        marslog(LOG_DBUG, "restricted_postproc count %ld klength %ld", count, klength);
    }

    // if(count != 1) {
    //     marslog(LOG_EXIT,"Wrong number of obs messages found %ld", count);
    // }

    // printf("size = %ld, outlen=%ld %ld %ld\n", size, *outlen, p-pp[0].buffer, q-pp[0].buffer);

    p = pp[0].buffer;
    if (size < *outlen) {
        memset(p + size, 0, (*outlen) - size);
    }

    return 0;
}

static err obs_postproc(ppbuffer_t* pp, long* nbuffer) {
    *nbuffer = 1;
    return obs_scalar_postproc(pp[0].buffer, pp[0].inlen, &pp[0].outlen);
}

static err obs_scalar_postproc(char* buffer, long inlen, long* outlen) {
    int issat;
    packed_key key;
    packed_key* keyptr = &key;

    char* p = buffer;
    char* q = buffer;

    unsigned long klength = 0;
    unsigned long count   = 0;
    long size             = inlen;

    *outlen = 0;


    while (inlen > 0) {

        int i;
        int ok                 = 1;
        unsigned int keylength = 0;
        int version;

        /* Move to next BUFR */
        while ((p[0] != 'B' || p[1] != 'U' || p[2] != 'F' || p[3] != 'R') && inlen > 0) {
            p++;
            inlen--;
        }

        if (inlen <= 0)
            break;

        version = p[7];

        if (!get_packed_key(p, keyptr)) {
            ok = 0;
            set_key_length(&key, 1);
        }

        if (!mars.privileged && KEY_RESTRICTED(keyptr)) {
            ppdata.restricted_cnt++;
            ok = 0;
        }

        issat = (KEY_TYPE(keyptr) == 2) || (KEY_TYPE(keyptr) == 3) || (KEY_TYPE(keyptr) == 12);

        /* WMO block ... */
        if (ok && ppdata.block_cnt > 0) {
            int wmo = (KEY_IDENT(keyptr)[0] - '0') * 10
                      + (KEY_IDENT(keyptr)[1] - '0');
            ok = 0;
            if (!issat) {
                for (i = 0; i < ppdata.block_cnt; i++)
                    if (ppdata.blocks[i] == wmo) {
                        ok = 1;
                        break;
                    }
            }
        }

        /* Types */
        if (ok && ppdata.type_cnt > 0) {
            ok = 0;
            for (i = 0; i < ppdata.type_cnt; i++)
                if (ppdata.types[i] == KEY_SUBTYPE(keyptr)) {
                    ok = 1;
                    break;
                }
        }

        /* Ident ... */
        if (ok && ppdata.ident_cnt > 0) {
            ok = 0;
            if (issat) {
                int ident = bufr_sat_id(p, keyptr);

                for (i = 0; i < ppdata.ident_cnt; i++)
                    if (ppdata.idents[i] == ident) {
                        ok = 1;
                        break;
                    }
            }
            else {
                int ident = 0;
                for (i = 0; i < 5; i++)
                    ident = 10 * ident + (KEY_IDENT(keyptr)[i] - '0');

                for (i = 0; i < ppdata.ident_cnt; i++)
                    if (ppdata.idents[i] == ident) {
                        ok = 1;
                        break;
                    }
            }
        }

        /* time . Note that time is irrelevant for SSMI (126) data */
        if (ok && ppdata.interval_cnt > 0 && KEY_SUBTYPE(keyptr) != 126) {
            double obs_date;

            ok       = 0;
            obs_date = key_2_datetime(keyptr);
            for (i = 0; i < ppdata.interval_cnt; i++) {
                if (obs_date >= ppdata.intervals[i].begin && obs_date <= ppdata.intervals[i].end) {
                    ok = 1;
                    break;
                }
            }
        }

        /* Check for area */
        if (ok && ppdata.area_cnt == 4) {
            if (issat) {
                if (KEY_LATITUDE1(keyptr) > ppdata.north || KEY_LATITUDE2(keyptr) < ppdata.south)
                    ok = 0;
                if ((KEY_LONGITUDE1(keyptr) < ppdata.west || KEY_LONGITUDE2(keyptr) > ppdata.east) && !(ppdata.west == ppdata.east))
                    ok = 0;
            }
            else {
                if (KEY_LATITUDE1(keyptr) > ppdata.north || KEY_LATITUDE1(keyptr) < ppdata.south)
                    ok = 0;
                if (ppdata.west < ppdata.east) {
                    if (KEY_LONGITUDE1(keyptr) < ppdata.west || KEY_LONGITUDE1(keyptr) > ppdata.east)
                        ok = 0;
                }
                else {
                    if (!(KEY_LONGITUDE1(keyptr) > ppdata.west || KEY_LONGITUDE1(keyptr) < ppdata.east))
                        ok = 0;
                }
#if 0
                    if(ok)
                    {
                    marslog(LOG_INFO,"longitude: %d, west: %d, east: %d ",
                            KEY_LONGITUDE1(keyptr),
                            ppdata.west,
                            ppdata.east);
                    }
#endif
            }
        }

        if (ok && issat && (ppdata.instrument_cnt > 0)) {
#if defined(BUFR_FILTER)
            fortint n_instr           = ppdata.instrument_cnt;
            fortint found             = 0;
            fortint e                 = 0;
            timer* filter_instruments = get_timer("Filter instruments", NULL, true);
            fortint orig_len          = inlen;
            char* orig_buf            = p;

            timer_start(filter_instruments);
            mbufr_mars_filter_(&n_instr, ppdata.instruments, &orig_len, orig_buf, &found, &e);
            timer_stop(filter_instruments, 0);
            if (e != 0)
                marslog(LOG_WARN, "Error %d while filtering instruments on message %d", e, ppdata.in_count);
            else
                ok = found;
#else
            marslog(LOG_WARN, "Filter by instrument not active");
#endif
        }

#ifdef OBSOLETE
        if (ok && (ppdata.no_duplicates || mars.remove_bufr_duplicates)) {
            if (add_key(&ppdata.top, (char*)&key)) {
                ppdata.dup_count++;
                ok = 0;
            }
        }
#endif

#ifdef ECMWF
        if (ok && mars.restriction) {
            if (restricted(keyptr)) {
                ok = 0;
                ppdata.res_count++;
            }
        }
#endif

        klength = key_length(p, keyptr);

        if (klength <= 4 || klength > inlen || p[klength - 4] != '7' || p[klength - 3] != '7' || p[klength - 2] != '7' || p[klength - 1] != '7') {
            /* Some obs are wrong... june in 1990 */
            unsigned char* s = (unsigned char*)(p + 4); /* skip BUFR */
            int len          = 8;
            int nsec         = 4;

            /* skip section 0 , check for BUFR version */
            char* sec1_start = p;
            if ((unsigned char)sec1_start[7] > 1)
                sec1_start += 8;
            else
                sec1_start += 4;
            if (SEC1_FLAGS(sec1_start, version) == 0) {
                marslog(LOG_WARN, "Report %d has no key but was included",
                        ppdata.in_count + 1);
                ok   = 1;
                nsec = 3;
            }

            for (i = 0; i < nsec && len <= inlen; i++) {
                int sec = (s[0] << 16) + (s[1] << 8) + s[2];
                len += sec;
                s += sec;
            }
            ppdata.bad_count++;

            set_key_length(&key, len);

            klength = key_length(p, keyptr);
            if (SEC1_FLAGS(p, version) != 0) {
                if (len <= 4 || len > inlen || p[klength - 4] != '7' || p[klength - 3] != '7' || p[klength - 2] != '7' || p[klength - 1] != '7') {
                    ok = 0;
                    set_key_length(&key, sizeof(long));
                    marslog(LOG_EROR, "Bad report found at position %d, skipped", ppdata.in_count + 1);
                }
            }
        }

        keylength = key_length(p, keyptr);

        set_key_length(&key, ((keylength + sizeof(long) - 1) / sizeof(long)) * sizeof(long));

        if (ok && keylength > inlen) {
            ok = 0;
            set_key_length(&key, sizeof(long));
            marslog(LOG_EROR, "Report to large found at position %d, skipped", ppdata.in_count + 1);
        }

        klength = key_length(p, keyptr);
        if (ok) {
            if (p != q) {
                memmove(q, p, klength);
            }
            q += klength;
            (*outlen) += klength;
            ppdata.out_count++;
        }

        p += klength;
        inlen -= klength;
        ppdata.in_count++;
        count += 1;
        marslog(LOG_DBUG, "obs_scalar_postproc count %ld klength %ld", count,
                klength);
    }

    // if(count != 1) {
    //   marslog(LOG_EXIT, "Wrong number of obs messages found %ld", count);
    // }

    // printf("size = %ld, outlen=%ld %ld %ld\n", size, *outlen, p-pp[0].buffer, q-pp[0].buffer);

    p = buffer;
    if (size < *outlen) {
        memset(p + size, 0, (*outlen) - size);
    }

    return 0;
}

}  // namespace mir

//----------------------------------------------------------------------------------------------------------------------


static void pp_callback_log_debug(void*, const char* msg) {
    marslog(LOG_DBUG, "%s", msg);
}
static void pp_callback_log_info(void*, const char* msg) {
    marslog(LOG_INFO, "%s", msg);
}
static void pp_callback_log_warning(void*, const char* msg) {
    marslog(LOG_WARN | LOG_ONCE, "%s", msg);
}
static void pp_callback_log_error(void*, const char* msg) {
    marslog(LOG_EROR, "%s", msg);
}


//----------------------------------------------------------------------------------------------------------------------

namespace marsclient {

class PProcMIR : public PProc {

public:  // methods
    PProcMIR(const char* name) :
        name_(name) {}

    virtual ~PProcMIR() {}

    virtual void print_version() {
        marslog(LOG_INFO, "package mir version: %s", mir_version_str());
        marslog(LOG_DBUG, "Linear algebra sparse backend is %s", eckit::linalg::LinearAlgebraSparse::name().c_str());
        marslog(LOG_DBUG, "Linear algebra dense backend is %s", eckit::linalg::LinearAlgebraDense::name().c_str());
    }

    virtual const std::string& name() const { return name_; }

    virtual err initialise(int argc, char** argv);

    virtual err ppinit(const request* r, postproc* proc);
    virtual err ppdone(void);
    virtual err ppcount(int* in, int* out);
    virtual err pparea(request* r);
    virtual fieldset* pp_fieldset(const char* file, request* filter);
    virtual err ppstyle(const request* r);
    virtual err pprotation(const request* r);
    virtual long ppestimate();
    virtual err makeuv(char* vo, char* d, long inlen, char* u, char* v, long* outlen);

protected:  // members
    err makeuv3(char* vo, char* d, long vo_len, long d_len, char* u, char* v, long* outlen);

private:  // members
    eckit::Mutex mutex_;

    std::string name_;
};

int PProcMIR::initialise(int argc, char** argv) {

    using namespace mir;

    try {
        eckit::Main::initialise(argc, argv);

        // Logging streams
        eckit::Log::debug<LibMir>().setCallback(&pp_callback_log_debug);
        eckit::Log::debug<eckit::LibEcKit>().setCallback(&pp_callback_log_debug);
        eckit::Log::info().setCallback(&pp_callback_log_info);
        eckit::Log::warning().setCallback(&pp_callback_log_warning);
        eckit::Log::error().setCallback(&pp_callback_log_error);
    }
    catch (std::exception& e) {
        eckit::Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl;
        return -1;
    }

    marslog(LOG_DBUG, "Post processing backend is %s", name().c_str());

    return 0;
}

err PProcMIR::ppstyle(const request* r) {
    return NOERR;
}


err PProcMIR::pprotation(const request*) {
    return NOERR;
}


#define PPROC_PRECISION "%.15g"
err PProcMIR::pparea(request* r) {
    using namespace mir;

    eckit::AutoLock<eckit::Mutex> lock(mutex_);

    const char* p;
    double ew = 0, ns = 0, n = 0, s = 0, e = 0, w = 0;
    double ew_ = 0, ns_ = 0, n_ = 0, s_ = 0, e_ = 0, w_ = 0;

    if ((p = get_value(r, "_AREA_N", 0)))
        n_ = n = atof(p);
    if ((p = get_value(r, "_AREA_S", 0)))
        s_ = s = atof(p);
    if ((p = get_value(r, "_AREA_E", 0)))
        e_ = e = atof(p);
    if ((p = get_value(r, "_AREA_W", 0)))
        w_ = w = atof(p);
    if ((p = get_value(r, "_GRID_EW", 0)))
        ew_ = ew = atof(p);
    if ((p = get_value(r, "_GRID_NS", 0)))
        ns_ = ns = atof(p);
    if ((p = get_value(r, "_GAUSSIAN", 0)))
        ew_ = ew = atof(p);

    if (ew == 0 && ns == 0)
        return 0;

    if (((ew_ != 0) && (ew_ != ew)) || (ns_ != ns)) {
        if (ns == 0) {
            marslog(LOG_WARN, "Grid not supported, changed from " PPROC_PRECISION " to " PPROC_PRECISION,
                    ew_, ew);
            set_value(r, "_GAUSSIAN", PPROC_PRECISION, ew);
            set_value(r, "GRID", "%g", ew);
        }
        else {
            marslog(LOG_WARN, "Grid not supported, changed from " PPROC_PRECISION "/" PPROC_PRECISION " to " PPROC_PRECISION "/" PPROC_PRECISION,
                    ew_, ns_, ew, ns);
            set_value(r, "_GRID_EW", PPROC_PRECISION, ew);
            set_value(r, "_GRID_NS", PPROC_PRECISION, ns);
            set_value(r, "GRID", PPROC_PRECISION, ew);
            add_value(r, "GRID", PPROC_PRECISION, ns);
        }
    }

    if (n_ != 0 || s_ != 0 || e_ != 0 || w_ != 0)
        if (n_ != n || s_ != s || e_ != e || w_ != w) {
            if ((e_ != e && e_ != (e - 360)) || (w_ != w && w_ != (w - 360)) || (n_ != n) || (s_ != s)) {
                marslog(LOG_WARN, "Area not compatible with grid");
                marslog(LOG_WARN, "Area changed from " PPROC_PRECISION "/" PPROC_PRECISION "/" PPROC_PRECISION "/" PPROC_PRECISION " to " PPROC_PRECISION "/" PPROC_PRECISION "/" PPROC_PRECISION "/" PPROC_PRECISION,
                        n_, w_, s_, e_,
                        n, w, s, e);
            }

            set_value(r, "_GRID_N", PPROC_PRECISION, n);
            set_value(r, "_GRID_W", PPROC_PRECISION, w);
            set_value(r, "_GRID_S", PPROC_PRECISION, s);
            set_value(r, "_GRID_E", PPROC_PRECISION, e);

            set_value(r, "AREA", PPROC_PRECISION, n);
            add_value(r, "AREA", PPROC_PRECISION, w);
            add_value(r, "AREA", PPROC_PRECISION, s);
            add_value(r, "AREA", PPROC_PRECISION, e);

            set_value(r, "_AREA_N", PPROC_PRECISION, n);
            set_value(r, "_AREA_S", PPROC_PRECISION, s);
            set_value(r, "_AREA_E", PPROC_PRECISION, e);
            set_value(r, "_AREA_W", PPROC_PRECISION, w);
        }

    return 0;
}
#undef PPROC_PRECISION

err PProcMIR::makeuv(char* vo, char* d, long inlen, char* u, char* v, long* outlen) {
    return makeuv3(vo, d, inlen, inlen, u, v, outlen);
}

err PProcMIR::makeuv3(char* vo, char* d, long vo_len, long d_len, char* u, char* v, long* outlen) {
    using namespace mir;

    eckit::AutoLock<eckit::Mutex> lock(mutex_);

    err e    = NOERR;
    auto out = int(*outlen);

    if (!job) {
        job.reset(new api::MIRJob());
    }

    if (mars.grib_postproc == 0) {
        marslog(LOG_EROR, "Env variable MARS_GRIB_POSTPROC has been set to 0");
        marslog(LOG_EROR, "and conversion to U/V requested");
        return POSTPROC_ERROR;
    }

    if (!ppdata.quiet) {
        marslog(LOG_INFO, "Deriving U and V from vorticity and divergence");
        ppdata.quiet = 1;
    }

    marslog(LOG_DBUG, "-> intuv%s3 in=%d out=%d", mars.use_intuvp ? "p" : "s", *outlen, out);
    ASSERT(mars.use_intuvp);

    timer_start(pptimer);
    if (mars.use_intuvp) {
        auto vo_length = int(vo_len);
        auto d_length  = int(d_len);

        e = intuvp3(vo, d, vo_length, d_length, u, v, out);

        vo_len = long(vo_length);
        d_len  = long(d_length);

        /*
         * if(in != out) { ppdata.inter_cnt+=2; }
         * FIXME: the above is generally true, specially with packing other than grid_simple.
         * Also the check wouldn't notice when converting spectral vo/d to spectral U/V
         * without truncating (sizes wouldn't change)
         */
        ppdata.inter_cnt += 2;

        marslog(LOG_DBUG, "MARS_USE_INTUVP set and parameters are U/V. Avoid calling intf2");
        ppdata.derive_uv += 2;
    }
    timer_stop(pptimer, 0);

    marslog(LOG_DBUG, "<- intuvp3 in=%d out=%d", *outlen, out);

    if (out > *outlen) {
        marslog(LOG_EROR, "intuvp3 output is %d bytes", out);
        marslog(LOG_EXIT, "Buffer is only %d bytes", *outlen);
    }

    *outlen = out;

    /* Check if rounding to a multiple of 4 changes */
    /* the actual size.                             */

    if (*outlen > (vo_len + long(sizeof(int)))) {
        marslog(LOG_DBUG, "intuvp3 returns bigger field %d > %d (vo)", *outlen, vo_len);
        /* return -3; */
    }

    if (*outlen > (d_len + long(sizeof(int)))) {
        marslog(LOG_DBUG, "intuvp3 returns bigger field %d > %d (d)", *outlen, d_len);
        /* return -3; */
    }

    return e;
}


long PProcMIR::ppestimate() {
    using namespace mir;

    return ppdata.estimate;
}

/* Initialise post-processing .... */
err PProcMIR::ppinit(const request* r, postproc* proc) {
    using namespace mir;

    eckit::AutoLock<eckit::Mutex> lock(mutex_);

    const char* p;
    const char* mars_postproc = get_value(r, "_POSTPROCESSING", 0);

    *proc = no_postproc;

    if (ppdata.busy) {
        marslog(LOG_EROR, "Post-processing package already open");
        return -2;
    }

    /* Reset */
    job.reset(new api::MIRJob);

    ppdata.derive_uv = 0;
    ppdata.busy      = 1;
    ppdata.inter_cnt = ppdata.in_count = ppdata.out_count = ppdata.bad_count = ppdata.res_count = 0;
    ppdata.restricted_cnt                                                                       = 0;

    pptimer     = get_timer("Post-processing", "postprocessing", false);
    memcpytimer = get_timer("Memory copy", NULL, false);

#ifdef OBSOLETE
    ppdata.dup_count = 0;
#endif

    if ((p = get_value(r, "GRIB", 0))) {
        ppdata.original_grib = EQ(p, "ORIGINAL");
    }

    if (mars_postproc && !(atol(mars_postproc))) {
        *proc = no_postproc;
    }
    else if (fetch(r)) {
        *proc = no_postproc;
    }
    else if (bias(r)) {
        *proc = no_postproc;
    }
    else if (feedback(r)) {
        *proc = mars.privileged ? no_postproc : restricted_postproc;
    }
    else if (/* message_is_netcdf() */ false) {
        *proc = no_postproc;
    }
    else if (is_ocean_netcdf(r) /*&& message_is_netcdf()*/) {
        *proc = no_postproc;  // don't post-process ocean (netcdf) data
    }
    else if (is_odb(r)) {
#ifdef ODB_SUPPORT
        /* Filter */
        const char* filter = get_value(r, "FILTER", 0);
        ppdata.odb_sql     = NULL;
        if (filter)
            ppdata.odb_sql = strcache(filter);

        marslog(LOG_DBUG, "filter is: %s", ppdata.odb_sql);
        *proc = odb_postproc;
#else
        marslog(LOG_EROR, "This MARS client doesn't support ODB");
        marslog(LOG_EROR, "Please, contact the MARS team");
        *proc = no_postproc;
#endif
    }
    else if (track(r)) {
        long time_count;
        long date_count;

        /* Ident */

        if (ppdata.idents)
            FREE(ppdata.idents);

        if ((ppdata.ident_cnt = count_values(r, "IDENT")))
            ppdata.idents = NEW_ARRAY(int, ppdata.ident_cnt);

        for (int i = 0; i < ppdata.ident_cnt; i++) {
            const char* p = no_quotes(get_value(r, "IDENT", i));
            int ident;
            int n = 0;

            if (p)
                n = strlen(p);

            /* Tropical cyclones have a alph num ident */
            char buf[8];
            int j;

            ident = 0;
            sprintf(buf, "%s", p);
            for (j = 0; j < n; j++) {
                if (islower(buf[j]))
                    buf[j] = toupper(buf[j]);
                ident = ident * 10 + (buf[j] - '0');
            }

            ppdata.idents[i] = ident;
            marslog(LOG_DBUG,
                    "Requested ident %d: %s [%s]. Internal representation: %d", i,
                    p, buf, ident);
        }

        /* Types */

        if (ppdata.types)
            FREE(ppdata.types);

        if ((ppdata.type_cnt = count_values(r, "OBSTYPE"))) {
            int zero     = 0;
            ppdata.types = NEW_ARRAY(int, ppdata.type_cnt);
            for (int i = 0; i < ppdata.type_cnt; i++) {
                ppdata.types[i] = atoi(get_value(r, "OBSTYPE", i));
                if (ppdata.types[i] == 0)
                    zero++;
            }

            if (zero > 0 && ppdata.type_cnt != zero) {
                marslog(LOG_WARN,
                        "Cannot mix types and subtypes when retrieving tracks");
                marslog(LOG_WARN, "Only corresponding types will be returned");
            }

            if (zero)
                ppdata.type_cnt = 0;
        }

        /* Time  */

        if (ppdata.intervals)
            FREE(ppdata.intervals);

        time_count          = count_values(r, "TIME");
        date_count          = count_values(r, "DATE");
        ppdata.interval_cnt = time_count * date_count;

        ppdata.intervals = NEW_ARRAY(time_interval, ppdata.interval_cnt);
        for (long i = 0; i < date_count; i++) {
            int j;
            long date = get_julian_from_request(r, i);
            for (j = 0; j < time_count; j++) {
                long time                                  = atol(get_value(r, "TIME", j));
                double dt                                  = date_time_2_datetime(date, time / 100, time % 100, 0);
                ppdata.intervals[i * time_count + j].begin = dt;
                ppdata.intervals[i * time_count + j].end   = dt;
            }
        }

        /* Area ... */
        ppdata.area_cnt = count_values(r, "AREA");
        if (ppdata.area_cnt == 4) {
            ppdata.north = atof(get_value(r, "_AREA_N", 0)) * 100000.0 + 9000000.0;
            ppdata.south = atof(get_value(r, "_AREA_S", 0)) * 100000.0 + 9000000.0;
            ppdata.east  = atof(get_value(r, "_AREA_E", 0)) * 100000.0 + 18000000.0;
            ppdata.west  = atof(get_value(r, "_AREA_W", 0)) * 100000.0 + 18000000.0;
            ppdata.east %= 36000000;
            ppdata.west %= 36000000;
        }

        *proc = track_postproc;
    }
    else if (observation(r)) {

        long time_count;
        long date_count;

        /* Ident */

        if (ppdata.idents)
            FREE(ppdata.idents);

        if ((ppdata.ident_cnt = count_values(r, "IDENT")))
            ppdata.idents = NEW_ARRAY(int, ppdata.ident_cnt);

        for (int i = 0; i < ppdata.ident_cnt; i++) {
            const char* p = get_value(r, "IDENT", i);
            int ident;

            if (is_number(p))
                ident = atoi(p);
            else {
                /* Ships have a alph num ident */
                char buf[8];
                int j;

                ident = 0;
                sprintf(buf, "%-5s", p);
                for (j = 0; j < 5; j++) {
                    if (islower(buf[j]))
                        buf[j] = toupper(buf[j]);
                    ident = ident * 10 + (buf[j] - '0');
                }
            }

            ppdata.idents[i] = ident;
        }

        /* Instruments */
        if (ppdata.instruments)
            FREE(ppdata.instruments);

        if ((ppdata.instrument_cnt = count_values(r, "INSTRUMENT")))
            ppdata.instruments = NEW_ARRAY(int, ppdata.instrument_cnt);

        for (int i = 0; i < ppdata.instrument_cnt; i++)
            ppdata.instruments[i] = atoi(get_value(r, "INSTRUMENT", i));

        /* Types */

        if (ppdata.types)
            FREE(ppdata.types);

        if ((ppdata.type_cnt = count_values(r, "OBSTYPE"))) {
            int zero     = 0;
            ppdata.types = NEW_ARRAY(int, ppdata.type_cnt);
            for (int i = 0; i < ppdata.type_cnt; i++) {
                ppdata.types[i] = atoi(get_value(r, "OBSTYPE", i));
                if (ppdata.types[i] == 0)
                    zero++;
            }

            if (zero > 0 && ppdata.type_cnt != zero) {
                marslog(LOG_WARN,
                        "Cannot mix types and subtypes when retrieving observations");
                marslog(LOG_WARN, "Only corresponding types will be returned");
            }

            if (zero)
                ppdata.type_cnt = 0;
        }

        /* WMO block */

        if (ppdata.blocks)
            FREE(ppdata.blocks);

        if ((ppdata.block_cnt = count_values(r, "BLOCK")))
            ppdata.blocks = NEW_ARRAY(int, ppdata.block_cnt);

        for (int i = 0; i < ppdata.block_cnt; i++)
            ppdata.blocks[i] = atoi(get_value(r, "BLOCK", i));

        /* Time  */

        if (ppdata.intervals)
            FREE(ppdata.intervals);

        time_count          = count_values(r, "TIME");
        date_count          = count_values(r, "DATE");
        ppdata.interval_cnt = time_count * date_count;

        /* if the time/to/time is not expanded in this version */
        if (time_count == 3 && (strcmp(get_value(r, "TIME", 1), "TO") == 0)) {
            long time1       = atoi(get_value(r, "TIME", 0));
            long time2       = atoi(get_value(r, "TIME", 2));
            ppdata.intervals = NEW_ARRAY(time_interval, date_count);
            for (long i = 0; i < date_count; i++) {
                long date                 = get_julian_from_request(r, i);
                ppdata.intervals[i].begin = date_time_2_datetime(date, time1 / 100, time1 % 100, 0);
                ppdata.intervals[i].end   = date_time_2_datetime(date, time2 / 100, time2 % 100, 59);
            }
            ppdata.interval_cnt = date_count;
        }
        else {
            long range = 0;
            if (count_values(r, "RANGE") != 0)
                range = atol(get_value(r, "RANGE", 0));
            ppdata.intervals = NEW_ARRAY(time_interval, ppdata.interval_cnt);
            for (long i = 0; i < date_count; i++) {
                int j;
                long date = get_julian_from_request(r, i);
                for (j = 0; j < time_count; j++) {
                    long time = atol(get_value(r, "TIME", j));
                    double dt = date_time_2_datetime(date, time / 100, time % 100, 0);
                    init_time_interval(&ppdata.intervals[i * time_count + j], dt,
                                       range * 60 + 59);
                }
            }
        }

        /* Area ... */
        ppdata.area_cnt = count_values(r, "AREA");
        if (ppdata.area_cnt == 4) {
            ppdata.north = atof(get_value(r, "_AREA_N", 0)) * 100000.0 + 9000000.0;
            ppdata.south = atof(get_value(r, "_AREA_S", 0)) * 100000.0 + 9000000.0;
            ppdata.east  = atof(get_value(r, "_AREA_E", 0)) * 100000.0 + 18000000.0;
            ppdata.west  = atof(get_value(r, "_AREA_W", 0)) * 100000.0 + 18000000.0;
            ppdata.east %= 36000000;
            ppdata.west %= 36000000;
        }

#ifdef OBSOLETE
        p                    = get_value(r, "DUPLICATES", 0);
        ppdata.no_duplicates = p && (*p == 'R');
#endif

        /* ppinfo(); */

        *proc = obs_postproc;
    }
    else if (image(r) || simulated_image(r))
        *proc = no_postproc;
    else if (gridded_observations(r) && !mars.gridded_observations_postproc)
        *proc = no_postproc;
    else {
        /* Check for vector parameters and rotation */
        boolean vector   = vector_parameter_requested(r);
        boolean rotation = get_value(r, "_ROTATION_LAT", 0) || get_value(r, "_ROTATION_LON", 0);

        *proc           = grib_postproc;
        ppdata.estimate = 0;

        if (!gridded_observations(r)) {
            mars.gridded_observations_postproc = 1;
        }

        /* By default, copy edition from intput to output: edition=0*/
        ppdata.edition = 0;
        if ((p = get_value(r, "FORMAT", 0))) {
            if (EQ(p, "GRIB1"))
                ppdata.edition = 1;
            if (EQ(p, "GRIB2"))
                ppdata.edition = 2;
            marslog(LOG_WARN, "Format conversion not enabled");
        }

        /* gridname */
        p = get_value(r, "_GRIDNAME", 0);
        if (p) {
            job->clear("grid");
            job->set("gridname", p);

            long n     = atoi(get_value(r, "_GAUSSIAN", 0));
            long guess = (2 * n) * (4 * n + 20); /* nb lats * nb points at equator for octahedral grid */
            if (guess > ppdata.estimate)
                ppdata.estimate = guess;
        }

        /* grid */
        if (get_value(r, "_GRID_EW", 0)) {
            job->clear("gridname");
            const double ew = atof(get_value(r, "_GRID_EW", 0));
            const double ns = atof(get_value(r, "_GRID_NS", 0));
            if (ew != 0 || ns != 0) {
                job->set("grid", ew, ns);
            }

            ppdata.estimate = (360.0 / ew + 1) * (180.0 / ns + 1);
        }

        /* area */
        if (get_value(r, "_AREA_N", 0)) {
            const double array[4] = {
                atof(get_value(r, "_AREA_N", 0)),
                atof(get_value(r, "_AREA_W", 0)),
                atof(get_value(r, "_AREA_S", 0)),
                atof(get_value(r, "_AREA_E", 0))};

            /* If AREA not specified (0/0/0/0), don't call intout to avoid
               interpolation sw. to be called when no post-processing is
               required */
            if (!(array[0] == 0 && array[1] == 0 && array[2] == 0 && array[3] == 0)) {
                job->set("area", array[0], array[1], array[2], array[3]);
                if (ppdata.estimate > 0) {

                    // Lower memory estimate with covered area fraction
                    // - overestimate slightly to avoid points included/excluded adjustments
                    // - limit to a minimum estimate (small areas/single points can cause problems)
                    const double overestimateFactor = 1.1;
                    const long minimumEstimate      = 1024;  // (magic!)

                    using eckit::geometry::Point2;
                    using eckit::geometry::Sphere;

                    Point2 WN(array[1], array[0]);
                    Point2 ES(array[3], array[2]);
                    double fraction = std::min(1., overestimateFactor * Sphere::area(1., WN, ES));
                    long guess      = std::max(minimumEstimate, long(fraction * double(ppdata.estimate)));

                    ppdata.estimate = guess;
                }
            }
            else {
                job->clear("area");
            }
        }

        /* frame */
        p = get_value(r, "FRAME", 0);
        if (p) {
            const long n = long(atoi(p));
            job->set("frame", n);
        }

        /* bitmap */
        p = get_value(r, "BITMAP", 0);
        if (p) {
            job->set("bitmap", tidy(p, false, true));
        }

        /* resol */
        p = get_value(r, "RESOL", 0);
        if (p) {
            const std::string resol(p);
            if (resol == "AV") {

                job->set("intgrid", "source");
                job->set("truncation", "none");
            }
            else if (p[0] == 'N' || p[0] == 'O' || p[0] == 'F') {

                job->set("intgrid", resol);
            }
            else if (resol != "AUTO") {

                job->set("truncation", atol(p));
            }
        }

        /* truncation */
        p = get_value(r, "TRUNCATION", 0);
        if (p) {
            job->set("truncation", p);
        }

        /* intgrid */
        p = get_value(r, "INTGRID", 0);
        if (p) {
            const std::string intgrid = tidy(p, true, true);
            if (rotation && intgrid == "none") {
                marslog(LOG_WARN, "With rotation, intgrid=none is ignored");
            }
            else {
                job->set("intgrid", intgrid);
            }
        }

        /* style */
        p = get_value(r, "STYLE", 0);
        if (p) {
            const std::string style = tidy(p, false, true);
            if (style == "DISSEMINATION") {
                marslog(LOG_WARN, "With style=dissemination, avoid intermediate packing");
                mars.use_intuvp = 1;
            }

            job->set("style", tidy(p, true, true));
        }

        /* rotation */
        if (rotation) {
            const double rotation[2] = {
                atof(get_value(r, "_ROTATION_LAT", 0)),
                atof(get_value(r, "_ROTATION_LON", 0))};
            job->set("rotation", rotation[0], rotation[1]);

            if (vector)
                *proc = grib_vector_postproc;
        }

        /* accuracy */
        if (mars.accuracy > 0) {
            job->set("accuracy", long(mars.accuracy));
        }

        /* packing */
        p = get_value(r, "PACKING", 0);
        if (p) {
            const std::string packing = tidy(p, true, true);
            job->set("packing", packing);
        }

        /* specification */
        if (get_value(r, "SPECIFICATION", 0)) {
            NOTIMP;
        }

        /* LSM */
        p = get_value(r, "LSM", 0);
        if (p) {
            job->set("lsm", EQ(p, "ON") ? true : false);
        }

        /* interpolation */
        p = get_value(r, "INTERPOLATION", 0);
        if (p) {
            const std::string i = tidy(p, false, true);
            if (i.length() && i[0] == '-') {
                job->set(i);
            }
            else {
                job->set("interpolation", tidy(p));
            }
        }

        /* ppdata.estimate contains the number of point, we need
        the header and sizeof(fortint) bytes per value */
        if (ppdata.estimate) {
            ppdata.estimate = ppdata.estimate * sizeof(int) + 4096;
        }
    }
    return 0;
}


err PProcMIR::ppdone(void) {
    using mir::ppdata;

    eckit::AutoLock<eckit::Mutex> lock(mutex_);

    if (!ppdata.busy)
        marslog(LOG_EROR, "Post-processing package already closed");
    ppdata.busy  = 0;
    ppdata.quiet = 0;

#ifdef OBSOLETE
    delete_node(ppdata.top);
    ppdata.top = NULL;
    if (ppdata.dup_count)
        marslog(LOG_INFO, "%d duplicates reports", ppdata.dup_count);
#endif

    if (ppdata.res_count)
        marslog(LOG_DBUG, "%d restricted reports found", ppdata.res_count);

    if (ppdata.inter_cnt) {
        char host[80];
        char s[1024] = "";
        gethostname(host, sizeof(host));
        if (mars.show_hosts)
            sprintf(s, "on '%s'", host);
        marslog(LOG_INFO, "%d field%s ha%s been interpolated %s",
                ppdata.inter_cnt,
                ppdata.inter_cnt == 1 ? "" : "s",
                ppdata.inter_cnt == 1 ? "s" : "ve",
                s);
    }
    log_statistics("interpolated", "%d", ppdata.inter_cnt);

    return 0;
}


err PProcMIR::ppcount(int* in, int* out) {
    using namespace mir;

    eckit::AutoLock<eckit::Mutex> lock(mutex_);

    if (ppdata.bad_count) {
        marslog(LOG_WARN, "%d report(s) have a wrong length in their key.", ppdata.bad_count);
        marslog(LOG_WARN, "Please inform Mars group");
    }

    if (ppdata.restricted_cnt) {
        marslog(LOG_WARN, "%d restricted report(s) filtered out.", ppdata.restricted_cnt);
    }

    *in  = ppdata.in_count;
    *out = ppdata.out_count;
    return 0;
}


fieldset* PProcMIR::pp_fieldset(const char* file, request* filter) {
    using namespace mir;

    eckit::AutoLock<eckit::Mutex> lock(mutex_);

    fieldset* v = read_fieldset(file, filter);
    fieldset* w = NULL;
    err e       = 0;
    int i;
    postproc proc;

    if (!v)
        return NULL;
    if ((e = ppinit(filter, &proc))) {
        marslog(LOG_EROR, "Interpolation initialisation failed (%d)", e);
        ppdone();
        return NULL;
    }


    w = new_fieldset(v->count);

    for (i = 0; i < v->count && e == 0; i++) {
        field* g = get_field(v, i, packed_mem);
        /* On Rotation, output header can be bigger than
           the input header. Give a bit of space */
        long len          = MAX(g->length + 5 * sizeof(int), ppestimate());
        const void* ibuff = NULL;
        size_t ibuff_len;
        char* obuff = (char*)reserve_mem(len);

        grib_get_message(g->handle, &ibuff, &ibuff_len);
        e = ppintf((char*)ibuff, g->length, obuff, &len, true);

        release_field(g);

        if (e == 0) {
            g = w->fields[i] = mars_new_field();
            g->handle        = grib_handle_new_from_message_copy(0, obuff, len);
            g->shape         = packed_mem;
            g->length        = len;
            g->refcnt++;
            save_fieldset(w);
        }

        release_mem(obuff);
    }

    ppdone();

    return e == 0 ? w : NULL;
}


static PProcBuilderT<PProcMIR> mirBuilder("MIR");


//----------------------------------------------------------------------------------------------------------------------

}  // namespace marsclient
