/*
 * © Copyright 1996-2012 ECMWF.
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 * In applying this licence, ECMWF does not waive the privileges and immunities
 * granted to it by virtue of its status as an intergovernmental organisation nor
 * does it submit to any jurisdiction.
 */

#include "mars.h"

extern base_class* targetbase;

static int visited = 0;

boolean enough(request* r, int count) {
    static int need       = 0;
    static boolean init   = false;
    static boolean fields = false;
    static request* p     = NULL;

    if (p != r || !init) {
        p    = r;
        need = count_fields(r);
        if (is_bufr(r))
            fields = false;
        else if (fetch(r))
            fields = false;
        else if (is_odb(r))
            fields = false;
        else
            fields = true;
        init = true;
    }

    if (!fields && count >= need && count > 0)
        return true;

    if (need != 0 && count == need)
        return true;

    /* In case of ALL */
    if (need == 0 && count != 0)
        return true;

    return false;
}

typedef struct retrieve_err_list {
    struct retrieve_err_list* next;
    err e;
    int cnt;
} retrieve_err_list_t;


static const request* multi_choices(const request* t, const request* req, const char* name, err* error) {
    const request* choices = get_subrequest(t, "choices", 0);
    *error                 = -2;
    if (choices) {
        const char* verb        = strcache(lowcase(req->name));
        const request* choice   = choices;
        const request* selected = NULL;
        const char* rules_file  = get_value(t, "rules", 0);

        /* Check the rules */


        if (rules_file) {
            rule* rules = read_check_file(config_file(rules_file));
            int e       = check_one_request(rules, req) ? NOERR : -1;
            free_rule(rules);
            if (e) {
                *error = 0;
                return NULL;
            }
        }

        while (choice) {
            const char* p;
            int i = 0;
            while ((p = get_value(choice, "verb", i++))) {
                if (p == verb) {
                    if (selected) {
                        marslog(LOG_EROR, "Database %s: ambiguous configuration for '%s'", name, verb);
                        print_one_request(selected);
                        marslog(LOG_EROR, "and");
                        print_one_request(choice);
                        return NULL;
                    }
                    selected = choice;
                }
            }
            if (i == 1) {
                marslog(LOG_WARN, "Database %s: incomplete configuration, missing 'verb'", name);
                print_one_request(choice);
            }

            choice = choice->next;
        }

        if (selected == NULL) {
            marslog(LOG_EROR, "Database %s: configuration does not cater for '%s'", name, verb);
            return NULL;
        }

        if (get_value(selected, "class", 0)) {
            return selected;
        }

        if (get_value(selected, "database", 0)) {
            if (EQ(get_value(selected, "database", 0), name)) {
                marslog(LOG_EROR, "Database %s: configuration is defined using itself '%s'", name);
                print_one_request(selected);
                return NULL;
            }
            return findbase(get_value(selected, "database", 0), req);
        }

        marslog(LOG_EROR, "Database %s: configuration entry should have 'database' or 'class'", name);
        print_one_request(selected);
        return NULL;
    }
    return t;
}

const request* findbase(const char* name, const request* req) {
    request* t = 0;
    request* s = mars.setup;
    err ignore;

    if (!name)
        return 0;

    while (s && !t) {
        if (EQ(s->name, "database")) {
            const char* p;
            int i = 0;

            while ((p = get_value(s, "name", i++)) && !t)
                if (strcasecmp(p, name) == 0)
                    t = s;
        }
        s = s->next;
    }

    if (t == 0) {
        request* s = mars.setup;

        marslog(LOG_EROR, "The database '%s' is not known", name);
        marslog(LOG_EROR, "The valid values are: ");

        while (s) {
            if (EQ(s->name, "database")) {
                const char* p;
                int i = 0;
                while ((p = get_value(s, "name", i++)))
                    marslog(LOG_INFO, "%s", p);
            }
            s = s->next;
        }
    }

    return multi_choices(t, req, name, &ignore);
}

database* openbase(const request* s, request* r, const char** name, request** cache, int mode) {
    const request* env = get_environ();
    const char* c      = get_value(s, "class", 0);
    const char* n      = get_value(s, "name", 0);
    const char* x      = get_value(s, "cache", 0);
    const char* a      = get_value(s, "active", 0);
    database* base     = 0;

    if (a && EQ(a, "false")) {
        marslog(LOG_EROR, "Database %s is not active", n);
        return NULL;
    }

    *name  = n;
    *cache = NULL;

    qmonitor("open %s for %s", n, mode == READ_MODE ? "read" : "write");
    base = database_open(base_class_by_name(c), n, r, env, mode);

    if (base && !database_check(base, r)) {
        database_close(base);
        qmonitor("close %s", n);
        return NULL;
    }

    *cache = findbase(x, r);

    return base;
}

static err retrieve(request* r, request* setup, database* target, int* cnt, postproc pproc) {
    request* w;
    int ret = NOERR;
    long length;
    boolean done = false;
    const char* name;
    database* base        = NULL;
    database* cache       = NULL;
    request* cachesetup   = 0;
    const char* cachename = 0;
    boolean fields;
    long64 total        = 0;
    long64 total_target = 0;

    *cnt = 0;

    /* Structure to return 2 or more fields from post-processing.
       Most times (ie, scalar post-processing), only 1 field is returned */
    static ppbuffer_t* pp = NULL;
    static int nbuffers   = 2;
    static long buflen    = 0;
    int i                 = 0;
    static boolean inform = true;

    buflen = mars.readany_buffer_size;

    if (ppestimate() > buflen) {
        buflen = ppestimate() + 1024 * 1024;
        if (inform) {
            marslog(LOG_WARN, "Using large buffer for post-processing (%sbytes)", bytename(buflen));
            inform = false;
        }
    }

    if (pp == NULL) {
        pp = reserve_mem(sizeof(ppbuffer_t) * nbuffers);
        for (i = 0; i < nbuffers; ++i) {
            pp[i].buffer = NULL;
            pp[i].buflen = buflen;
            pp[i].outlen = 0;
        }
    }

    for (i = 0; i < nbuffers; ++i) {
        if (pp[i].buflen < buflen) {
            if (pp[i].buffer)
                release_mem(pp[i].buffer);
            pp[i].buflen = buflen;
            pp[i].buffer = NULL;
        }

        if (pp[i].buffer == NULL) {
            marslog(LOG_DBUG, "Work buffer is %d", pp[i].buflen);
            pp[i].buffer = reserve_mem(pp[i].buflen);
        }
    }

    base = openbase(setup, r, &name, &cachesetup, READ_MODE);

    if (!base) {
        return TOO_SHORT_ERR;
    }

    /* rewind target */
    database_control(target, CNTL_REWIND, NULL, 0);

    visited++;

    ret = NOERR;

    if (is_bufr(r)) {
        w      = NULL;
        cache  = NULL;
        fields = false;

        /* For AI or AF we receive metadata request from DHS */
        if (feedback(r))
            w = clone_one_request(r);
    }
    else if (fetch(r)) {
        fields = false;
        w      = clone_one_request(r);
        cache  = NULL;
    }
    else if (is_odb(r)) {
        fields = false;
        w      = clone_one_request(r);
        cache  = NULL;
    }
    else {
        fields = true;
        w      = clone_one_request(r);
        cache  = NULL;
    }

    while (!done) {
        long len     = pp[0].buflen;
        char* buffer = pp[0].buffer;
        long n       = 0;

        length = pp[0].buflen;

        ret = database_read(base, w, buffer, &length);

        switch (ret) {
            case NOERR:

                total += length;

                /* open the cache if needed */
                if (cachesetup && !cache) {

                    if (!is_bufr(r)) {
                        request* dummy;
                        cache = openbase(cachesetup, r, &cachename, &dummy, WRITE_MODE);
                    }

                    /* don't try to open more than once */

                    cachesetup = NULL;
                }

                /* write to cache before pproc */

                if (cache) {
                    long len = length;
                    if (database_write(cache, w, buffer, &len) != 0) {
                        database_close(cache);
                        cache = NULL;
                    }
                }

                pp[0].inlen  = length;
                pp[0].outlen = pp[0].buflen;
                n            = 0;
                ret          = pproc(pp, &n);
                length       = pp[0].outlen;
                buffer       = pp[0].buffer;

                if (ret == 0) {
                    int i = 0;
                    for (i = 0; i < n; ++i) {
                        long written = pp[i].outlen;
                        if (fields)
                            grib_to_request(w, pp[i].buffer, written);
                        ret = database_write(target, w, pp[i].buffer, &written);
                        if (ret == NOERR)
                            (*cnt)++;
                        else
                            done = true;
                        total_target += written;

                        mars.retrieve_size += written;
                        if (mars.max_retrieve_size && mars.retrieve_size > mars.max_retrieve_size) {
                            marslog(LOG_EROR, "Maximum retrieve size %s reached. Please split your request.", bytename(mars.max_retrieve_size));
                            ret  = QUOTA_ERROR;
                            done = true;
                        }
                    }
                }
                else {
                    if (ret == BUF_TO_SMALL) {
                        marslog(LOG_WARN,
                                "Buffer is too small (%d bytes, %d needed. Retrying)",
                                buflen, length);
                        release_mem(buffer);
                        buffer = NULL;
                        buflen = length;
                    }

                    done = true;
                }
                break;

            case ODB_FOUND_EOF:
                (*cnt)++;
            case EOF:
                ret  = 0;
                done = true;
                break;

            case BUF_TO_SMALL:
                marslog(LOG_WARN,
                        "Buffer is too small (%d bytes, %d needed. Retrying)",
                        buflen, length);
                release_mem(buffer);
                buffer = NULL;
                buflen = length;
                done   = true;
                break;

            case RETRY_ERR:
                marslog(LOG_WARN, "Retrying..");
                (*cnt) = 0;
                database_control(target, CNTL_REWIND, NULL, 0);
                done = true;
                break;

            case NOT_FOUND_7777:
                marslog(LOG_EROR, "Group 7777 not found at the end of GRIB message");
                database_control(target, CNTL_ERROR, &ret, sizeof(ret));
                done = true;
                break;

            case POSTPROC_ERROR:
                marslog(LOG_EROR, "Error in interpolation. Exit");
                database_control(target, CNTL_ERROR, &ret, sizeof(ret));
                done = true;
                break;

            default:
                marslog(LOG_DBUG, "Get error %d while reading data", ret);
                done = true;
                break;
        }
    }

    /* ppdone(); */

    for (i = 0; i < nbuffers; ++i) {
        if (pp[i].buffer)
            release_mem(pp[i].buffer);
        pp[i].buffer = NULL;
    }

    free_all_requests(w);

    if (ret == NOERR) {

        if (enough(r, *cnt)) {
            if (fetch(r)) {
                /* some message here */
            }
            if (observation(r) || track(r) || feedback(r)) {
                int in, out;
                ppcount(&in, &out);
                marslog(LOG_INFO, "%d reports retrieved from '%s'", in, name);
                if (in != out) {
                    marslog(LOG_INFO, "%d reports left after filtering", out);
                }
                log_statistics("reports", "%d", out);
            }
            else if (bias(r)) {
                marslog(LOG_INFO, "%d reports retrieved from '%s'", *cnt, name);
                log_statistics("reports", "%d", *cnt);
            }
            else if (is_odb(r)) {
                marslog(LOG_INFO, "%d ODB message%s retrieved from '%s'", *cnt, *cnt == 1 ? "" : "s", name);
                log_statistics("odbs", "%d", *cnt);
            }
            else {
                marslog(LOG_INFO, "%d field%s retrieved from '%s'", *cnt, *cnt == 1 ? "" : "s", name);
                log_statistics("fields", "%d", *cnt);
            }

            log_statistics("database", "%s", name);
            log_statistics("bytes", "%lld", total);
            log_statistics("written", "%lld", total_target);
        }
        else
            ret = TOO_SHORT_ERR; /* we don't have enough data */
    }

    if (base) {
        database_close(base);
        qmonitor("close %s", name);
    }
    if (cache) {
        database_close(cache);
        qmonitor("close %s", cachename);
    }

    return ret;
}

void mars_task(void) {
}

static err visit_database(request* r, request* s, database* target, int need, int* cnt, postproc pproc) {
    int count = 0;
    err e     = 0;
    char buf[1024];
    timer* t;
    int retry          = 0;
    int slp            = 2;
    const char* dbname = get_value(s, "name", 0);

    sprintf(buf, "Visiting %s", dbname);
    t = get_timer(buf, NULL, true);

    s = multi_choices(s, r, dbname, &e);
    if (!s) {
        return e ? e : TOO_SHORT_ERR;
    }

    timer_start(t);
    do {
        marslog(LOG_DBUG, "Visiting database '%s'", dbname);
        e = retrieve(r, s, target, &count, pproc);
        if (e == RETRY_ERR) {
            if (retry < mars.maxretries) {
                retry++;
                marslog(LOG_WARN, "Sleeping %d minutes", slp);
                sleep(slp * 60);
            }
            else {
                e = -2;
                marslog(LOG_WARN, "Giving up after %d retries", retry);
            }
        }

        if (e == RETRY_FOREVER_ERR) {
            if (slp < 20)
                slp += 2;
            marslog(LOG_WARN, "Sleeping %d minutes", slp);
            sleep(slp * 60);
        }

        if (e == RETRY_SAME_DATABASE) {
            marslog(LOG_WARN, "Request will be retried");
            sleep(5);
        }

    } while (e == BUF_TO_SMALL || e == RETRY_ERR || e == RETRY_FOREVER_ERR || e == RETRY_SAME_DATABASE);
    timer_stop(t, 0);

    *cnt = *cnt + count;
    if (need != *cnt && count > 0) {
        marslog(LOG_WARN, "Visiting database %s : expected %d, got %d", dbname, need, count);
    }

    return e;
}

err handle_retrieve(request* r, void* data) {
    int need = count_fields(r);
    int cnt  = 0;
    database* target;
    err e, f;
    request* u   = r->next;
    request* env = get_environ();
    request* s   = mars.setup;
    int bases    = count_values(r, "DATABASE");
    postproc pproc;
    boolean isbufr = is_bufr(r);
    boolean fields = !isbufr && !fetch(r) && !image(r) && !is_odb(r);

    const char* p      = get_value(r, "EXPECT", 0);
    boolean expect     = p ? atol(p) : 0;
    boolean expect_any = (p && (atol(p) == 0));

    visited = 0;

    if (expect_any) {
        marslog(LOG_INFO, "Requesting any number of %s (request describes %d)", fields ? "fields" : "observations", count_fields_in_request(r));
    }
    else if (fields) {
        if (all_is_used(r)) {
            marslog(LOG_WARN, "Cannot compute number of fields from request.");
            marslog(LOG_WARN, "Try to avoid the use of the value 'ALL'");
        }
        else {
            if (need == 0) {
                marslog(LOG_WARN, "The request does not represent any field");
                marslog(LOG_WARN, "Inform the MARS group if you think it is unusual");
            }
            else {
                marslog(LOG_INFO, "Requesting %d field%s", need, need == 1 ? "" : "s");
            }
        }
    }

    e      = -1;
    target = database_open(targetbase, NULL, r, env, WRITE_MODE);
    if (target == NULL)
        return -1;

    if (isbufr && mars.bufr_empty_target) {
        /* For obs, make sure we have at least an empty file */
        long length = 0;
        if (database_write(target, 0, 0, &length) != 0) {
            return -2;
        }
    }

    r->next = NULL; /* avoid sending to much on the net */

    ppinit(r, &pproc);

    /* A list of statuses for each of the databases visited */
    retrieve_err_list_t db_errs;
    db_errs.next                     = NULL;
    db_errs.e                        = 0;
    retrieve_err_list_t* last_db_err = &db_errs;

    if (bases != 0) {
        int i;
        for (i = 0; i < bases; i++) {
            const char* b    = get_value(r, "DATABASE", i);
            const request* t = findbase(b, r);
            if (t) {
                int last     = i == (bases - 1);
                boolean save = mars.infomissing;

                if (last)
                    mars.infomissing = true;
                int cnt_old      = cnt;
                e                = visit_database(r, t, target, need, &cnt, pproc);
                mars.infomissing = save;

                last_db_err->next = malloc(sizeof(retrieve_err_list_t));
                last_db_err       = last_db_err->next;
                last_db_err->next = NULL;
                last_db_err->e    = e;
                last_db_err->cnt  = cnt - cnt_old;

                if (e == NOERR)
                    break;
                if (e == QUOTA_ERROR)
                    break;
            }
        }
    }
    else {
        while (s) {
            if (EQ(s->name, "database")) {
                const char* p = get_value(s, "visit", 0);
                if (p == 0 || !EQ(p, "false")) {
                    int cnt_old = cnt;
                    e           = visit_database(r, s, target, need, &cnt, pproc);

                    last_db_err->next = malloc(sizeof(retrieve_err_list_t));
                    last_db_err       = last_db_err->next;
                    last_db_err->next = NULL;
                    last_db_err->e    = e;
                    last_db_err->cnt  = cnt - cnt_old;

                    if (e == NOERR)
                        break;
                    if (e == QUOTA_ERROR)
                        break;
                }
            }

            s = s->next;
        }
    }

    r->next = u;

    ppdone();

    /* If the last visited database returned no error, then we are all good!
     * Otherwise, the handling depends on the data type */

    if (e == EOF || e == TOO_SHORT_ERR) {

        if (cnt == 0 && visited > 0) {
            e = NOERR;
            for (retrieve_err_list_t* err = db_errs.next; err != NULL; err = err->next) {
                if (err->e != NOERR && err->e != EOF && err->e != TOO_SHORT_ERR) {
                    e = err->e;
                }
            }
        }

        if (e) {
            marslog(LOG_EROR, "Last error is %d", e);
        }

        if (!enough(r, cnt) && is_bufr(r) && visited > 0) {
            e = NOERR;
        }
        else if (!enough(r, cnt)) {
            const char* expect = get_value(r, "EXPECT", 0);
            if (expect == NULL || atol(expect) != 0) {
                if (all_is_used(r)) {
                    marslog(LOG_EROR, "MARS considers %d fields retrieved are not enough", cnt);
                    e = -1;
                }
                else {
                    if (need != cnt) {
                        marslog(LOG_EROR, "Expected %d, got %d.", need, cnt);
                        e = -1;
                    }
                }
            }
            else {
                if (e == TOO_SHORT_ERR || e == NOERR) {
                    long length = 0;
                    marslog(LOG_WARN, "No data, but EXPECT was provided");
                    database_write(target, 0, 0, &length);
                    e = 0;
                }
            }
        }
    }

    /* cleanup */

    retrieve_err_list_t* next_err = db_errs.next;
    while (next_err != NULL) {
        retrieve_err_list_t* tmp = next_err;
        next_err                 = next_err->next;
        free(tmp);
    }

    if (e)
        database_control(target, CNTL_ERROR, &e, sizeof(e));

    f = database_close(target);
    if (f && !e)
        e = f;

    if (visited == 0) {
        marslog(LOG_EROR, "Could not find a MARS database to perform request");
        e = -2;
    }

    if (e)
        marslog(LOG_EROR, "Request failed");

    return e;
}
