/***************************** LICENSE START ***********************************

 Copyright 2012 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

#include <fstream>
#include <iostream>
#include <map>
#include <sstream>
#include <string>
#include <vector>

#include "fstream_mars_fix.h"

#include <time.h>

#include "Metview.h"
#include "MvObs.h"
#include "MvException.h"
#include "MvMiscellaneous.h"
#include "MvGeoPoints.h"

const char cTab = 9;  // ascii code for 'tab'
const char cComma = ',';


const std::string TEMP_2M = "airTemperatureAt2M";  // 12004;

enum eLevel
{
    LVL_Surface,
    LVL_SingleLevel,
    LVL_ThicknessLayer,
    LVL_Occurrence,
    LVL_CoordinateValue,
    LVL_CoordinateRange
};

enum eOutput
{
    eBUFR,
    eGEOPOINTS,
    ePOLAR_VECTOR,
    eXY_VECTOR,
    eCSV,
    eNCOLS
};

std::string outTypeStr[] = {"BUFR", "GEOPOINTS", "POLAR_VECTOR", "XY_VECTOR", "CSV", "NCOLS"};

//__________________________________________________________ ObsFilter

class ObsFilter : public MvService
{
public:
    ObsFilter();
    ~ObsFilter();

    ObsFilter(const ObsFilter& anOther) = delete;
    void operator=(const ObsFilter& anOther) = delete;

    void serve(MvRequest&, MvRequest&);

protected:
    void getControl();
    void getParams();
    void getData();
    void getParameterSpecifiers();
    void getTypes();
    void getTimes();
    void getWmoIds();
    void getLocation();
    void getCustom();
    MvRequest createRequest();
    void add();
    void close();
    float surfaceValue(long parInd);
    float pressureLevelValue(float level, long parInd);
    float layerValue(float level1, float level2, long parInd);
    float occurranceValue(int occurranceInd, long parIndex);
    float coordinateLevelValue(const std::string&, float, int);
    float coordinateRangeValue(const std::string&, float, float, int);

private:
    std::string moduleLabel_{"ObsFiler-> "};
    long _counter{0};
    MvRequest _in;
    MvRequest _data;
    MvObs obs_;
    MvObsSet* _inSet{nullptr};
    MvObsSetIterator* _iter{nullptr};
    eOutput outType_{eBUFR};
    MvObsSet* _outBufr{nullptr};
    std::ofstream _outFile;
    eLevel _level{LVL_Surface};
    float _firstLevel{0.};
    float _secondLevel{0.};
    long _occurrenceIndex{0};
    std::vector<std::string> param_;
    ENextReturn _obsOrMsg{NR_returnMsg};  //-- for multisubset msgs (filtering also with subset data?)
    bool _includeMissingValues{false};
    float _missingValue{GEOPOINTS_MISSING_VALUE};
    bool failOnError_{true};
    bool failOnEmptyOutput_{false};  //-- this breaks backwards compatibility
    std::string _coordLevelDescriptor;
    bool hasAttributeCondition_{false};
};


//_________________________________________________________

ObsFilter::ObsFilter() :
    MvService("OBSFILTER")
{
}
//_________________________________________________________

ObsFilter::~ObsFilter()
{
    delete _iter;
    delete _inSet;
    delete _outBufr;
    obs_.clear();
}


void ObsFilter::getControl()
{
    if (strcmp(_in("FAIL_ON_ERROR"), "NO") == 0)
        failOnError_ = false;
    else
        failOnError_ = true;  // default value

    if (strcmp(_in("FAIL_ON_EMPTY_OUTPUT"), "YES") == 0)
        failOnEmptyOutput_ = true;
    else
        failOnEmptyOutput_ = false;  // default value

    //-- O U T P U T --
    outType_ = (eOutput)((int)_in("OUTPUT"));
}


//_________________________________________________________
void ObsFilter::getData()
{
    //-- D A T A --
    _in.getValue(_data, "DATA");

    if (!_data) {
        throw MvException("empty or unspecified input data");
    }

    _inSet = new MvObsSet(_data);
    _iter = new MvObsSetIterator(*_inSet);
}

//_________________________________________________________
void ObsFilter::getParameterSpecifiers()
{
    //-- P A R A M E T E R --
    int nrParams = _in.countValues("PARAMETER");

    param_.clear();
    if (nrParams == 0)  //-- first descriptor (parameter)
        param_.push_back(TEMP_2M);
    else
        param_.push_back((const char*)_in("PARAMETER"));

    if (nrParams > 1)  //-- if more descriptors (parameters)
    {
        for (int n = 1; n < nrParams; ++n)
            param_.push_back((const char*)_in("PARAMETER", n));
    }
    if ((outType_ == ePOLAR_VECTOR || outType_ == eXY_VECTOR) && nrParams < 2)
        throw MvException("ObsFilter-> Too few parameters defined. Wind data takes two parameters!!");

    const char* myMissingData = _in("MISSING_DATA");
    if (myMissingData && strcmp(myMissingData, "INCLUDE") == 0) {
        _includeMissingValues = true;
        _missingValue = (double)_in("MISSING_DATA_VALUE");
    }

    // Check if there are attributes in params
    for (auto& i : param_) {
        if (i.find("->") != std::string::npos) {
            hasAttributeCondition_ = true;
            break;
        }
    }

    //-- L E V E L --
    const char* levelStr = _in("LEVEL");
    if (levelStr && strcmp(levelStr, "SINGLE") == 0)
        _level = LVL_SingleLevel;
    else if (levelStr && strcmp(levelStr, "THICKNESS") == 0)
        _level = LVL_ThicknessLayer;
    else if (levelStr && strcmp(levelStr, "OCCURRENCE") == 0)
        _level = LVL_Occurrence;
    else if (levelStr && strcmp(levelStr, "DESCRIPTOR_VALUE") == 0)
        _level = LVL_CoordinateValue;
    else if (levelStr && strcmp(levelStr, "DESCRIPTOR_RANGE") == 0)
        _level = LVL_CoordinateRange;
    else
        _level = LVL_Surface;  // default value

    if (_in.countValues("FIRST_LEVEL") > 0)
        _firstLevel = (double)_in("FIRST_LEVEL");
    else
        _firstLevel = 0.;  // default value

    if (_in.countValues("SECOND_LEVEL") > 0)
        _secondLevel = (double)_in("SECOND_LEVEL");
    else
        _secondLevel = 0.;  // default value

    if (_in.countValues("OCCURRENCE_INDEX") > 0)
        _occurrenceIndex = (long)_in("OCCURRENCE_INDEX");
    else
        _occurrenceIndex = 0L;  // default value

    if (_in.countValues("LEVEL_DESCRIPTOR") > 0)
        _coordLevelDescriptor = (const char*)_in("LEVEL_DESCRIPTOR");
    else
        _coordLevelDescriptor = "";  // 0L;  //default value

    // Check if there are attributes in _coordLevelDescriptor

    if (_coordLevelDescriptor.find("->") != std::string::npos) {
        hasAttributeCondition_ = true;
    }
}
//_________________________________________________________
void ObsFilter ::getTypes()
{
    int i;

    // It requires that this parameter returns number(s)
    const char* obsTypeStr = (const char*)_in("OBSERVATION_TYPES");
    if (obsTypeStr) {
        long myType = -999;
        for (i = 0; i < _in.countValues("OBSERVATION_TYPES"); i++) {
            if ((myType = (long)_in("OBSERVATION_TYPES", i)) >= 0)
                _iter->setMessageType((int)myType);
        }
    }

    // It requires that this parameter returns number(s)
    const char* obsSubTypeStr = (const char*)_in("OBSERVATION_SUBTYPES");
    if (obsSubTypeStr) {
        long mySubType = -999;
        for (i = 0; i < _in.countValues("OBSERVATION_SUBTYPES"); i++) {
            if ((mySubType = (long)_in("OBSERVATION_SUBTYPES", i)) >= 0)
                _iter->setMessageSubtype((int)mySubType);
        }
    }
}
//_________________________________________________________
void ObsFilter ::getTimes()
{
    const char* obsTimeStr = _in("DATE_AND_TIME_FROM");
    if (obsTimeStr && strcmp(obsTimeStr, "DATA") == 0)
        _iter->useObsTime(true);

    const char* dateStr = _in("DATE");
    const char* timeStr = _in("TIME");
    if (dateStr && strcmp(dateStr, "ANY") == 0 &&
        timeStr && strcmp(timeStr, "ANY") == 0) {
        // both DATE and TIME are ANY
    }  // TIME==ANY && DATE==value is illegal
    else {
        int myTime = (int)_in("TIME");
        int myDelta = (int)_in("RESOLUTION_IN_MINS");

        if (dateStr && strcmp(dateStr, "ANY") == 0)  // only TIME is given
        {
            myDelta = 100 * (myDelta / 60) + myDelta % 60;  // minutes into HHMM
            if ((myDelta % 100) != 0)
                _iter->setTimeRangeWithoutDate(myTime - myDelta - 40  // 40=100-60!!!
                                               ,
                                               myTime + myDelta);
            else
                _iter->setTimeRangeWithoutDate(myTime - myDelta, myTime + myDelta);
        }
        else  // both TIME and DATE given
        {
            long myDate = (int)_in("DATE");
            TDynamicTime myObsTime(myDate, myTime);
            _iter->setTimeRange(myObsTime, myDelta);
        }
    }
}
//_________________________________________________________
void ObsFilter ::getWmoIds()
{
    int i;

    const char* blockStr = _in("WMO_BLOCKS");
    if (blockStr && strcmp(blockStr, "ANY") == 0) {
    }
    else {
        long myWmoBlock;
        for (i = 0; i < _in.countValues("WMO_BLOCKS"); i++) {
            if ((myWmoBlock = (long)_in("WMO_BLOCKS", i)) > 0)
                _iter->setWmoBlock((int)myWmoBlock);
        }
        _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
    }

    const char* stationStr = _in("WMO_STATIONS");
    if (stationStr && strcmp(stationStr, "ANY") == 0) {
    }
    else {
        long myWmoStation;
        for (i = 0; i < _in.countValues("WMO_STATIONS"); i++)  // int i; ????
        {
            if ((myWmoStation = (long)_in("WMO_STATIONS", i)) > 0)
                _iter->setWmoStation(myWmoStation);
        }
        _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
    }
}
//_________________________________________________________
void ObsFilter ::getLocation()
{
    const char* locFilterStr = _in("LOCATION_FILTER");
    if (locFilterStr && strcmp(locFilterStr, "AREA") == 0) {
        const char* areaStr = _in("AREA");
        if (areaStr) {
            if (_in.countValues("AREA") == 4) {
                float y1 = (double)_in("AREA", 0);
                float x1 = (double)_in("AREA", 1);
                float y2 = (double)_in("AREA", 2);
                float x2 = (double)_in("AREA", 3);
                MvLocation loc1(y1, x1);
                MvLocation loc2(y2, x2);
                _iter->setArea(loc1, loc2);
                _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
            }
            else
                sendProgress("ObsFilter: Errors in AREA-parameter, Area filter not set!");
        }
    }


    if (locFilterStr && strcmp(locFilterStr, "LINE") == 0) {
        const char* lineStr = _in("LINE");
        if (lineStr) {
            if (_in.countValues("LINE") == 4) {
                float myDeltaInM = (double)_in("DELTA_IN_KM") * 1000;
                float y1 = (double)_in("LINE", 0);
                float x1 = (double)_in("LINE", 1);
                float y2 = (double)_in("LINE", 2);
                float x2 = (double)_in("LINE", 3);
                MvLocation loc1(y1, x1);
                MvLocation loc2(y2, x2);
                _iter->setXSectionLine(loc1, loc2, myDeltaInM);
                _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
            }
            else
                sendProgress("ObsFilter: Errors in LINE-parameter, Line filter not set!");
        }
    }
}
//_________________________________________________________
void ObsFilter ::getCustom()
{
    const char* customFilterStr = _in("CUSTOM_FILTER");
    std::string customDescriptor = "";
    if ((const char*)_in("CUSTOM_PARAMETER"))
        customDescriptor = (const char*)_in("CUSTOM_PARAMETER");
    float customFirstValue = 0;
    float customSecondValue = 0;
    int mySelectValueCount = _in.countValues("CUSTOM_VALUES");
    if (mySelectValueCount > 0)
        customFirstValue = (double)_in("CUSTOM_VALUES", 0);
    if (mySelectValueCount > 1)
        customSecondValue = (double)_in("CUSTOM_VALUES", 1);

    if (customFilterStr && strcmp(customFilterStr, "VALUE") == 0) {
        for (int i = 0; i < mySelectValueCount; i++) {
            _iter->select(customDescriptor, (double)_in("CUSTOM_VALUES", i));
        }
        _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
    }

    if (customFilterStr && strcmp(customFilterStr, "RANGE") == 0) {
        if (mySelectValueCount == 2) {
            _iter->selectRange(customDescriptor, customFirstValue, customSecondValue);
            _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
        }
        else
            throw MvException("ObsFilter-> Custom Filter/Range: number of CUSTOM_VALUES != 2 !!!");
    }

    if (customFilterStr && strcmp(customFilterStr, "EXCLUDE") == 0) {
        if (mySelectValueCount == 2) {
            _iter->excludeRange(customDescriptor, customFirstValue, customSecondValue);
            _obsOrMsg = NR_returnObs;  //-- have to study each subset (obs)
        }
        else
            throw MvException("ObsFilter-> Custom Filter/ExcludeRange: number of CUSTOM_VALUES != 2");
    }

    // Check if there are attributes in custom param
    if (customDescriptor.find("->") != std::string::npos) {
        hasAttributeCondition_ = true;
    }
}
//_________________________________________________________
void ObsFilter ::getParams()
{
    _obsOrMsg = NR_returnMsg;  //-- set msg level as default

    //-- these have no effect on _obsOrMsg --
    getParameterSpecifiers();
    getTypes();

    //-- these may have an effect on _obsOrMsg --
    getTimes();
    getWmoIds();
    getLocation();
    getCustom();

    if (outType_ == eBUFR) {
        if (_obsOrMsg == NR_returnObs)
            sendProgress("ObsFilter-> Filtering with data values requested, have to study each subset (obs)");
        else
            sendProgress("ObsFilter-> Filtering on msg level (only section header info used)");
    }
    else
        _obsOrMsg = NR_returnObs;  //-- for geopoints always on obs level!!!
}

//_________________________________________________________
MvRequest
ObsFilter ::createRequest()
{
    switch (outType_) {
        case eBUFR:  //-- BUFR
        {
            MvRequest x("BUFR");  //-- BUFR return request
            x("TEMPORARY") = 1;
            x("PATH") = marstmp();
            _outBufr = new MvObsSet(x, "w");

            return x;
        }

        case eCSV:  //-- CSV
        {
            char* fileName = marstmp();

            MvRequest x("TABLE");  //-- CSV return request (TABLE)
            x("TEMPORARY") = 1;
            x("PATH") = fileName;

            _outFile.open(fileName, std::ios::out);
            //-- construct CSV header line
            for (unsigned int p = 0; p < param_.size(); ++p) {
                if (p > 0)
                    _outFile << cComma;
                _outFile << "'" << param_[p] << "'";
            }
            _outFile << std::endl;

            return x;
        }

        case eGEOPOINTS:
        case ePOLAR_VECTOR:
        case eXY_VECTOR: {
            char* fileName = marstmp();

            MvRequest x("GEOPOINTS");  //-- GEOPOINTS return request
            x("TEMPORARY") = 1;
            x("PATH") = fileName;

            _outFile.open(fileName, std::ios::out);

            _outFile << "#GEO" << std::endl;  //-- construct GEOPOINTS header

            if (outType_ == ePOLAR_VECTOR || outType_ == eXY_VECTOR)
                _outFile << "#FORMAT " << outTypeStr[outType_] << std::endl;

            if (outType_ == eGEOPOINTS)
                _outFile << "PARAMETER = " << param_[0] << std::endl;
            else
                _outFile << "PARAMETER = " << param_[0] << cTab << param_[1] << std::endl;

            _outFile << "#lat" << cTab << "long" << cTab
                     << "level" << cTab << "date" << cTab
                     << "time" << cTab;
            if (outType_ == eGEOPOINTS)
                _outFile << "value";
            else
                _outFile << "speed|u" << cTab << "direction|v";
            _outFile << std::endl;

            _outFile << "#DATA" << std::endl;

            return x;
        }

        case eNCOLS: {
            char* fileName = marstmp();

            MvRequest x("GEOPOINTS");
            x("TEMPORARY") = 1;
            x("PATH") = fileName;

            _outFile.open(fileName, std::ios::out);

            _outFile << "#GEO" << std::endl;
            _outFile << "#FORMAT NCOLS" << std::endl;

            _outFile << "#COLUMNS " << std::endl
                     << "stnid" << cTab << "latitude" << cTab << "longitude" << cTab
                     << "level" << cTab << "date" << cTab
                     << "time" << cTab;

            for (auto& p : param_) {
                _outFile << p << cTab;
            }

            _outFile << std::endl
                     << "#DATA" << std::endl;

            return x;
        }
    }

    MvRequest x("ERROR");  //-- internal error if here
    return x;
}
//_________________________________________________________
float ObsFilter::surfaceValue(long parInd)
{
    return obs_.valueC(param_[parInd]);
}
//_________________________________________________________
float ObsFilter::pressureLevelValue(float level, long parInd)
{
    return obs_.valueByPressureLevelC(level, param_[parInd]);
}
//_________________________________________________________
float ObsFilter::layerValue(float level1, float level2, long parInd)
{
    return obs_.valueByLayerC(level1, level2, param_[parInd]);
}
//_________________________________________________________
float ObsFilter::occurranceValue(int occurranceInd, long parIndex)
{
    return obs_.valueByOccurrenceC(occurranceInd, param_[parIndex]);
}
//_________________________________________________________
float ObsFilter::coordinateLevelValue(const std::string& coordDesc, float level, int parInd)
{
    return obs_.valueByLevelC(coordDesc, level, param_[parInd]);
}
//_________________________________________________________
float ObsFilter::coordinateRangeValue(const std::string& coordDesc, float level1, float level2, int parInd)
{
    return obs_.valueByLevelRangeC(coordDesc, level1, level2, param_[parInd]);
}

//_________________________________________________________
void ObsFilter ::add()
{
    // Expanded message if it is needed
    if (outType_ != eBUFR)
        obs_.expand();

    if (outType_ == eBUFR)  //-- BUFR
    {
        if (_counter == 0)  //-- BUFR: first time
        {
            //-- changed 2011-12-30/vk:
            //--  o target BUFR file is set to contain single subsets, but this is in effect
            //--     only when filtering using data values from BUFR Section 4
            //--  o this setting has no effect when filtering only with metadata values in
            //--     BUFR Section 1; in such cases multisubset messages are copied as they
            //--     are (metadata is common to all subsets in a multisubset msg)
            _outBufr->setSubsetMax(1);  //-- obs_.msgSubsetCount() );
        }

        //-- if filtering requires values from observation then multisubset
        //-- messages have to be filtered on observation (subset) level !
        if (_obsOrMsg == NR_returnMsg) {
            _outBufr->write(obs_);              //-- BUFR: filtering with "meta data" only
            _counter += obs_.msgSubsetCount();  //-- BUFR: i.e. write the whole message!
        }
        else {
            _outBufr->add(obs_);  //-- BUFR: filtering also with data values
            _counter++;           //-- BUFR: write current subset only!
        }
    }
    else if (outType_ == eCSV)  //-- CSV
    {
        std::ostringstream os;
        bool abortOnMissing = false;
        for (unsigned int p = 0; p < param_.size(); ++p) {
            if (p > 0)
                os << cComma;

            float val;
            switch (_level) {
                case LVL_Surface:
                    val = surfaceValue(p);
                    break;
                case LVL_SingleLevel:
                    val = pressureLevelValue(_firstLevel, p);
                    break;
                case LVL_ThicknessLayer:
                    val = layerValue(_firstLevel, _secondLevel, p);
                    break;
                case LVL_Occurrence:  //-- CSV: this does not make much sence here !!!
                    val = occurranceValue(_occurrenceIndex, p);
                    break;
                case LVL_CoordinateValue:
                    val = coordinateLevelValue(_coordLevelDescriptor, _firstLevel, p);
                    break;
                case LVL_CoordinateRange:
                    val = coordinateRangeValue(_coordLevelDescriptor, _firstLevel, _secondLevel, p);
                    break;
                default:
                    break;
            }

            if (obs_.elementValueType(param_[0]) == CODES_TYPE_STRING) {
                //-- string value has only basic retrieve...
                os << obs_.stringValue(param_[0]);
            }
            else {
                if (val == kBufrMissingValue)  //-- CSV: missing value?
                {
                    if (_includeMissingValues)
                        val = _missingValue;  //-- CSV: chage to user defined missing value
                    else {
                        abortOnMissing = true;  //-- CSV: force ignoring the whole obs
                        break;
                    }
                }

                os << val;  //-- CSV: write value or user defined missing value
            }
        }

        if (!abortOnMissing) {
            _outFile << os.str() << std::endl;
            _counter++;
        }
    }

    // traditional geopoints
    else if (outType_ != eNCOLS) {
        float myValue = 0;
        float myDirec = 0;  //-- for geovectors

        switch (_level) {
            case LVL_Surface:
                myValue = surfaceValue(0);
                if (outType_ > eGEOPOINTS)
                    myDirec = surfaceValue(1);
                break;

            case LVL_SingleLevel:
                myValue = pressureLevelValue(_firstLevel, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = pressureLevelValue(_firstLevel, 1);
                break;

            case LVL_ThicknessLayer:
                myValue = layerValue(_firstLevel, _secondLevel, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = layerValue(_firstLevel, _secondLevel, 1);
                break;

            case LVL_Occurrence:
                myValue = occurranceValue(_occurrenceIndex, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = occurranceValue(_occurrenceIndex, 1);
                break;

            case LVL_CoordinateValue:
                myValue = coordinateLevelValue(_coordLevelDescriptor, _firstLevel, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = coordinateLevelValue(_coordLevelDescriptor, _firstLevel, 1);
                break;

            case LVL_CoordinateRange:
                myValue = coordinateRangeValue(_coordLevelDescriptor, _firstLevel, _secondLevel, 0);
                if (outType_ > eGEOPOINTS)
                    myDirec = coordinateRangeValue(_coordLevelDescriptor, _firstLevel, _secondLevel, 1);
                break;

            default:
                break;
        }

        if (myValue != kBufrMissingValue || _includeMissingValues) {
            MvLocation myLoc = obs_.location();
            if (myLoc.latitude() != kBufrMissingValue) {
                _outFile << myLoc.latitude() << cTab;
            }
            else {
                _outFile << _missingValue << cTab;
            }
            if (myLoc.longitude() != kBufrMissingValue) {
                _outFile << myLoc.longitude() << cTab;
            }
            else {
                _outFile << _missingValue << cTab;
            }

            TStaticTime myTime = obs_.obsTime();

            _outFile << _firstLevel << cTab;

            _outFile << myTime.GetYear();
            _outFile.width(2);
            _outFile.fill('0');
            _outFile << myTime.GetMonth();
            _outFile.width(2);
            _outFile.fill('0');
            _outFile << myTime.GetDay() << cTab;
            _outFile.width(2);
            _outFile.fill('0');
            _outFile << myTime.GetHour();
            _outFile.width(2);
            _outFile.fill('0');
            _outFile << myTime.GetMin() << cTab;

            if (obs_.elementValueType(param_[0]) == CODES_TYPE_STRING) {
                //-- string value has only basic retrieve...
                _outFile << obs_.stringValue(param_[0]);
            }
            else {
                if (_includeMissingValues && myValue == kBufrMissingValue)
                    myValue = _missingValue;

                _outFile << myValue;

                if (outType_ > eGEOPOINTS)
                    _outFile << cTab << myDirec;  //-- should we check against missing value?
            }

            _outFile << std::endl;

            _counter++;
        }
    }

    // Multi column geopoints
    else {
        std::vector<float> vals(param_.size(), 0);

        for (size_t p = 0; p < param_.size(); p++) {
            float val = kBufrMissingValue;
            switch (_level) {
                case LVL_Surface:
                    val = surfaceValue(p);
                    break;
                case LVL_SingleLevel:
                    val = pressureLevelValue(_firstLevel, p);
                    break;
                case LVL_ThicknessLayer:
                    val = layerValue(_firstLevel, _secondLevel, p);
                    break;
                case LVL_Occurrence:
                    val = occurranceValue(_occurrenceIndex, p);
                    break;
                case LVL_CoordinateValue:
                    val = coordinateLevelValue(_coordLevelDescriptor, _firstLevel, p);
                    break;
                case LVL_CoordinateRange:
                    val = coordinateRangeValue(_coordLevelDescriptor, _firstLevel, _secondLevel, p);
                    break;
                default:
                    break;
            }

            vals[p] = val;

            if (val == kBufrMissingValue && !_includeMissingValues)
                return;
        }

        MvLocation myLoc = obs_.location();
        TStaticTime myTime = obs_.obsTime();
        std::string ident = obs_.headerIdent();
        ident = metview::stationIdForWritingToFile(ident);

        _outFile << ident << cTab;
        if (myLoc.latitude() != kBufrMissingValue) {
            _outFile << myLoc.latitude() << cTab;
        }
        else {
            _outFile << _missingValue << cTab;
        }
        if (myLoc.longitude() != kBufrMissingValue) {
            _outFile << myLoc.longitude() << cTab;
        }
        else {
            _outFile << _missingValue << cTab;
        }

        _outFile << _firstLevel << cTab;

        _outFile << myTime.GetYear();
        _outFile.width(2);
        _outFile.fill('0');
        _outFile << myTime.GetMonth();
        _outFile.width(2);
        _outFile.fill('0');
        _outFile << myTime.GetDay() << cTab;
        _outFile.width(2);
        _outFile.fill('0');
        _outFile << myTime.GetHour();
        _outFile.width(2);
        _outFile.fill('0');
        _outFile << myTime.GetMin() << cTab;


        for (size_t p = 0; p < param_.size(); p++) {
            if (obs_.elementValueType(param_[p]) == CODES_TYPE_STRING) {
                //-- string value has only basic retrieve...
                _outFile << obs_.stringValue(param_[p]);
            }
            else {
                if (_includeMissingValues && vals[p] == kBufrMissingValue)
                    vals[p] = _missingValue;

                _outFile << vals[p];
            }

            if (p < param_.size() - 1)
                _outFile << cTab;
        }

        _outFile << std::endl;
        _counter++;
    }

    obs_.clearIterator();
}

//_________________________________________________________
void ObsFilter ::close()
{
    if (outType_ == eBUFR)
        _outBufr->close();
    else
        _outFile.close();
}

//_________________________________________________________
void ObsFilter ::serve(MvRequest& in, MvRequest& out)
{
    std::cout << "ObsFilter::serve() in" << std::endl;
    in.print();

    time_t myStartTime;
    time(&myStartTime);
    clock_t myStartCpu = clock();
    _counter = 0;

    // The input request needs to be expanded with flag EXPAND_LAST_NAME.
    // This is required by functions getTypes and getCustom.
    _in = in.ExpandRequest("ObsFilterDef", "ObsFilterRules", EXPAND_LAST_NAME);

    // get some params upfront
    getControl();

    bool hasData=false;

    // get data. We do it in a separate block to check if the
    // input is empty or invalid. FailOnError does not apply at this stage!
    try {     
        // load the data
        getData();
        hasData=true;
    } catch (MvException& e) {
        if (!failOnEmptyOutput_) {
            // creates a properly formatted (with the exception of bufr)
            // empty output file
            getParameterSpecifiers();
            out = createRequest();
            close();
        }
        std::string err = "ObsFilter-> Input error: ";
        err += e.what();
        setError(0, err.c_str());
    }

    if (hasData) {
        try {
            getParams();  //-- new _inSet & _iter

            // the output request
            MvRequest x = createRequest();

            // we turn off caching compressed data - having it on does not improve performance for
            // obsfilter
            _inSet->setCacheCompressedData(false);

            while ((obs_ = (*_iter)(_obsOrMsg))) {
                add();
            }

            // close the output file
            close();

             // initialise MvObs (serve function may be called iteractively)
            // TODO: when can it be iteratively called?
            obs_.clear();

            out = x;

            time_t myEndTime;
            time(&myEndTime);
            double wallClockTime = difftime(myEndTime, myStartTime);
            clock_t myEndCpu = clock();
            double cpuTime = ((double)(myEndCpu - myStartCpu)) / (double)CLOCKS_PER_SEC;  // CLK_TCK;
            sendProgress("ObsFilter-> Filtered %d observations (out of %d) into a %s file in %d secs (cpu %d secs)", _counter, _inSet->obsCount(), outTypeStr[(int)outType_].c_str(), (int)wallClockTime, (int)cpuTime);
        }
        catch (MvException& e) {
            std::string err = e.what();
            if (failOnError_) {
                std::string msg = moduleLabel_ + "Abort[FAIL_ON_ERROR=Yes] Filter error : " + err;
                throw MvException(msg.c_str());
            } else {
                std::string msg = moduleLabel_ + "Filter error: " + err;
                setError(0, msg.c_str());
            }
        }
    }

    // TODO: improve cleanup
    delete _iter;
    _iter = nullptr;

    delete _inSet;
    _inSet = nullptr;

    if (_counter == 0 && failOnEmptyOutput_) {
        throw MvException(moduleLabel_ + "Abort[FAIL_ON_EMPTY_OUTPUT=Yes] output is empty!");
    }

    std::cout << "ObsFilter::serve() out" << std::endl;
    out.print();
}
//_________________________________________________________

int main(int argc, char** argv)
{
    MvApplication theApp(argc, argv);
    ObsFilter filter;

    theApp.run();
}
