/**
 *
 * This file is part of Tulip (http://tulip.labri.fr)
 *
 * Authors: David Auber and the Tulip development Team
 * from LaBRI, University of Bordeaux
 *
 * Tulip is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License
 * as published by the Free Software Foundation, either version 3
 * of the License, or (at your option) any later version.
 *
 * Tulip is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU General Public License for more details.
 *
 */
#include <iostream>
#include <qapplication.h>
#include <qtimer.h>
#include <tulip/TulipPluginHeaders.h>
#include <tulip/DownloadManager.h>
#include <tulip/TlpQtTools.h>
#include "WebImport.h"

using namespace std;
using namespace tlp;

class UrlElement {
public:
  std::string http_prefix;
  std::string data;
  std::string server;
  std::string url;
  std::string clean_url;

  UrlElement();
  UrlElement(const UrlElement &c);
  UrlElement &operator=(const UrlElement &) = default;
  bool load();
  void clear();
  bool isHtmlPage();
  bool is_http() const {
    return !http_prefix.empty();
  }
  UrlElement parseUrl(const std::string &href);
  void setUrl(const std::string &);
  std::string getUrl() const {
    return clean_url.size() ? clean_url : url;
  }

private:
  void fill(std::string &result);
  bool siteconnect(const std::string &server, const std::string &url, bool headonly);

private:
  HttpContext *context;

public:
  bool isValid() {
    return !server.empty();
  }

  bool isRedirected() {
    return context && context->redirected;
  }

  UrlElement getRedirection() {
    return parseUrl(context->data);
  }
  int getCode() {
    return context->code;
  }
};

namespace std {
template <>
struct less<UrlElement> {
  bool operator()(const UrlElement &lhs, const UrlElement &rhs) const {
    if (lhs.server.compare(rhs.server) < 0)
      return true;

    if (lhs.server.compare(rhs.server) > 0)
      return false;

    return lhs.getUrl().compare(rhs.getUrl()) < 0;
  }
};
} // namespace std

HttpContext::HttpContext()
    : status(false), code(-1), reply(nullptr), processed(false), redirected(false), isHtml(false) {}

HttpContext::~HttpContext() {
  if (reply) {
    reply->close();
    reply->deleteLater();
  }
}

void HttpContext::request(const std::string &url, bool head) {
  processed = isHtml = redirected = false;
  QNetworkRequest request(QUrl(url.c_str()));

  if (head) {
    reply = DownloadManager::getInstance()->head(request);
    connect(reply, SIGNAL(finished()), this, SLOT(headerReceived()));
  } else {
    reply = DownloadManager::getInstance()->get(request);
    connect(reply, SIGNAL(finished()), this, SLOT(finished()));
  }
}

void HttpContext::finished() {
  // check to see if it is the request we made
  if (reply != qobject_cast<QNetworkReply *>(sender()))
    return;

  // OK
  processed = true;

  // set status of the request
  if ((status = (reply->error() == QNetworkReply::NoError)))
    data = reply->readAll().data();

  reply->close();
  reply->deleteLater();
  reply = nullptr;
}

void HttpContext::headerReceived() {
  // check to see if it is the request we made
  if (reply != qobject_cast<QNetworkReply *>(sender()))
    return;

  processed = true;

  if ((status = isHtml = (reply->error() == QNetworkReply::NoError))) {
    QVariant value = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);

    if (value.canConvert<int>()) {
      code = value.toInt();

      if (code > 399) /* error codes */
        isHtml = false;
      else if ((code > 299) && (code < 305 || code == 307)) {
        /* redirection codes */
        redirected = true;
        QVariant redirectionTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);

        if (!redirectionTarget.isNull())
          data = QStringToTlpString(redirectionTarget.toUrl().toString());
        else
          data = "";

        reply->close();
        reply->deleteLater();
        reply = nullptr;
      }

      return;
    }

    /* normal codes */
    value = reply->header(QNetworkRequest::ContentTypeHeader);
    status = isHtml =
        (value.canConvert<QString>() && value.toString().contains(QString("text/html")));
    reply->close();
    reply->deleteLater();
    reply = nullptr;
  }
}

void HttpContext::timeout() {
  if (!processed)
    tlp::warning() << "time-out occurs" << endl;

  // if timeout occurs
  // just indicates the we failed during processing
  processed = true;
}

void HttpContext::setTimer(QTimer *timer) {
  connect(timer, SIGNAL(timeout()), SLOT(timeout()));
}

UrlElement::UrlElement() : http_prefix("http://"), data(""), context(nullptr) {}
UrlElement::UrlElement(const UrlElement &c)
    : http_prefix(c.http_prefix), data(""), server(c.server), url(c.url), clean_url(c.clean_url),
      context(nullptr) {}

void UrlElement::setUrl(const std::string &theUrl) {
  url = theUrl;
  size_t len = theUrl.find_first_of("?", 0);

  if (len != string::npos)
    clean_url = theUrl.substr(0, len);
  else
    clean_url.clear();
}

void UrlElement::fill(std::string &result) {
  result += context->data;
}

void UrlElement::clear() {
  if (context) {
    delete context;
    context = nullptr;
  }

  data = "";
}

bool UrlElement::load() {
  if (!siteconnect(server, url, false))
    return false;

  fill(data);
  return true;
}

static const char *not_html_extensions[] = {
    ".bmp", ".css", ".doc", ".ico", ".exe", ".gif", ".gz",  ".js", ".jpeg", ".jpg", ".pdf",
    ".png", ".ps",  ".rss", ".tar", ".tgz", ".wav", ".zip", ".z",  nullptr /* must be the last */
};

bool UrlElement::isHtmlPage() {
  string lowercase(url);
  size_t len = lowercase.length();

  for (size_t i = 0; i < len; ++i)
    lowercase[i] = tolower(lowercase[i]);

  for (unsigned int i = 0; not_html_extensions[i]; i++)
    if (lowercase.rfind(not_html_extensions[i], len) != string::npos)
      return false;

  if (!siteconnect(server, url, true))
    return false;

  return context->isHtml;
}

bool UrlElement::siteconnect(const std::string &server, const std::string &path, bool headonly) {
  // check that we actually got data..
  if (server.empty())
    return false;

  if (!context)
    context = new HttpContext();

  string thePath("/");

  // prefix the path with / if it doesn't start with it..
  if (path.c_str()[0] != '/')
    thePath += url;
  else
    thePath = url;

  string url("http://");
  url += server.c_str() + thePath;

  // start the request
  context->request(url, headonly);

  // block until the request is finished
  // or there is timeout
  QTimer timer;
  timer.setSingleShot(true);
  context->setTimer(&timer);
  timer.start(20000);

  while (!context->processed) {
    QCoreApplication::processEvents();
  }

  timer.stop();
  return context->status && context->code < 400;
} /* end, siteconnect */

static const char *rejected_protocols[] = {
    "ftp:", "gopher:", "sftp:", "javascript:", "mms:", "mailto:", nullptr /* must be the last */
};

UrlElement UrlElement::parseUrl(const std::string &href) {
  UrlElement newUrl;
  string lowercase(href);
  size_t i, len = lowercase.length();

  for (i = 0; i < len; ++i)
    lowercase[i] = tolower(lowercase[i]);

  for (i = 0; rejected_protocols[i] != nullptr; i++) {
    if (lowercase.find(rejected_protocols[i]) != string::npos)
      break;
  }

  if (rejected_protocols[i]) {
    newUrl.http_prefix.clear();

    if (i != 3 /* no javascript */)
      newUrl.server = href;

    return newUrl;
  }

  size_t pos = 0;
  bool host = false;
  pos = lowercase.rfind("://", len);

  if (pos == string::npos)
    pos = 0;
  else {
    host = true;

    if (lowercase[pos - 1] == 's')
      newUrl.http_prefix = "https://";

    pos += 3;
  }

  if (host) {
    size_t endhost = lowercase.find_first_of("/ ", pos);

    if (endhost == string::npos)
      endhost = len;

    string hostname = href.substr(pos, endhost - pos);
    newUrl.server = hostname;
    newUrl.setUrl(href.substr(endhost));
  } else {
    size_t querystart = lowercase.find_first_of("#", pos); /* previously ?#  instead of # */

    if (querystart != string::npos)
      len = querystart;

    string theUrl = href.substr(pos, len - pos);

    if (theUrl.empty())
      return newUrl;

    // Manage relative urls
    if (theUrl[0] != '/') {
      string urlreference(this->url);
      size_t findUp = urlreference.rfind("/", urlreference.length());

      if (findUp == string::npos) {
        urlreference.clear();
        urlreference.append(1, '/');
      } else
        urlreference = urlreference.substr(0, findUp + 1);

      size_t pos;

      // remove space chars at the beginning
      for (pos = 0; pos < theUrl.size(); ++pos) {
        char c = theUrl[pos];

        if (c != ' ' && c != '\t')
          break;
      }

      if (pos > 0)
        theUrl = theUrl.substr(pos);

      while ((pos = theUrl.find("./")) != string::npos) {
        if (pos == 0) {
          theUrl = theUrl.substr(2);
          continue;
        }

        if (theUrl[pos - 1] == '.') {
          theUrl = theUrl.substr(3);
          findUp = urlreference.rfind('/', findUp - 1);

          if (findUp == string::npos) {
            tlp::warning() << "bad url reference, to much ../" << endl;
            return newUrl;
          }

          urlreference = urlreference.substr(0, findUp + 1);
        } else {
          tlp::warning() << "bad url reference, to much ../" << endl;
          return newUrl;
        }
      }

      theUrl = urlreference + theUrl;
    }

    if (theUrl != "/") {
      newUrl.setUrl(theUrl);
      newUrl.server = this->server;
    }
  }

  return newUrl;
}

static const char *paramHelp[] = {
    // server
    "This parameter defines the web server that you want to inspect. No need for http:// at the "
    "beginning; http protocol is always assumed. No need for / at the end.",

    // initial page
    "This parameter defines the first web page to visit. No need for / at the beginning.",

    // max number of links
    "This parameter defines the maximum number of nodes (different pages) allowed in the extracted "
    "graph.",

    // non http links
    "This parameter indicates if non http links (https, ftp, mailto...) have to be extracted.",

    // other server
    "This parameter indicates if links or redirection to other server pages have to be followed.",

    // compute layout
    "This parameter indicates if a layout of the extracted graph has to be computed.",

    // page color
    "This parameter indicates the color used to display nodes.",

    // link color
    "This parameter indicates the color used to display links.",

    // link color
    "This parameter indicates the color used to display redirections."};

struct WebImport : public ImportModule {
  PLUGININFORMATION("Web Site", "Auber", "15/11/2004",
                    "Imports a new graph from Web site structure (one node per page).", "1.1",
                    "Misc")

  std::deque<UrlElement> toVisit;
  std::set<UrlElement> visited;
  std::map<UrlElement, tlp::node> nodes;
  tlp::StringProperty *labels;
  tlp::StringProperty *urls;
  tlp::ColorProperty *colors;
  tlp::Color *redirectionColor;
  unsigned int maxSize;
  bool visitOther;
  bool extractNonHttp;

  WebImport(tlp::PluginContext *context)
      : ImportModule(context), labels(nullptr), urls(nullptr), colors(nullptr),
        redirectionColor(nullptr), maxSize(1000), visitOther(false), extractNonHttp(true) {
    addInParameter<string>("server", paramHelp[0], "www.labri.fr");
    addInParameter<string>("web page", paramHelp[1], "");
    addInParameter<int>("max size", paramHelp[2], "1000");
    addInParameter<bool>("non http links", paramHelp[3], "false");
    addInParameter<bool>("other server", paramHelp[4], "false");
    addInParameter<bool>("compute layout", paramHelp[5], "true");
    addInParameter<Color>("page color", paramHelp[6], "(240, 0, 120, 128)");
    addInParameter<Color>("link color", paramHelp[7], "(96,96,191,128)");
    addInParameter<Color>("redirection color", paramHelp[8], "(191,175,96,128)");
    addDependency("FM^3 (OGDF)", "1.2");
  }

  string urlDecode(const string &url) {
    string buffer = "";
    int len = url.length();

    for (int i = 0; i < len; ++i) {
      char ch = url.at(i);

      if (ch == '%') {
        int chnum = 0;
        ch = url.at(++i);
        chnum = ch - '0';

        if (chnum > 9) {
          if (ch >= 'A')
            chnum = 10 + (ch - 'A');
          else
            chnum = 10 + (ch - 'a');
        }

        chnum *= 16;
        ch = url.at(++i);

        if (ch - '0' > 9) {
          if (ch >= 'A')
            chnum += ch - 'A';
          else
            chnum += ch - 'a';
        } else
          chnum += ch - '0';

        buffer += chnum;
      } else {
        buffer += ch;
      }
    }

    return buffer;
  }

  //========================================================
  bool addNode(const UrlElement &url, node &n) {
    if (nodes.find(url) == nodes.end()) {
      // no more added node after maxSize
      if (graph->numberOfNodes() == maxSize) {
        n = node();
        return false;
      }

      n = graph->addNode();
      stringstream str;
      str << url.server;

      if (url.url[0] != '/')
        str << "/";

      str << url.getUrl();
      labels->setNodeValue(n, urlDecode(str.str()));
      ostringstream oss;

      if (url.is_http()) {
        oss << url.http_prefix.c_str();
      }

      oss << str.str();
      urls->setNodeValue(n, oss.str());
      nodes[url] = n;
      return true;
    }

    n = nodes[url];
    return false;
  }

  //========================================================
  bool addEdge(const UrlElement &source, const UrlElement &target, const char *type,
               const Color *color) {
    node sNode, tNode;
    bool sAdded = addNode(source, sNode);
    bool tAdded = addNode(target, tNode);

    // if new nodes can not be added (numberOfNodes() == maxSize)
    // stop adding edge
    if (!sNode.isValid() || !tNode.isValid())
      return false;

    if (sAdded || tAdded || ((sNode != tNode) && !graph->existEdge(sNode, tNode).isValid())) {
      edge e = graph->addEdge(sNode, tNode);

      if (type)
        labels->setEdgeValue(e, type);

      if (color)
        colors->setEdgeValue(e, *color);
    }

    return true;
  }
  //========================================================
  bool nextUrl(UrlElement &url) {
    // tlp::warning() << __PRETTY_FUNCTION__ << endl;
    while (!toVisit.empty()) {
      url = toVisit.front();
      toVisit.pop_front();

      if (visited.find(url) == visited.end()) {
        visited.insert(url);
        return true;
      }
    }

    return false;
  }
  //========================================================
  void findAndTreatUrls(const string &lowercase, const string &balise, UrlElement &url) {
    size_t llen = lowercase.length();
    size_t len = llen;

    while (len != string::npos) {
      len = lowercase.rfind(balise, len);

      if (len != string::npos) {
        bool urlFound = true;
        // find the next '=' then the first '"'
        size_t start = len + balise.length();
        len--;
        char c = '=';

        for (; start < llen; start++) {
          if (lowercase[start] == c || ((c == '"') && lowercase[start] == '\'')) {
            // found =
            if (c == '=') {
              // now looking for "
              c = '"';
              continue;
            } else {
              c = lowercase[start];
              break;
            }
          }

          // only space chars allowed between balise and '='
          // and between '=' and the first '"' too
          if (lowercase[start] != ' ') {
            urlFound = false;
            break;
          }
        }

        if (urlFound) {
          ++start;
          size_t end = start;

          // find the end of the string
          for (; end < llen; end++) {
            if (lowercase[end] == c)
              break;
          }

          if (end != start) {
            string newurl = url.data.substr(start, end - start);
            parseUrl(newurl, url);
          }
        }
      }
    }
  }
  //========================================================
  // parsehtml - parse the given html and add to queue
  void parseHtml(UrlElement &url) {
    // tlp::warning() << __PRETTY_FUNCTION__ << endl << flush;
    if (url.data.empty())
      return;

    string lowercase(url.data);

    for (size_t i = 0; i < lowercase.length(); ++i)
      lowercase[i] = tolower(lowercase[i]);

    findAndTreatUrls(lowercase, " href", url);
    findAndTreatUrls(lowercase, " src", url);
  }
  //========================================================
  void addUrl(const UrlElement &url, bool _toVisit) {
    if (visited.find(url) != visited.end())
      return;

    if (_toVisit && url.is_http())
      toVisit.push_back(url);
  }
  //========================================================
  void parseUrl(const string &href, UrlElement &starturl) {
    UrlElement newUrl = starturl.parseUrl(href);

    if (newUrl.isValid() && (extractNonHttp || newUrl.is_http()) &&
        addEdge(starturl, newUrl, nullptr, nullptr))
      addUrl(newUrl, visitOther || (newUrl.server == starturl.server));
  }
  //========================================================
  bool start() {
    UrlElement url;
    unsigned step = 20;

    while (nextUrl(url)) {
      if (url.isHtmlPage()) {
        unsigned int nbNodes = graph->numberOfNodes();

        if (pluginProgress && ((nbNodes % step) == 0)) {
          pluginProgress->setComment(string("Visiting ") + urlDecode(url.server + url.url));

          if (pluginProgress->progress(nbNodes, maxSize) != TLP_CONTINUE)
            return pluginProgress->state() != TLP_CANCEL;
        }

#ifndef NDEBUG
        tlp::warning() << "Visiting: " << url.server << url.url << " ..." << std::endl << flush;
#endif

        if (url.isRedirected()) {
          UrlElement redirection = url.getRedirection();

          if (redirection.isValid()) {
#ifndef NDEBUG
            tlp::warning() << endl
                           << "redirected to " << redirection.server << redirection.url << endl;
#endif

            if (addEdge(url, redirection, "redirection", redirectionColor))
              addUrl(redirection, visitOther || redirection.server == url.server);
          } else
            tlp::warning() << endl << "invalid redirection" << endl;
        } else {
          url.load();
          parseHtml(url);
          url.clear();
#ifndef NDEBUG
          tlp::warning() << " done" << endl << flush;
#endif
        }
      }

#ifndef NDEBUG
      else
        tlp::warning() << "Omitting : " << url.server << url.url << " ==> [not html]" << endl;

#endif
    }

    return true;
  }

  bool importGraph() override {
    string server = "www.labri.fr";
    string url;
    bool computelayout = true;
    Color pColor(255, 0, 0);
    Color lColor(0, 0, 255, 128);
    Color rColor(255, 255, 0, 128);
    maxSize = 1000;
    visitOther = false;
    extractNonHttp = true;

    if (dataSet != nullptr) {
      dataSet->get("server", server);
      dataSet->get("web page", url);
      dataSet->get("max size", maxSize);
      dataSet->get("non http links", extractNonHttp);
      dataSet->get("other server", visitOther);
      dataSet->get("compute layout", computelayout);
      dataSet->get("page color", pColor);
      dataSet->get("link color", lColor);
      dataSet->get("redirection color", rColor);
    }

    UrlElement mySite;
    size_t pos = server.find("http://");

    if (pos == 0)
      // remove http:// prefix
      server = server.substr(7);

    // remove / prefix
    if (server[0] == '/')
      server = server.substr(1);

    if (server[server.size() - 1] == '/')
      // remove / suffix
      server = server.substr(0, server.size() - 1);

    mySite.server = server;

    if (url[0] == '/')
      // remove / prefix
      url = url.substr(1);

    mySite.setUrl(string("/") + url);
    mySite.data = "";

    labels = graph->getProperty<StringProperty>("viewLabel");
    labels->setAllEdgeValue(string("link"));
    urls = graph->getProperty<StringProperty>("url");
    colors = graph->getProperty<ColorProperty>("viewColor");
    colors->setAllNodeValue(pColor);
    colors->setAllEdgeValue(lColor);
    redirectionColor = &rColor;

    graph->getProperty<IntegerProperty>("viewShape")
        ->setAllNodeValue(14); // GlyphManager::glyphId("2D - Circle")

    if (!mySite.load()) {
      if (pluginProgress) {
        stringstream sstr;
        sstr << "Unable to access http://" << mySite.server << mySite.url << " (ERROR "
             << mySite.getCode() << ')';
        pluginProgress->setError(sstr.str());
      }

      return false;
    }

    node n;
    toVisit.push_back(mySite);
    addNode(mySite, n);

    if (pluginProgress) {
      pluginProgress->showPreview(false);
      pluginProgress->setComment(std::string("Visiting ") + mySite.server + mySite.url);
    }

    if (!start())
      return false;

    if (computelayout) {
      pluginProgress->setComment("Layouting extracted graph using FM³...");
      string errMsg;
      // apply FM³
      LayoutProperty *layout = graph->getProperty<LayoutProperty>("viewLayout");
      return graph->applyPropertyAlgorithm("FM^3 (OGDF)", layout, errMsg, nullptr, pluginProgress);
    }

    return true;
  }
};

PLUGIN(WebImport)
