/****************************************************************************************
 * Copyright (c) 2007 Maximilian Kossick <maximilian.kossick@googlemail.com>            *
 * Copyright (c) 2008 Seb Ruiz <ruiz@kde.org>                                           *
 *                                                                                      *
 * This program is free software; you can redistribute it and/or modify it under        *
 * the terms of the GNU General Public License as published by the Free Software        *
 * Foundation; either version 2 of the License, or (at your option) any later           *
 * version.                                                                             *
 *                                                                                      *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY      *
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A      *
 * PARTICULAR PURPOSE. See the GNU General Public License for more details.             *
 *                                                                                      *
 * You should have received a copy of the GNU General Public License along with         *
 * this program.  If not, see <http://www.gnu.org/licenses/>.                           *
 ****************************************************************************************/

#define DEBUG_PREFIX "ScanResultProcessor"

#include "ScanResultProcessor.h"

#include "ArtistHelper.h"
#include "core/support/Debug.h"
#include "core/meta/support/MetaConstants.h"
#include "core/meta/support/MetaUtility.h"

#include <QDir>
#include <QFileInfo>
#include <QListIterator>

ScanResultProcessor::ScanResultProcessor( Collections::SqlCollection *collection )
/** The ScanResulProcessor class takes the results from the ScanManager and puts them into the database.
 *  The Processsor is doing this in several steps.
 *  First it copies the complete database into temporary tables.
 *  Then it reads all the information into internal data structures deleting the temporary tables.
 *  Then it takes the results from the ScanManager and updates the structures.
 *  Afterwards the copying is done in reverse.
 */
    : m_collection( collection )
    , m_storage( 0 )
    , m_setupComplete( false )
    , m_type( FullScan )
    , m_aftPermanentTablesUrlString()
{
    m_aftPermanentTablesUrlString << "playlist_tracks";
}

ScanResultProcessor::~ScanResultProcessor()
{
    //everything has a URL, so enough to just delete from here

    // TODO: Check if protection against double deletes is needed after all. (Ralf)

    QSet<QString*> deletedUrls; //prevent double deletes
    foreach( QString *slist, m_urlsHashByUid )
    {
        if( slist )
        {
            if( !deletedUrls.contains( slist ) )
            {
                delete[] slist;
                deletedUrls.insert( slist );
            }
        }
        else
            debug() << "GAAH! Tried to double-delete a value in m_urlsHashByUid";
    }

    QSet<QString*> deletedAlbums; //prevent double deletes
    foreach( QLinkedList<QString*> *list, m_albumsHashByName )
    {
        if( list )
        {
            foreach( QString *slist, *list )
            {
                if( slist )
                {
                    if( !deletedAlbums.contains( slist ) )
                    {
                        delete[] slist;
                        deletedAlbums.insert( slist );
                    }
                    else
                        debug() << "GAAH! Tried to double-delete a value in m_albumsHashByName";
                }
            }
            delete list;
        }
    }

    QSet<QString*> deletedTracks; //prevent double deletes
    foreach( QLinkedList<QString*> *list, m_tracksHashByAlbum )
    {
        if( list )
        {
            foreach( QString *slist, *list )
            {
                if( slist )
                {
                    if( !deletedTracks.contains( slist ) )
                    {
                        delete[] slist;
                        deletedTracks.insert( slist );
                    }
                    else
                        debug() << "GAAH! Tried to double-delete a value in m_tracksHashByAlbum";
                }
            }
            delete list;
        }
    }
}

void
ScanResultProcessor::setScanType( ScanType type )
{
    m_type = type;
}

void
ScanResultProcessor::addDirectory( const QString &dir, uint mtime )
{
    //DEBUG_BLOCK
    //debug() << "addDirectory on " << dir << " with mtime " << mtime;
    if( dir.isEmpty() )
    {
        //debug() << "got directory with no path from the scanner, not adding";
        return;
    }
    setupDatabase();
    int deviceId = m_collection->mountPointManager()->getIdForUrl( dir );
    QString rdir = m_collection->mountPointManager()->getRelativePath( deviceId, dir );
    QString query = QString( "SELECT         id, changedate               "
                             "FROM           directories_temp             "
                             "WHERE          deviceid = %1 AND dir = '%2';" )
                        .arg( QString::number( deviceId ), m_storage->escape( rdir ) );
    QStringList res = m_storage->query( query );
    if( res.isEmpty() )
    {
        QString insert = QString( "INSERT INTO directories_temp(deviceid,changedate,dir) VALUES (%1,%2,'%3');" )
                        .arg( QString::number( deviceId ), QString::number( mtime ),
                                m_storage->escape( rdir ) );
        int id = m_storage->insert( insert, "directories_temp" );
        m_directories.insert( dir, id );
    }
    else
    {
        if( res[1].toUInt() != mtime )
        {
            QString update = QString( "UPDATE directories_temp SET changedate = %1 WHERE id = %2;" )
                                .arg( QString::number( mtime ), res[0] );
            m_storage->query( update );
        }
        m_directories.insert( dir, res[0].toInt() );
        m_collection->dbUpdater()->removeFilesInDirFromTemporaryTables( deviceId, rdir );
    }
}

void
ScanResultProcessor::addImage( const QString &path, const QList< QPair<QString, QString> > covers )
{
    m_imageMap[path] = covers;
}

void
ScanResultProcessor::doneWithImages()
{
    if( m_imageMap.isEmpty() )
        return;

    //now -- find the best candidate with heuristics, then throw the rest away
    const QString path = findBestImagePath( m_imageMap.keys() );
    if( path.isEmpty() )
        return;

    typedef QPair<QString, QString> StringPair;
    QList< StringPair > covers = m_imageMap[path];

    foreach( const StringPair &key, covers )
    {
        if( key.first.isEmpty() || key.second.isEmpty() )
            continue;

        int artist = genericId( &m_artists, key.first, &m_nextArtistNum );
        int album  = albumId( key.second, artist );

        // Will automatically add the image path to the database if needed
        imageId( path, album );
    }

    m_imageMap.clear();
}

QString
ScanResultProcessor::findBestImagePath( const QList<QString> &paths )
{
    //DEBUG_BLOCK
    //if there is only one Image, we already know who is the winner
    if(paths.size()<=1)
        return paths.first();

    int goodnessPriority = 3;
    QString goodPath;
    foreach( const QString &path, paths )
    {
        if( path.startsWith( "amarok-sqltrackuid://" ) )
            return path;

        QString file = QFileInfo( path ).completeBaseName();
        
        //prioritize "front"
        if( file.contains( "front", Qt::CaseInsensitive ) ||
                file.contains( i18nc( "Front cover of an album", "front" ), Qt::CaseInsensitive ) )
        {
            goodnessPriority = 0;
            goodPath = path;
        }

        //then: try "cover"
        if( file.contains( "cover", Qt::CaseInsensitive ) ||
                file.contains( i18nc( "(Front) Cover of an album", "cover" ), Qt::CaseInsensitive ) )
        {
            if( goodnessPriority > 1 )
            {
                goodnessPriority = 1;
                goodPath = path;
            }
        }

        //next: try "folder" (some applications apparently use this)
        //using compare and not contains to not hit "Folder-Back" or something.
        if( file.compare( "folder", Qt::CaseInsensitive ) == 0)
        {
            if( goodnessPriority > 2 )
            {
                goodnessPriority = 2;
                goodPath = path;
            }
        }
    }
    if( !goodPath.isEmpty() )
        return goodPath;

    //finally: pick largest image -- often a high-quality blowup of the front
    //so that people can print it out
    qint64 size = 0;
    QString current;
    foreach( const QString &path, paths )
    {
        QFileInfo info( path );
        if( info.size() > size )
        {
            size = info.size();
            current = path;
        }
    }
    return current;
}

void
ScanResultProcessor::commit()
{
    if( !m_setupComplete )
    {
        debug() << "Database temporary table setup did not complete due to no directories needing to be processed.";
        return;
    }
    if( m_type == ScanResultProcessor::IncrementalScan )
    {
        SqlMountPointManager *manager = m_collection->mountPointManager();
        const QStringList dirs        = m_directories.keys();
        foreach( const QString &dir, dirs )
        {
            int deviceid = manager->getIdForUrl( dir );
            const QString rpath = manager->getRelativePath( deviceid, dir );
            m_collection->dbUpdater()->removeFilesInDir( deviceid, rpath );
        }
    }
    else
    {
        m_collection->dbUpdater()->cleanPermanentTables();
    }

    copyHashesToTempTables();

    debug() << "temp_tracks: " << m_storage->query("select count(*) from tracks_temp");
    debug() << "tracks before commit: " << m_storage->query("select count(*) from tracks");
    m_collection->dbUpdater()->copyToPermanentTables();
    debug() << "tracks after commit: " << m_storage->query("select count(*) from tracks");
    m_collection->dbUpdater()->removeTemporaryTables();

    m_collection->dbUpdater()->deleteAllRedundant( "album" );
    m_collection->dbUpdater()->deleteAllRedundant( "artist" );
    m_collection->dbUpdater()->deleteAllRedundant( "genre" );
    m_collection->dbUpdater()->deleteAllRedundant( "composer" );
    m_collection->dbUpdater()->deleteAllRedundant( "year" );


    updateAftPermanentTablesUrlString();
    updateAftPermanentTablesUidString();

    connect( this, SIGNAL( changedTrackUrlsUids( const ChangedTrackUrls &, const TrackUrls & ) ),
             m_collection, SLOT( updateTrackUrlsUids( const ChangedTrackUrls &, const TrackUrls & ) ) );

    emit changedTrackUrlsUids( m_changedUrls, m_changedUids );

    debug() << "Sending changed signal";
    m_collection->sendChangedSignal();
}

void
ScanResultProcessor::rollback()
{
    m_collection->dbUpdater()->removeTemporaryTables();
}

void
ScanResultProcessor::processDirectory( const QList<QVariantMap > &data )
{
    //DEBUG_BLOCK
    setupDatabase();
    //using the following heuristics:
    //if more than one album is in the dir, use the artist of each track as albumartist
    //if all tracks have the same artist, use it as albumartist
    //try to find the albumartist A: tracks must have the artist A or A feat. B (and variants)
    //if no albumartist could be found, it's a compilation

    QSet<QString> artists;
    QString album;
    bool multipleAlbums = false;
    if( !data.isEmpty() )
        album = data[0].value( Meta::Field::ALBUM ).toString();

    foreach( const QVariantMap &row, data )
    {
        artists.insert( row.value( Meta::Field::ARTIST ).toString() );
        if( row.value( Meta::Field::ALBUM ).toString() != album )
            multipleAlbums = true;
    }

    if( multipleAlbums || album.isEmpty() || artists.size() == 1 )
    {
        foreach( const QVariantMap &row, data )
        {
            QString uid = row.value( Meta::Field::UNIQUEID ).toString();
            if( m_uidsSeenThisScan.contains( uid ) )
            {
                QString originalLocation = ( ( m_urlsHashByUid.contains( uid ) &&
                                             m_urlsHashByUid[uid] != 0 ) ?
                                             m_collection->mountPointManager()->getAbsolutePath( m_urlsHashByUid[uid][UrlColDevice].toInt(), m_urlsHashByUid[uid][UrlColRPath]) : "(unknown)" );
                debug() << "Skipping file with uniqueid " << uid << " as it was already seen this scan," <<
                           "file is at " << row.value( Meta::Field::URL ).toString() << ", original file is at " << originalLocation;
            }
            else
            {
                int artist = genericId( &m_artists, row.value( Meta::Field::ARTIST ).toString(), &m_nextArtistNum );
                //debug() << "artist found = " << artist;
                addTrack( row, artist );
                m_uidsSeenThisScan.insert( uid );
            }
        }
    }
    else
    {
        QString albumArtist = findAlbumArtist( artists, data.count() );
        //debug() << "albumArtist found = " << albumArtist;
        //an empty string means that no albumartist was found
        int artist = albumArtist.isEmpty() ? 0 : genericId( &m_artists, albumArtist, &m_nextArtistNum );
        //debug() << "artist found = " << artist;

        //debug() << "albumartist " << albumArtist << "for artists" << artists;
        foreach( const QVariantMap &row, data )
        {
            QString uid = row.value( Meta::Field::UNIQUEID ).toString();
            if( m_uidsSeenThisScan.contains( uid ) )
            {
                QString originalLocation = ( ( m_urlsHashByUid.contains( uid ) &&
                                             m_urlsHashByUid[uid] != 0 ) ?
                                             m_collection->mountPointManager()->getAbsolutePath( m_urlsHashByUid[uid][UrlColDevice].toInt(), m_urlsHashByUid[uid][UrlColRPath] ) : "(unknown)" );
                debug() << "Skipping file with uniqueid " << uid << " as it was already seen this scan," <<
                           "file is at " << row.value( Meta::Field::URL ).toString() << ", original file is at " << originalLocation;
            }
            else
            {
                addTrack( row, artist );
                m_uidsSeenThisScan.insert( uid );
            }
        }
    }
}

QString
ScanResultProcessor::findAlbumArtist( const QSet<QString> &artists, int trackCount ) const
{
    //DEBUG_BLOCK
    QMap<QString, int> artistCount;
    foreach( const QString &artist, artists )
    {
        QString actualArtist = ArtistHelper::realTrackArtist( artist );
        if( actualArtist.isEmpty() )
        {
            //TODO error handling
        }
        else
        {
            if( artistCount.contains( actualArtist ) )
                artistCount.insert( actualArtist, artistCount.value( actualArtist ) + 1 );
            else
                artistCount.insert( actualArtist, 1 );
        }
    }
    QString albumArtist;
    int count = 0;
    foreach( int value, artistCount )
    {
        if( value > count )
        {
            albumArtist = artistCount.key( value );
            count = value;
        }
    }
    //if an artist is the primary artist of each track in the directory, assume the artist is the albumartist
    return count == trackCount ? albumArtist : QString();
}

void
ScanResultProcessor::addTrack( const QVariantMap &trackData, int albumArtistId )
{
    //DEBUG_BLOCK
    //debug() << "albumArtistId = " << albumArtistId;
    //amarok 1 stored all tracks of a compilation in different directories.
    //when using its "Organize Collection" feature
    //try to detect these cases
    QString albumName = trackData.value( Meta::Field::ALBUM ).toString();
    int album = 0;

    QString path = trackData.value( Meta::Field::URL ).toString();

    QFileInfo file( path );

    QDir dir = file.dir();

    //do not check existing albums if there is more than one file in the directory
    //see comments in checkExistingAlbums

    //TODO: find a better way to ignore non-audio files than the extension matching below
    if( !m_filesInDirs.contains( dir.absolutePath() ) )
    {
        dir.setFilter( QDir::Files | QDir::Readable | QDir::CaseSensitive );
        QStringList filters;
        filters << "*.[mM][pP]3" << "*.[oO][gG][gG]" << "*.[oO][gG][aA]" << "*.[fF][lL][aA][cC]" << "*.[wW][mM][aA]" << "*.[mM]4[aAbB]";
        dir.setNameFilters( filters );
        m_filesInDirs.insert( dir.absolutePath(), dir.count() );
    }

    if( m_filesInDirs.value( dir.absolutePath() ) == 1 )
    {
        album = checkExistingAlbums( albumName );
    }

    QString uid = trackData.value( Meta::Field::UNIQUEID ).toString();

    int artist = genericId( &m_artists, trackData.value( Meta::Field::ARTIST ).toString(), &m_nextArtistNum );
    int genre = genericId( &m_genres, trackData.value( Meta::Field::GENRE ).toString(), &m_nextGenreNum );
    int composer = genericId( &m_composers, trackData.value( Meta::Field::COMPOSER ).toString(), &m_nextComposerNum );
    int year = genericId( &m_years, trackData.value( Meta::Field::YEAR ).toString(), &m_nextYearNum );

    if( !album ) //no compilation
    {
        album = albumId( albumName, albumArtistId );
        //debug() << "album set to " << album;
    }

    const int created  = file.created().toTime_t();
    const int modified = file.lastModified().toTime_t();

    //urlId will take care of the urls table part of AFT
    int url = urlId( path, uid );
/*
    foreach( QString key, m_urlsHashByUid.keys() )
    debug() << "Key: " << key << ", list: " << *m_urlsHashByUid[key];
    foreach( int key, m_urlsHashById.keys() )
    debug() << "Key: " << key << ", list: " << *m_urlsHashById[key];
    typedef QPair<int, QString> blahType; //QFOREACH is stupid when it comes to QPairs
    foreach( blahType key, m_urlsHashByLocation.keys() )
    debug() << "Key: " << key << ", list: " << *m_urlsHashByLocation[key];
*/
    QString *trackList = new QString[TrackColMaxCount];

    int id = m_nextTrackNum;
    // debug() << "Appending new track number with tracknum: " << id;
    trackList[TrackColId]          = QString::number( m_nextTrackNum++ );
    trackList[TrackColUrl]         = QString::number( url );
    trackList[TrackColArtist]      = QString::number( artist );
    trackList[TrackColAlbum]       = QString::number( album );
    trackList[TrackColGenre]       = QString::number( genre );
    trackList[TrackColComposer]    = QString::number( composer );
    trackList[TrackColYear]        = QString::number( year );
    trackList[TrackColTitle]       = trackData[ Meta::Field::TITLE ].toString();
    trackList[TrackColComment]     = trackData[ Meta::Field::COMMENT ].toString();
    trackList[TrackColTrackNumber] = trackData[ Meta::Field::TRACKNUMBER ].toString();
    trackList[TrackColDiscNumber]  = trackData[ Meta::Field::DISCNUMBER ].toString();
    trackList[TrackColBitRate]     = trackData[ Meta::Field::BITRATE ].toString();
    trackList[TrackColLength]      = trackData[ Meta::Field::LENGTH ].toString();
    trackList[TrackColSampleRate]  = trackData[ Meta::Field::SAMPLERATE ].toString();
    trackList[TrackColFileSize]    = trackData[ Meta::Field::FILESIZE ].toString();
    trackList[TrackColFileType]    = trackData[ Meta::Field::CODEC ].toString();
    if( trackData.contains( Meta::Field::BPM ) )
        trackList[TrackColBpm] = QString::number( trackData[ Meta::Field::BPM ].toDouble() ).replace( ',' , '.' );
    trackList[TrackColCreated]     = QString::number( created );
    trackList[TrackColModified]    = QString::number( modified );

    if( trackData.contains( Meta::Field::ALBUMGAIN ) && trackData.contains( Meta::Field::ALBUMPEAKGAIN ) )
    {
        trackList[TrackColAlbumGain] = trackData[ Meta::Field::ALBUMGAIN ].toString();
        trackList[TrackColAlbumPeakGain] = trackData[ Meta::Field::ALBUMPEAKGAIN ].toString();
    }
    if( trackData.contains( Meta::Field::TRACKGAIN ) && trackData.contains( Meta::Field::TRACKPEAKGAIN ) )
    {
        trackList[TrackColTrackGain] = trackData[ Meta::Field::TRACKGAIN ].toString();
        trackList[TrackColTrackPeakGain] = trackData[ Meta::Field::TRACKPEAKGAIN ].toString();
    }

    //insert into hashes
    if( m_tracksHashByUrl.contains( url ) && m_tracksHashByUrl[url] != 0 )
    {
        //debug() << "m_tracksHashByUrl already contains url " << url;
        //need to replace, not overwrite/add a new one
        QString *oldValues = m_tracksHashByUrl[url];
        for( int i = 1; i < TrackColMaxCount; i++ ) //not 0 because we want to keep old ID
            oldValues[i] = trackList[i];
        delete[] trackList;
        trackList = oldValues;
        id = oldValues[TrackColId].toInt();
        m_nextTrackNum--;
    }
    else
    {
        m_tracksHashByUrl.insert( url, trackList );
        m_tracksHashById.insert( id, trackList );
    }

    //debug() << "album = " << album;

    if( m_tracksHashByAlbum.contains( album ) && m_tracksHashByAlbum[album] != 0 )
    {
        //contains isn't the fastest on linked lists, but in reality this is on the order of maybe
        //ten quick pointer comparisons per track on average...probably lower
        //debug() << "trackList is " << trackList;
        if( !m_tracksHashByAlbum[album]->contains( trackList ) )
        {
            //debug() << "appending trackList to m_tracksHashByAlbum";
            m_tracksHashByAlbum[album]->append( trackList );
        }
        else
        {
            //debug() << "not appending trackList to m_tracksHashByAlbum";
        }

    }
    else
    {
        QLinkedList<QString*> *list = new QLinkedList<QString*>();
        list->append( trackList );
        m_tracksHashByAlbum[album] = list;
    }
}

int
ScanResultProcessor::genericId( QHash<QString, int> *hash, const QString &value, int *currNum )
{
    //DEBUG_BLOCK
    if( hash->contains( value ) )
        return hash->value( value );
    else
    {
        int id = *currNum;
        hash->insert( value, (*currNum)++ );
        return id;
    }
}

int
ScanResultProcessor::imageId( const QString &image, int albumId )
{
    // assume the album is valid
    if( albumId < 0 )
        return -1;

    QPair<QString, int> key( image, albumId );
    if( m_images.contains( key ) )
        return m_images.value( key );

    int imageId = -1;
    if( m_imagesFlat.contains( image ) )
        imageId = m_imagesFlat[image];
    else
    {
        imageId = m_nextImageNum;
        m_imagesFlat[image] = m_nextImageNum++;
    }

    if( imageId >= 0 )
    {
        if( m_albumsHashById.contains( albumId ) && m_albumsHashById[albumId] != 0 )
        {
            QString *list = m_albumsHashById[albumId];
            list[AlbumColMaxImage] = QString::number( imageId );
        }
        m_images.insert( key, imageId );
    }

    return imageId;
}

/**
 *  Returns the id for an album with the matching album name and albumArtistId.
 *  It also increments the MaxImage count of the album.
 */
int
ScanResultProcessor::albumId( const QString &album, int albumArtistId )
{
    //DEBUG_BLOCK
    //debug() << "Looking up album " << album;
    //albumArtistId == 0 means no albumartist
    QPair<QString, int> key( album, albumArtistId );
    if( m_albums.contains( key ) )
    {
        //debug() << "m_albums contains album/albumArtistId key";
        // if we already have the key but the artist == 0,
        // UPDATE the image field so that we won't forget the cover for a compilation
        int id = m_albums.value( key );
        if ( albumArtistId == 0 )
        {
            if( m_albumsHashByName.contains( album ) && m_albumsHashByName[album] != 0 )
            {
                QString *slist;
                int maxImage = 0;
                QLinkedList<QString*> *llist = m_albumsHashByName[album];
                foreach( QString* list, *llist )
                {
                    if( !(list[AlbumColMaxImage].isEmpty()) && list[AlbumColMaxImage].toInt() > maxImage )
                    {
                        slist = list;
                        maxImage = list[AlbumColMaxImage].toInt();
                    }
                }
                if( maxImage > 0 )
                {
                    if( m_albumsHashById.contains( id ) && m_albumsHashById[id] != 0 )
                    {
                        QString *list = m_albumsHashById[id];
                        list[AlbumColMaxImage] = QString::number( maxImage );
                    }
                }
            }
        }
        return id;
    }

    int id = 0;
    if( m_albumsHashByName.contains( album ) && m_albumsHashByName[album] != 0 )
    {
        //debug() << "Hashes contain it";
        QLinkedList<QString*> *llist = m_albumsHashByName[album];
        foreach( QString *slist, *llist )
        {
            //debug() << "albumArtistId = " << albumArtistId;
            //debug() << "Checking list: " << *slist;
            if( slist[AlbumColArtist].isEmpty() && albumArtistId == 0 )
            {
                //debug() << "artist is empty and albumArtistId = 0, returning " << slist->at( 0 );
                id = slist[AlbumColId].toInt();
                break;
            }
            else if( slist[AlbumColArtist].toInt() == albumArtistId )
            {
                //debug() << "artist == albumArtistId,  returning " << slist->at( 0 );
                id = slist[AlbumColId].toInt();
                break;
            }
        }
    }
    if( !id )
    {
        //debug() << "Not found! Inserting...";
        id = albumInsert( album, albumArtistId );
    }
    m_albums.insert( key, id );
    //debug() << "returning id = " << id;
    return id;
}

int
ScanResultProcessor::albumInsert( const QString &album, int albumArtistId )
{
    //DEBUG_BLOCK
    int returnedNum = m_nextAlbumNum++;
    QString* albumList = new QString[AlbumColMaxCount];
    albumList[AlbumColId]     = QString::number( returnedNum );
    albumList[AlbumColTitle]  = album;
    if (albumArtistId)
        albumList[AlbumColArtist] = QString::number( albumArtistId );

    m_albumsHashById[returnedNum] = albumList;
    if( m_albumsHashByName.contains( album ) && m_albumsHashByName[album] != 0 )
    {
        if( !m_albumsHashByName[album]->contains( albumList ) )
            m_albumsHashByName[album]->append( albumList );
    }
    else
    {
        QLinkedList<QString*> *list = new QLinkedList<QString*>();
        list->append( albumList );
        m_albumsHashByName[album] = list;
    }
    //debug() << "albumInsert returning " << returnedNum;
    return returnedNum;
}

int
ScanResultProcessor::urlId( const QString &url, const QString &uid )
{
/*
    DEBUG_BLOCK
    foreach( QString key, m_urlsHashByUid.keys() )
    debug() << "Key: " << key << ", list: " << *m_urlsHashByUid[key];
    foreach( int key, m_urlsHashById.keys() )
    debug() << "Key: " << key << ", list: " << *m_urlsHashById[key];
    typedef QPair<int, QString> blahType; //QFOREACH is stupid when it comes to QPairs
    foreach( blahType key, m_urlsHashByLocation.keys() )
    debug() << "Key: " << key << ", list: " << *m_urlsHashByLocation[key];
*/
    QFileInfo fileInfo( url );
    const QString dir = fileInfo.absoluteDir().absolutePath();
    int dirId = directoryId( dir );
    int deviceId = m_collection->mountPointManager()->getIdForUrl( url );
    QString rpath = m_collection->mountPointManager()->getRelativePath( deviceId, url );

    QPair<int, QString> locationPair( deviceId, rpath );
    //debug() << "in urlId with url = " << url << " and uid = " << uid;
    //debug() << "checking locationPair " << locationPair;
/*
    if( m_urlsHashByLocation.contains( locationPair ) )
    {
        QStringList values;
        if( m_urlsHashByLocation[locationPair] != 0 )
            values = *m_urlsHashByLocation[locationPair];
        else
            values << "zero";
        //debug() << "m_urlsHashByLocation contains it! It is " << values;
    }
*/
    const QString *currUrlIdValues = 0;
    if( m_urlsHashByUid.contains( uid ) && m_urlsHashByUid[uid] != 0 )
        currUrlIdValues = m_urlsHashByUid[uid];
    else if( m_urlsHashByLocation.contains( locationPair ) && m_urlsHashByLocation[locationPair] != 0 )
        currUrlIdValues = m_urlsHashByLocation[locationPair];

    if( !currUrlIdValues )  //fresh -- insert
    {
        //debug() << "locationPair did not match!";
        int returnedNum = m_nextUrlNum++;
        QString *list = new QString[UrlColMaxCount];
        list[UrlColId]     = QString::number( returnedNum );
        list[UrlColDevice] = QString::number( deviceId );
        list[UrlColRPath]  = rpath;
        list[UrlColDir]    = QString::number( dirId );
        list[UrlColUid]    = uid;

        m_urlsHashByUid[uid] = list;
        m_urlsHashById[returnedNum] = list;
        m_urlsHashByLocation[locationPair] = list;
        return returnedNum;
    }

    if( currUrlIdValues[UrlColId]    == QString::number( deviceId ) &&
        currUrlIdValues[UrlColRPath] == rpath &&
        currUrlIdValues[UrlColDir]   == QString::number( dirId ) &&
        currUrlIdValues[UrlColUid]   == uid
      )
    {
        //everything matches, don't need to do anything, just return the ID
        //debug() << "Everything matches, just returning id";
        return currUrlIdValues[UrlColId].toInt();
    }

    if( currUrlIdValues[UrlColUid] == uid )
    {
        //we found an existing entry with this uniqueid, update the deviceid and path
        //Note that we ignore the situation where both a UID and path was found; UID takes precedence
        //debug() << "found entry with this UID";
        if( m_urlsHashByUid.contains( uid ) && m_urlsHashByUid[uid] != 0 )
        {
            //debug() << "m_urlsHashByUid contains this UID, updating deviceId and path";
            QString *list = m_urlsHashByUid[uid];
            //debug() << "list from UID hash is " << list << " with values " << *list;
            QPair<int, QString> oldLocationPair( list[UrlColDevice].toInt(), list[UrlColRPath] );
            list[UrlColDevice] = QString::number( deviceId );
            list[UrlColRPath]  = rpath;
            list[UrlColDir]    = QString::number( dirId );
            //debug() << "Hash updated UID-based values for uid " << uid;
            //Now remove original locations if they exist
            if( m_urlsHashByLocation.contains( locationPair )
                && m_urlsHashByLocation[locationPair] != 0
                && m_urlsHashByLocation[locationPair] != list )
            {
                //debug() << "If condition checked out; removing old stuff";
                //Have existing entries for both location and UID, so to prevent conflicts remove the old
                //entry. This can happen if for instance a track with a changed UID is added in
                //two places to the collection.
                QString *oldList = m_urlsHashByLocation[locationPair];
                //debug() << "old list is " << oldList << " with contents " << *oldList;
                m_urlsHashById.remove( oldList[UrlColId].toInt() );
                m_urlsHashByUid.remove( oldList[UrlColUid] );
                delete[] oldList;
            }
            m_urlsHashByLocation[locationPair] = list;
            m_urlsHashByLocation.remove( oldLocationPair );
        }
        m_permanentTablesUrlUpdates.insert( uid, url );
        m_changedUrls.insert( uid, QPair<QString, QString>( m_collection->mountPointManager()->getAbsolutePath( currUrlIdValues[UrlColDevice].toInt(), currUrlIdValues[UrlColRPath] ), url ) );
        return currUrlIdValues[UrlColId].toInt();
    }

    if( currUrlIdValues[UrlColDevice] == QString::number( deviceId ) &&
        currUrlIdValues[UrlColRPath] == rpath )
    {
        //We found an existing path; give it the most recent UID value
        //debug() << "In urlId, found deviceid " << QString::number( deviceId ) << " and rpath " << rpath;
        if( m_urlsHashByLocation.contains( locationPair ) && m_urlsHashByLocation[locationPair] != 0 )
        {
            QString *list = m_urlsHashByLocation[locationPair];
            //debug() << "Replacing hash " << list->at( 4 ) << " with " << uid;
            QString oldId = list[UrlColUid];
            list[UrlColUid] = uid;

            // -- delete the old list with the new uid
            if( m_urlsHashByUid.contains( uid )
                && m_urlsHashByUid[uid] != 0 
                && m_urlsHashByUid[uid] != list )
            {
                QString *oldList = m_urlsHashByUid[uid];
                m_urlsHashById.remove( oldList[UrlColId].toInt() );
                m_urlsHashByLocation.remove( QPair<int, QString>( oldList[UrlColDevice].toInt(), oldList[UrlColRPath] ) );
                delete[] oldList;
            }
            m_urlsHashByUid[uid] = list;
            m_urlsHashByUid.remove( oldId );
        }
        m_permanentTablesUidUpdates.insert( url, uid );
        m_changedUids.insert( currUrlIdValues[UrlColUid], uid );
        return currUrlIdValues[UrlColId].toInt();
    }

    debug() << "AFT algorithm died...you should not be here!  Returning something negative and bad.";
    return -666;
}

void
ScanResultProcessor::updateAftPermanentTablesUrlString()
{
    //DEBUG_BLOCK
    if( m_permanentTablesUrlUpdates.isEmpty() )
        return;

    QStringList res = m_storage->query( "SHOW VARIABLES LIKE 'max_allowed_packet';" );
    if( res.size() < 2 || res[1].toInt() == 0 )
    {
        debug() << "Uh oh! For some reason MySQL thinks there isn't a max allowed size!";
        return;
    }
    int maxSize = res[1].toInt() / 3; //for safety, due to multibyte encoding

    foreach( const QString &table, m_aftPermanentTablesUrlString )
    {
        QString queryStart = QString( "UPDATE %1 SET url = CASE uniqueid").arg( table );
        QString query = queryStart;
        QString query2;
        QString queryNext;
        QString query2Next;
        bool first = true;
        foreach( const QString &key, m_permanentTablesUrlUpdates.keys() )
        {
            queryNext = QString( " WHEN '%1' THEN '%2'" ).arg( m_storage->escape( key ),
                                                       m_storage->escape( m_permanentTablesUrlUpdates[key] ) );
            if( first )
                query2Next = QString( "'%1'" ).arg( m_storage->escape( key ) );
            else
                query2Next = QString( ", '%1'" ).arg( m_storage->escape( key ) );

            // + 20 is for the END WHERE section below
            if( query.length() + query2.length() + queryNext.length() + query2Next.length() + 25 >= maxSize )
            {
                query += QString( " END WHERE uniqueid IN(%1);" ).arg( query2 );
                m_storage->query( query );


                query = queryStart + queryNext;
                if( query2Next.startsWith( ", " ) )
                    query2 = query2Next.remove( 0, 2 ); //get rid of the ", "
                else
                    query2 = query2Next;
            }
            else
            {
                query += queryNext;
                query2 += query2Next;
            }

            first = false;
        }

        if( !query2.isEmpty() ) // will be empty if we already queried and added nothing new
        {
            query += QString( " END WHERE uniqueid IN(%1);" ).arg( query2 );
            m_storage->query( query );
        }
    }
}

void
ScanResultProcessor::updateAftPermanentTablesUidString()
{
    //DEBUG_BLOCK
    if( m_permanentTablesUidUpdates.isEmpty() )
        return;

    QStringList res = m_storage->query( "SHOW VARIABLES LIKE 'max_allowed_packet';" );
    if( res.size() < 2 || res[1].toInt() == 0 )
    {
        debug() << "Uh oh! For some reason MySQL thinks there isn't a max allowed size!";
        return;
    }
    int maxSize = res[1].toInt() / 3; //for safety, due to multibyte encoding

    foreach( const QString &table, m_aftPermanentTablesUrlString )
    {
        QString queryStart = QString( "UPDATE %1 SET uniqueid = CASE url").arg( table );
        QString query = queryStart;
        QString query2;
        QString queryNext;
        QString query2Next;
        bool first = true;
        foreach( const QString &key, m_permanentTablesUidUpdates.keys() )
        {
            queryNext = QString( " WHEN '%1' THEN '%2'" ).arg( m_storage->escape( key ),
                                                       m_storage->escape( m_permanentTablesUidUpdates[key] ) );
            if( first )
                query2Next = QString( "'%1'" ).arg( m_storage->escape( key ) );
            else
                query2Next = QString( ", '%1'" ).arg( m_storage->escape( key ) );

            // + 20 is for the END WHERE section below
            if( query.length() + query2.length() + queryNext.length() + query2Next.length() + 20 >= maxSize )
            {
                query += QString( " END WHERE url IN(%1);" ).arg( query2 );
                m_storage->query( query );

                query = queryStart + queryNext;
                if( query2Next.startsWith( ", " ) )
                    query2 = query2Next.remove( 0, 2 ); //get rid of the ", "
                else
                    query2 = query2Next;
            }
            else
            {
                query += queryNext;
                query2 += query2Next;
            }

            first = false;
        }

        if( !query2.isEmpty() ) // will be empty if we already queried and added nothing new
        {
            query += QString( " END WHERE url IN(%1);" ).arg( query2 );
            m_storage->query( query );
        }
    }
}

int
ScanResultProcessor::directoryId( const QString &dir )
{
    if( m_directories.contains( dir ) )
        return m_directories.value( dir );

    int deviceId = m_collection->mountPointManager()->getIdForUrl( dir );
    QString rpath = m_collection->mountPointManager()->getRelativePath( deviceId, dir );
    if( !rpath.endsWith( '/' ) )
    {
        rpath += '/';
    }
    QString query = QString( "SELECT id, changedate FROM directories_temp WHERE deviceid = %1 AND dir = '%2';" )
                        .arg( QString::number( deviceId ), m_storage->escape( rpath ) );
    QStringList result = m_storage->query( query );
    if( result.isEmpty() )
    {
        return 0;
    }
    else
    {
        m_directories.insert( dir, result[0].toInt() );
        return result[0].toInt();
    }
}

/**
 *  Note that this function does not really check the albums.
 *  Instead it seems to sort the tracks into the existing albums but the full functionality is a mistery to me (Ralf)
 */
int
ScanResultProcessor::checkExistingAlbums( const QString &album )
{
    //DEBUG_BLOCK
    //debug() << "looking for album " << album;
    // "Unknown" albums shouldn't be handled as compilations
    if( album.isEmpty() )
        return 0;

    //check if this album already exists, ignoring the albumartist
    //if it does, and if each file of the album is alone in its directory
    //it's probably a compilation.
    //this handles A1 compilations that were automatically organized by Amarok
    if( !m_albumsHashByName.contains( album ) || m_albumsHashByName[album] == 0 )
    {
        //debug() << "hashByName doesn't contain album, or it's zero";
        return 0;
    }

    QStringList trackIds;
    QLinkedList<QString*> *llist = m_albumsHashByName[album];
    QLinkedList<int> albumIntList;
    foreach( QString* albumList, *llist )
        albumIntList.append( albumList[AlbumColId].toInt() ); //list of album IDs, now find tracks

    QLinkedList<int> trackIntList;
    foreach( int albumInt, albumIntList )
    {
        if( !m_tracksHashByAlbum.contains( albumInt ) || m_tracksHashByAlbum[albumInt] == 0 )
            continue;
        foreach( QString* slist, *m_tracksHashByAlbum[albumInt] )
            trackIntList.append( slist[TrackColId].toInt() ); //list of tracks matching those album IDs
    }

    //note that there will be a 1:1 mapping between tracks and urls, although the id is not necessarily the same
    //and there may be more urls than tracks -- this means that this track list is all we need
    //the big mama
    int l_deviceid;
    QString l_rpath, l_trackId, l_albumId, l_albumArtistId, l_currentPath;
    foreach( int track, trackIntList )
    {
        if( !m_tracksHashById.contains( track ) || m_tracksHashById[track] == 0 )
            continue;
        QString *trackList = m_tracksHashById[track];
        int trackUrl = trackList[TrackColUrl].toInt();
        int trackAlbum = trackList[TrackColAlbum].toInt();

        if( !m_urlsHashById.contains( trackUrl ) || m_urlsHashById[trackUrl] == 0 )
            continue;
        QString *urlList = m_urlsHashById[trackUrl];

        if( !m_albumsHashById.contains( trackAlbum ) || m_albumsHashById[trackAlbum] == 0 )
            continue;
        QString *albumList = m_albumsHashById[trackAlbum];

        l_deviceid = urlList[UrlColDevice].toInt();
        l_rpath = urlList[UrlColRPath];
        l_trackId = QString::number( track );
        l_albumId = trackList[TrackColAlbum];
        l_albumArtistId = albumList[TrackColArtist];
        l_currentPath = m_collection->mountPointManager()->getAbsolutePath( l_deviceid, l_rpath );
        QFileInfo info( l_currentPath );
        uint dirCount = m_filesInDirs.value( info.dir().absolutePath() );
        if( dirCount == 1 )
        {
            trackIds << l_trackId;
        }
    }

    //debug() << "trackIds = " << trackIds;
    if( trackIds.isEmpty() )
    {
        //debug() << "trackIds empty, returning zero";
        return 0;
    }
    else
    {
        trackIds << QString::number( -1 );
        int compilationId = albumId( album, 0 );
        QString compilationString = QString::number( compilationId );
        foreach( const QString &trackId, trackIds )
        {
            int value = trackId.toInt();
            if( m_tracksHashById.contains( value ) && m_tracksHashById[value] != 0 )
            {
                QString* list = m_tracksHashById[value];
                list[TrackColAlbum] = compilationString;
            }
        }
        //debug() << "returning " << compilationId;
        return compilationId;
    }
}

void
ScanResultProcessor::setupDatabase()
{
//     DEBUG_BLOCK
    if( !m_setupComplete )
    {
        debug() << "Setting up database";
        m_collection->dbUpdater()->createTemporaryTables();
        if( m_type == IncrementalScan )
        {
            m_collection->dbUpdater()->prepareTemporaryTables();
        }
        else
        {
            m_collection->dbUpdater()->prepareTemporaryTablesForFullScan();
        }
        m_setupComplete = true;
        populateCacheHashes();
        /*
        debug() << "Next URL num: " << m_nextUrlNum;
        foreach( QString key, m_urlsHashByUid.keys() )
            debug() << "Key: " << key << ", list: " << *m_urlsHashByUid[key];
        foreach( int key, m_urlsHashById.keys() )
            debug() << "Key: " << key << ", list: " << *m_urlsHashById[key];
        typedef QPair<int, QString> blahType; //QFOREACH is stupid when it comes to QPairs
        foreach( blahType key, m_urlsHashByLocation.keys() )
            debug() << "Key: " << key << ", list: " << *m_urlsHashByLocation[key];
        debug() << "Next album num: " << m_nextAlbumNum;
        //foreach( int key, m_albumsHashById.keys() )
        //    debug() << "Key: " << key << ", list: " << *m_albumsHashById[key];
        //foreach( QString key, m_albumsHashByName.keys() )
        //{
        //    foreach( QStringList* list, *m_albumsHashByName[key] )
        //       debug() << "Key: " << key << ", list ptrs: " << *list;
        //}
        debug() << "Next track num: " << m_nextTrackNum;
        //foreach( int key, m_tracksHashById.keys() )
        //    debug() << "Key: " << key << ", list: " << *m_tracksHashById[key];
        //foreach( int key, m_tracksHashByUrl.keys() )
        //    debug() << "Key: " << key << ", list: " << *m_tracksHashByUrl[key];
        //foreach( int key, m_tracksHashByAlbum.keys() )
        //{
        //    foreach( QStringList* list, *m_tracksHashByAlbum[key] )
        //        debug() << "Key: " << key << ", list: " << *list;
        //}
        // */
    }

}

/**
 *  Pulls all url, album and track data from the temporary tables and puts them into the different hashes.
 */
void
ScanResultProcessor::populateCacheHashes()
{
    DEBUG_BLOCK

    //urls
    QStringList res = m_storage->query( "SELECT * FROM urls_temp ORDER BY id ASC;" );
    int reserveSize = ( res.size() / UrlColMaxCount ) * 2; //Reserve plenty of space to bring insertion and lookup close to O(1)
    m_urlsHashByUid.reserve( reserveSize );
    m_urlsHashById.reserve( reserveSize );
    m_urlsHashByLocation.reserve( reserveSize );
    int index = 0;
    int lastNum = 0;
    while( index < res.size() )
    {
        if( !res.at( index + UrlColUid ).startsWith( "amarok-sqltrackuid" ) )
        {
            debug() << "UHOH: Found track with invalid uid of " << res.at( index + UrlColUid );
            index += UrlColMaxCount;
            continue;
        }
        else
        {
            QString *currUrl = new QString[UrlColMaxCount];
            lastNum = res.at( index ).toInt();
            for( int i = 0; i < UrlColMaxCount; i++ )
                currUrl[i] = res.at(index++);
            m_urlsHashByUid.insert( currUrl[UrlColUid], currUrl );
            m_urlsHashById.insert( lastNum, currUrl );
            QPair<int, QString> locationPair( currUrl[UrlColDevice].toInt(), currUrl[UrlColRPath] );
            //debug() << "inserting locationPair " << locationPair;
            m_urlsHashByLocation.insert( locationPair, currUrl );
        }
    }
    m_nextUrlNum = lastNum + 1;
    m_storage->query( "DELETE FROM urls_temp;" );

    //albums
    res = m_storage->query( "SELECT * FROM albums_temp ORDER BY id ASC;" );
    reserveSize = ( res.size() / AlbumColMaxCount ) * 2;
    m_albumsHashByName.reserve( reserveSize );
    m_albumsHashById.reserve( reserveSize );
    index = 0;
    lastNum = 0;
    while( index < res.size() )
    {
        QString *currAlbum = new QString[AlbumColMaxCount];
        lastNum = res.at( index ).toInt();
        for( int i = 0; i < AlbumColMaxCount; i++ )
            currAlbum[i] = res.at(index++);

        m_albumsHashById.insert( lastNum, currAlbum );

        if( m_albumsHashByName.contains( currAlbum[AlbumColTitle] ) )
        {
            QLinkedList<QString*> *linkedAlbums = m_albumsHashByName[currAlbum[AlbumColTitle]];
            linkedAlbums->append( currAlbum );
        }
        else
        {
            QLinkedList<QString*> *linkedAlbums = new QLinkedList<QString*>();
            linkedAlbums->append( currAlbum );
            m_albumsHashByName.insert( currAlbum[AlbumColTitle], linkedAlbums);
        }
    }
    m_nextAlbumNum = lastNum + 1;
    m_storage->query( "DELETE FROM albums_temp;" );

    //tracks
    res = m_storage->query( "SELECT * FROM tracks_temp ORDER BY id ASC;" );
    reserveSize = ( res.size() / TrackColMaxCount ) * 2;
    m_tracksHashById.reserve( reserveSize );
    index = 0;
    lastNum = 0;
    while( index < res.size() )
    {
        // qDebug() << "Reading track "<<m_nextTrackNum<<" at index "<<index<<" with id "<<res.at(index);
        QString *currTrack = new QString[TrackColMaxCount];
        lastNum = res.at( index ).toInt();
        for( int i = 0; i < TrackColMaxCount; i++ )
            currTrack[i] = res.at(index++);
        m_tracksHashById.insert( lastNum, currTrack );
        m_tracksHashByUrl.insert( currTrack[TrackColUrl].toInt(), currTrack );

        int currAlbum = currTrack[TrackColAlbum].toInt();
        if( m_tracksHashByAlbum.contains( currAlbum ) )
        {
            QLinkedList<QString*> *linkedTracks = m_tracksHashByAlbum[currAlbum];
            linkedTracks->append( currTrack );
        }
        else
        {
            QLinkedList<QString*> *linkedTracks = new QLinkedList<QString*>();
            linkedTracks->append( currTrack );
            m_tracksHashByAlbum.insert( currAlbum, linkedTracks );
        }
    }
    m_nextTrackNum = lastNum + 1;
    m_storage->query( "DELETE FROM tracks_temp;" );

    //artists
    res = m_storage->query( "SELECT * FROM artists_temp ORDER BY id ASC;" );
    m_artists.reserve( res.size() );
    index = 0;
    lastNum = 0;
    while( index < res.size() )
    {
        lastNum = res.at( index++ ).toInt();
        m_artists.insert( res.at( index++ ), lastNum );
    }
    m_nextArtistNum = lastNum + 1;
    m_storage->query( "DELETE FROM artists_temp;" );

    //composers
    res = m_storage->query( "SELECT * FROM composers_temp ORDER BY id ASC;" );
    m_composers.reserve( res.size() );
    index = 0;
    lastNum = 0;
    while( index < res.size() )
    {
        lastNum = res.at( index++ ).toInt();
        m_composers.insert( res.at( index++ ), lastNum );
    }
    m_nextComposerNum = lastNum + 1;
    m_storage->query( "DELETE FROM composers_temp;" );

    //genres
    res = m_storage->query( "SELECT * FROM genres_temp ORDER BY id ASC;" );
    m_genres.reserve( res.size() );
    index = 0;
    lastNum = 0;
    while( index < res.size() )
    {
        lastNum = res.at( index++ ).toInt();
        m_genres.insert( res.at( index++ ), lastNum );
    }
    m_nextGenreNum = lastNum + 1;
    m_storage->query( "DELETE FROM genres_temp;" );

    //images
    res = m_storage->query( "SELECT * FROM images_temp ORDER BY id ASC;" );
    m_imagesFlat.reserve( res.size() );
    index = 0;
    lastNum = 0;
    while( index < res.size() )
    {
        lastNum = res.at( index++ ).toInt();
        m_imagesFlat.insert( res.at( index++ ), lastNum );
    }
    m_nextImageNum = lastNum + 1;
    m_storage->query( "DELETE FROM images_temp;" );

    //years
    res = m_storage->query( "SELECT * FROM years_temp ORDER BY id ASC;" );
    m_years.reserve( res.size() );
    index = 0;
    lastNum = 0;
    while( index < res.size() )
    {
        lastNum = res.at( index++ ).toInt();
        m_years.insert( res.at( index++ ), lastNum );
    }
    m_nextYearNum = lastNum + 1;
    m_storage->query( "DELETE FROM years_temp;" );

}

// --- some help functions for the query
QString
ScanResultProcessor::nullString(const QString &str) const
{
    if (str.isEmpty())
        return "NULL";
    return str;
}

QString
ScanResultProcessor::escaString(const QString &str) const
{
    return "'"+m_storage->escape(str)+"'";
}

//QLocale is set by default from LANG, but this will use , for floats, which screws up the SQL.
QString
ScanResultProcessor::doubString(const QString &str) const
{
    if (str.isEmpty())
        return "NULL";
    return QString( str ).replace( ',' , '.' );
}


void
ScanResultProcessor::copyHashesToTempTables()
{
    /*
    debug() << "Next URL num: " << m_nextUrlNum;
    foreach( QString key, m_urlsHashByUid.keys() )
        debug() << "Key: " << key << ", list: " << *m_urlsHashByUid[key];
    foreach( int key, m_urlsHashById.keys() )
        debug() << "Key: " << key << ", list: " << *m_urlsHashById[key];
    typedef QPair<int, QString> blahType; //QFOREACH is stupid when it comes to QPairs
    foreach( blahType key, m_urlsHashByLocation.keys() )
        debug() << "Key: " << key << ", list: " << *m_urlsHashByLocation[key];
    debug() << "Next album num: " << m_nextAlbumNum;

    foreach( int key, m_tracksHashById.keys() )
        debug() << "Key: " << key << ", list: " << *m_tracksHashById[key];
    foreach( int key, m_tracksHashByUrl.keys() )
        debug() << "Key: " << key << ", list: " << *m_tracksHashByUrl[key];
    foreach( int key, m_tracksHashByAlbum.keys() )
    {
        debug() << "Key: " << key;
        foreach( QStringList* item, *m_tracksHashByAlbum[key] )
            debug() << "list: " << item << " is " << *item;
    }
    */
 
    DEBUG_BLOCK
    QString query;
    QString queryStart;
    QString currQuery;
    QStringList res;
    bool valueReady;

    res = m_storage->query( "SHOW VARIABLES LIKE 'max_allowed_packet';" );
    if( res.size() < 2 || res[1].toInt() == 0 )
    {
        debug() << "Uh oh! For some reason MySQL thinks there isn't a max allowed size!";
        return;
    }
    debug() << "obtained max_allowed_packet is " << res[1];
    int maxSize = res[1].toInt() / 3; //for safety, due to multibyte encoding

    //urls
    debug() << "urls key size is " << m_urlsHashById.keys().size();
    queryStart = "INSERT INTO urls_temp VALUES ";
    query = queryStart;
    valueReady = false;
    QList<int> keys = m_urlsHashById.keys();
    qSort( keys );
    QSet<int> invalidUrls;
    foreach( int key, keys )
    {
        QString *currUrl = m_urlsHashById[key];
        if( !currUrl[UrlColUid].startsWith( "amarok-sqltrackuid" ) )
        {
            debug() << "UHOH: Trying to insert invalid entry into urls with id of " << currUrl[UrlColId] << " and uid of " << currUrl[UrlColUid];
            invalidUrls << currUrl[UrlColId].toInt();
            continue;
        }
        //debug() << "inserting following list: " << currUrl;
        currQuery =   "(" + currUrl[UrlColId] + ","
                          + nullString(currUrl[UrlColDevice]) + ","
                          + escaString(currUrl[UrlColRPath]) + ","
                          + nullString(currUrl[UrlColDir]) + ","
                          + escaString(currUrl[UrlColUid]) + ")"; //technically allowed to be NULL but it's the primary key so won't get far
        if( query.size() + currQuery.size() + 1 >= maxSize - 3 ) // ";"
        {
            query += ";";
            //debug() << "inserting " << query << ", size " << query.size();
            m_storage->insert( query, QString() );
            query = queryStart;
            valueReady = false;
        }

        if( !valueReady )
        {
            query += currQuery;
            //debug() << "appending " << currQuery;
            valueReady = true;
        }
        else
        {
            //debug() << "appending , + " << currQuery;
            query += "," + currQuery;
        }
    }
    if( query != queryStart )
    {
        query += ";";
        //debug() << "inserting " << query << ", size " << query.size();
        m_storage->insert( query, QString() );
    }
    keys.clear();

    //albums
    queryStart = "INSERT INTO albums_temp VALUES ";
    query = queryStart;
    valueReady = false;
    keys = m_albumsHashById.keys();
    qSort( keys );
    foreach( int key, keys )
    {
        QString *currAlbum = m_albumsHashById[key];
        currQuery =   "(" + currAlbum[AlbumColId] + ","
                          + escaString(currAlbum[AlbumColTitle]) + ","
                          + nullString(currAlbum[AlbumColArtist]) + ","
                          + nullString(currAlbum[AlbumColMaxImage]) + ")";
        if( query.size() + currQuery.size() + 1 >= maxSize - 3 ) // ";"
        {
            query += ";";
            //debug() << "inserting " << query << ", size " << query.size();
            m_storage->insert( query, QString() );
            query = queryStart;
            valueReady = false;
        }

        if( !valueReady )
        {
            query += currQuery;
            valueReady = true;
        }
        else
            query += "," + currQuery;
    }
    if( query != queryStart )
    {
        query += ";";
        //debug() << "inserting " << query << ", size " << query.size();
        m_storage->insert( query, QString() );
    }
    keys.clear();

    //tracks
    debug() << "tracks key size is " << m_tracksHashById.keys().size();
    queryStart = "INSERT INTO tracks_temp VALUES ";
    query = queryStart;
    valueReady = false;
    keys = m_tracksHashById.keys();
    qSort( keys );
    foreach( int key, keys )
    {
        // debug() << "key = " << key << ", id = " << m_tracksHashById.value(key)[TrackColId];
        QString *currTrack = m_tracksHashById.value(key);
        if( invalidUrls.contains( currTrack[TrackColUrl].toInt() ) )
        {
            debug() << "UHOH: Skipping track with url id " << currTrack[TrackColUrl] << " because it's in the invalidUrls set";
            continue;
        }

        currQuery =   "(" + currTrack[TrackColId] + ","
                          + nullString(currTrack[TrackColUrl]) + ","
                          + nullString(currTrack[TrackColArtist]) + ","
                          + nullString(currTrack[TrackColAlbum]) + ","
                          + nullString(currTrack[TrackColGenre]) + ","
                          + nullString(currTrack[TrackColComposer]) + ","
                          + nullString(currTrack[TrackColYear]) + ","
                          + escaString(currTrack[TrackColTitle]) + ","
                          + escaString(currTrack[TrackColComment]) + ","
                          + nullString(currTrack[TrackColTrackNumber]) + ","
                          + nullString(currTrack[TrackColDiscNumber]) + ","
                          + nullString(currTrack[TrackColBitRate]) + ","
                          + nullString(currTrack[TrackColLength]) + ","
                          + nullString(currTrack[TrackColSampleRate]) + ","
                          + nullString(currTrack[TrackColFileSize]) + ","
                          + nullString(currTrack[TrackColFileType]) + ","
                          + doubString(currTrack[TrackColBpm]) + ","
                          + nullString(currTrack[TrackColCreated]) + ","
                          + nullString(currTrack[TrackColModified]) + ","
                          + doubString(currTrack[TrackColAlbumGain]) + ","
                          + doubString(currTrack[TrackColAlbumPeakGain]) + ","
                          + doubString(currTrack[TrackColTrackGain]) + ","
                          + doubString(currTrack[TrackColTrackPeakGain]) + ")";
        if( query.size() + currQuery.size() + 1 >= maxSize - 3 ) // ";"
        {
            query += ";";
            //debug() << "inserting " << query << ", size " << query.size();
            m_storage->insert( query, QString() );
            query = queryStart;
            valueReady = false;
        }

        if( !valueReady )
        {
            query += currQuery;
            valueReady = true;
        }
        else
            query += "," + currQuery;
    }
    if( query != queryStart )
    {
        query += ";";
        //debug() << "inserting " << query << ", size " << query.size();
        m_storage->insert( query, QString() );
    }

    genericCopyHash( "artists", &m_artists, maxSize );
    genericCopyHash( "composers", &m_composers, maxSize );
    genericCopyHash( "genres", &m_genres, maxSize );
    genericCopyHash( "images", &m_imagesFlat, maxSize );
    genericCopyHash( "years", &m_years, maxSize );
}

void
ScanResultProcessor::genericCopyHash( const QString &tableName, const QHash<QString, int> *hash, int maxSize )
{
    QString currString;
    QString currQuery;
    QString queryStart = "INSERT INTO " + tableName + "_temp VALUES ";
    QString query = queryStart;
    bool valueReady = false;
    QStringList keys = hash->keys();
    QHash<int, QString> sortedHash;
    foreach( const QString &key, keys )
        sortedHash.insert( hash->value( key ), key );
    QList<int> intKeys = sortedHash.keys();
    qSort( intKeys );
    foreach( int key, intKeys )
    {

        currString = sortedHash[key];
        //currQuery =   "(" + QString::number( hash->value( key ) ) + ",'" + m_storage->escape( key ) + "')";
        currQuery =   "(" + QString::number( key ) + ",'" + m_storage->escape( sortedHash[key] ) + "')";
        if( query.size() + currQuery.size() + 1 >= maxSize - 3 ) // ";"
        {
            query += ";";
            //debug() << "inserting " << query << ", size " << query.size();
            m_storage->insert( query, QString() );
            query = queryStart;
            valueReady = false;
        }

        if( !valueReady )
        {
            query += currQuery;
            valueReady = true;
        }
        else
            query += "," + currQuery;
    }
    if( query != queryStart )
    {
        query += ";";
        //debug() << "inserting " << query << ", size " << query.size();
        m_storage->insert( query, QString() );
    }
}

#include "ScanResultProcessor.moc"

