/***************************************************************************
 *   Copyright (C) 2005 by xavier helluy   *
 *   xavier.helluy@web.de   *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   51 Franklin Steet, Fifth Floor, Boston, MA 02110-1301, USA.             *
 ***************************************************************************/

#include <kcluceneindexerdaemon.h>
#include <kio/metainfojob.h>
#include <kio/previewjob.h>
#include <iostream>

#include <kprocess.h>

#include <kapplication.h>
#include <kglobal.h>
#include <kprocess.h>
#include <assert.h>
#include <qeventloop.h>
#include <kio/netaccess.h>
#include <kio/job.h>
#include <kstandarddirs.h>
#include <kconfigbackend.h>

#include <kdebug.h>
#include "urldispatcher.h"
#include <kdebug.h>
#include <dcopclient.h>
#include <dcopobject.h>
#include <qtimer.h>

#include <kio/netaccess.h>

#include <qfileinfo.h>
#include <kfilemetainfo.h>
 
 
/*run as SYNC indexer over the user-input urls
TODO: accept command line for scripting cron-jobs
TODO: do  ioslaves to extract text and ioslaves to extracts images and medias too.

Not fast. It will copy to a local file all non-local URLs.
This allows to index archives in archives like war files for examples
This allows to index remote sites.
This is more time consumming, and a bit more complex to programm, in order not to
mix all the URLs.
One has:
- a realMasterURL
- a localMasterTmpURL if remote realURL
- possibly many localMinorURLs for archive-like documents
And whose URLS relationships must survive recursion !!
index(item = realMasterURL, realURL = realMasterURL)
	copy realMasterURL to localMasterTmpURL; 
	now it is an archive. Open with a tar/zip
	realMasterURL = realMasterURL#tar:/
	localURL = tar:localURL
	listdir(localURL, localMasterTmpURL, realMasterURL#tar:/);
	if files to index metadata: must be copyed ugain
		copy file newLocalURL
		index(newlocalItem, realMasterURL#tar:/ + )
	if dir 
		 realMasterURL#tar:/dir.name ?? or last /---/ from dir.url ?
		listdir(newlocalURL, realMasterURL#tar:/dir);
		if file: check if name and URL ok: this must not be "Redirection", like digikamtags
			
		if dir2
			listdir(newlocalURL, realMasterURL#tar:/dir/dir2);
		if archive
			listdir(newlocalURL, realMasterURL#tar:/dir/dir2/#zip:/);	
			index(localAtomicItem,realMasterURL#tar:/dir/dir2/#zip:/) )
	
	index(localMasterTmpURL, realMasterURL)
Use subURLS, this will be supported in kde one day.
One needs to remember the localMasterTmpURL and the realMasterURL.
Now localMasterTmpURL is a Zip dir. I need to create


Normaly, 2 options when dealing with a archive-like doc containing text and images:
- I index both types of course, but at the end, one has many urls but a single doc.
This will confuse the user, but I WANT TO SEE THE IMAGES INSIDE MY DOCS
- I think the easier is to add a URL-filtering at the query-level or the results-level. to remove unwanted results, like the metainfo.xml or mimetype.xml to appear.
*/

KCLuceneIndexerDaemon::KCLuceneIndexerDaemon() 
	: QObject() , KCLuceneIndexerDaemonDCOPIface(),
          DCOPObject("KCLuceneIndexerDaemon"),
	  m_indexName			(QString::null),
	  m_indexLocation		(QString::null),
	  m_overWriteIndex		(FALSE),
	  m_overWriteMedia		(FALSE),
	  m_syncIndex			(FALSE),
	  m_updateIndex			(FALSE),
	  m_appendIndex			(TRUE),
	  m_includeSubFolders		(TRUE),
	  m_excludeSubFolders		(TRUE),
	  m_insideArchive		(TRUE),
	  m_callingClient		(""),
	  m_query			(""),
	  m_mediaType			("unknown"),
	  m_mediaName			("unknown"),
	  m_maxEntriesToIndex		(100),
	  m_countEntriesToIndex	        (0),
	  m_counterRunOptimize	        (0),
	  m_maxIndexedEntries		(100),
	  m_countIndexedEntries		(0),
	  m_mode			("APPED")
	  
{
	m_idReader = 0;
	m_idSearcher = 0;
	m_writer = 0;
	m_doc = 0;
	
	//preset converters to nothing
	m_convert = new Converters(); 

	m_itemsToIndex.setAutoDelete(TRUE);
	m_itemsToIndex.clear();
	//dcop communication
	connect (this, SIGNAL(indexRequestReceived()) , this, SLOT(startNewIndexRequest()));
	
	connect (this, SIGNAL(deleteHitsCallReceived()) , this, SLOT(startDeleteHits()));
	
	//TODO: setup a default database for "on the fly" indexing
	
	//automatic logout
	m_timer = new QTimer( this );
	connect( m_timer, SIGNAL(timeout()), this, SLOT(timerDone()) );
}


KCLuceneIndexerDaemon::~KCLuceneIndexerDaemon()
{
	delete m_convert;
	
}

void KCLuceneIndexerDaemon::timerDone()
{
	kdDebug() << "KCLuceneIndexerDaemon::timerDone QUIT DAEMON NOW" <<endl;

	kapp->quit();
}

void KCLuceneIndexerDaemon::startNewIndexRequest()
{
	QTimer::singleShot( 1*1*10, this, SLOT(startIndexing()));
// 	indexFiles();
}

//start indexing :-)
//the problem of Lucene: you cannot delete data from the index and add data at the same time
void KCLuceneIndexerDaemon::startIndexing()
{
	//create a standard analyser
	m_an = new lucene::analysis::standard::StandardAnalyzer();
	
	//check if index exists already
	bool indexExists = checkIfIndexExists( m_indexLocation);

	if ( !indexExists)
	{
		//no index directory found. CREATE ONE
		initIndexDirectory(m_indexLocation, TRUE);
		startOverWriteMode();
	}
	else
	{
		if (m_overWriteIndex)
		{
			//index found. But OVERWRITE mode selected, so destroy existing index  
			initIndexDirectory(m_indexLocation, TRUE);
			startOverWriteMode();
		}
		else
		{
			//init the directory pointer
			initIndexDirectory(m_indexLocation, FALSE);

			if (m_overWriteMedia)
			{
				startOverWriteMediaMode();
			}
			
			else if (m_syncIndex)
			{
				startSyncMode();
			}
		
			else if (m_updateIndex)	
			{
				startUpdateMode();
			}
	
			else if (m_appendIndex)
			{
				startAppendMode();
			}
		}
	}
	
	QByteArray data;
	kapp->dcopClient()->send ((QCString)
			 m_callingClient,"CLuceneIndexerWidget","indexerFinished()",data );
	kdDebug() << "KCLuceneIndexerDaemon::indexFiles:dcop INDEXING FINISHED:" 
			<<  m_callingClient  << endl;
	
	//start countdown. quit after 15 min idle
	int delay = 15*60*1000;
	//this should never happend
	if ( m_timer->isActive() )
	{
		m_timer->stop();
		m_timer->start( delay, TRUE );
	}
	else
	{
		m_timer->start( delay, TRUE );
	}

	//cleanup
	m_d->close();
 	//delete m_d;
	delete m_an;
	m_d = 0;
}

//start OVERWRITE MODE
void KCLuceneIndexerDaemon::startOverWriteMode()
{
	kdDebug() <<"KCLuceneIndexerDaemon::startOverWriteMode()"<<endl;

	//we erased the index, and can now add new items
	m_mode = "OVERWRITE";
	
	//this writer as initialized, will not erase the content of the indexRequestReceived
	//it is not OWNER of the Index
	//this means m_writer->close(FALSE) will not close the Directory m_d
	//but  m_writer->close(TRUE) would do.
	initIndexWriter(m_d, m_an, TRUE);
	
	//prepare list of KFileItems and of urls strings from user-input
	KFileItemList itemsList;
	itemsList.clear();
	stringListToKFileItemList(m_includeItems, itemsList, FALSE);
	
	//recurse into dirctories and archives if requested. 
	indexingLoopOverItems( itemsList, FALSE, TRUE);
}

//start APPEND MODE
void KCLuceneIndexerDaemon::startAppendMode()
{
	kdDebug() <<"KCLuceneIndexerDaemon::startAppendMode()"<<endl;
	
	m_mode = "APPEND";
	
	//this writer as initialized, will not erase the content of the indexRequestReceived
	//it is not OWNER of the Index
	//this means m_writer->close(FALSE) will not close the Directory m_d
	//but  m_writer->close(TRUE) will do.
	initIndexWriter(m_d, m_an, FALSE);
	
	//prepare list of KFileItems and of urls strings
	KFileItemList itemsList;
	itemsList.clear();
	stringListToKFileItemList(m_includeItems, itemsList, FALSE);
	
	//recurse into dirctories and archives if requested. 
	//do not check for updates
	indexingLoopOverItems( itemsList, FALSE, TRUE);
	//indexWriter deleted at the end of writeLoopOverURLs
}

//start OVERWRITE MEDIA MODE
void KCLuceneIndexerDaemon::startOverWriteMediaMode()
{
	kdDebug() <<"KCLuceneIndexerDaemon::startOverWriteMediaMode()"<<endl;
	
	m_mode = "MEDIA";

	//init indexReader
	openReaderAndSearcher();

	//erase all Media fields
	eraseMediaEntries();
	
	//close reader and start a writer
	closeReaderAndSearcher();
	
	initIndexWriter(m_d, m_an, FALSE);
		
	//prepare list of KFileItems and of urls strings
	KFileItemList itemsList;
	itemsList.clear();
	stringListToKFileItemList(m_includeItems, itemsList, FALSE);
	
	//starts a readLoop in automatic mode
	indexingLoopOverItems( itemsList, FALSE, TRUE);
}


//start UPDATE MODE
void KCLuceneIndexerDaemon::startUpdateMode()
{
	kdDebug() <<"KCLuceneIndexerDaemon::startUpdateMode()"<<endl;
	
	m_mode = "UPDATE";

	//init indexReader
	openReaderAndSearcher();

	//prepare list of KFileItems and of urls strings
	KFileItemList itemsList;
	itemsList.clear();
	stringListToKFileItemList(m_includeItems, itemsList, FALSE);
	
	//starts a readLoop in automatic mode
	indexingLoopOverItems( itemsList, TRUE, TRUE);
}

//start SYNC MODE
void KCLuceneIndexerDaemon::startSyncMode()
{
	kdDebug() <<"KCLuceneIndexerDaemon::startSyncMode()"<<endl;
	
	m_mode = "SYNC";

	//init indexReader
	openReaderAndSearcher();
	//starts a readLoop over all index entries
	indexingLoopOverIndexEntries();
}

void KCLuceneIndexerDaemon::eraseMediaEntries()
{
	QString query = m_mediaType + "/" + m_mediaName;
	kdDebug()<< "KCLuceneIndexerDaemon::eraseMediaEntries() CALLED" << query << endl;

	//parse query  see http://java2.5341.com/msg/95918.html
	TCHAR * input = (wchar_t *)query.ucs2();
		
	Term * t = new Term(_T("MediaKey"), input);
	int m = m_idReader->deleteTerm(t);
	kdDebug() << "KCLuceneIndexerDaemon::eraseMediaEntries Media DELETED NUMER: " << m << endl;
	
	delete t;
}


//a fun HACK to get digikamtags in the database
void KCLuceneIndexerDaemon::buildDigikamQMap(const QString & digikamURLstr,  KFileItemList & lst)
{
	kdDebug()<<"KCLuceneIndexerDaemon::buildDigikamQMap() SIZE: "<<  lst.count()<< endl;

	const KFileItem * item;
	for (item=lst.first(); item; item=lst.next()) 
	{
		kdDebug()<<"KCLuceneIndexerDaemon::buildDigikamQMap() SIZE: "<< item->url().pathOrURL() << " |" << digikamURLstr << endl;

		m_digikamListURLs.append(item->url().pathOrURL());
		m_digikamListTags.append(digikamURLstr );
	}
	
	kdDebug()<<"KCLuceneIndexerDaemon::buildDigikamQMap() SIZE ITEM: "<<  lst.count()<< "  SIZE DIGI" << m_digikamListTags.count() <<  endl;

}

//tansforms urls from ioslave listdir into a KFileItemList. Removes unwanted urls: self,.,..
void KCLuceneIndexerDaemon::listDirEntriesToKFileItemList(const KIO::UDSEntryList & lst, KFileItemList & itemList, const KURL  u, KURL * realURL)
{
	kdDebug()<<"KCLuceneIndexerDaemon::listDirEntriesToKFileItemList() SIZE: "<<  lst.count()<< endl;
	
	itemList.clear();
	

	//speed HACK ignore first entry. it is the mothership 
// 	it++;
	//more "speed" if i get the uds entry insteed of always building KFileItem 
	KIO::UDSEntryListConstIterator it;
	for (it = lst.begin(); it !=  lst.end(); ++it) 
	{
		//get the MIMETYPE now.
		//FIXME:very SLOW do I could use MimetypeJob if the url is not local and returns mimetype = octet/stream ?
		KFileItem * t =  new KFileItem ( (*it) , u ,FALSE, TRUE);
		//if mimetype is octet/stream, try using a kiojob
		if ( t->mimetype() == "application/octet-stream" )
		{
			//try again with a job
			QString mime = KIO::NetAccess::mimetype ( t->url() , 0);
			if ( mime != "application/octet-stream" )
			{
				mode_t mod = t->mode();
				KURL  url = t->url();
				delete t;
				t =  new KFileItem ( url, mime, mod);
			}
		}
		if ( !(t->url().fileName() == "." || t->url().fileName() == "..") )
		{
			if (! (t->url() == u))
			{	
				//check for realURL not a null pointer
				if ( realURL != 0)
				{
					realURL->addPath(t->name());
					
					//add the realURL to the item
					t->setExtraData ( "realURL" , new KURL(*realURL));
				}
				
				itemList.append( new KFileItem (*t) );
				kdDebug()<<"KCLuceneIndexerDaemon::listDirEntriesToKFileItemList() : "<<  t->url()		 << endl;
				delete t;
			}
			else
			{
				delete t;
			}
		}
		else
		{
			delete t;
		}
		
	}
}

//this should returns 2 lists with same urls.
//it used to convert the user input-urls into KFileItems. They must exists
void KCLuceneIndexerDaemon::stringListToKFileItemList( QStringList & includeURLs, KFileItemList & itemList, bool fromDir)
{
	kdDebug()<< "KCLuceneIndexerDaemon::stringListToKFileItemList" << includeURLs << endl;
// 	QStringList::Iterator it2 =  includeURLs.begin();
	for (QStringList::Iterator it = includeURLs.begin(); it != includeURLs.end();)
	{
		kdDebug()<< "KCLuceneIndexerDaemon::stringListToKFileItemList" << *it << endl;

		KURL url = KURL::fromPathOrURL(*it);
		url.adjustPath(-1);
		kdDebug()<< "KCLuceneIndexerDaemon::stringListToKFileItemList" << url << endl;

		//create file item
		m_entry.clear();
		bool urlExists = KIO::NetAccess::stat(url ,m_entry, 0 );
		if ( urlExists )
		{
			kdDebug()<< "KCLuceneIndexerDaemon::stringListToKFileItemList URL EXISTS"   << endl;

			//create file item, get Mimetype immediately.
			KFileItem * t =  new KFileItem ( m_entry , url ,FALSE, fromDir);
			//if mimetype is octet/stream, try using a kiojob
			if ( t->mimetype() == "application/octet-stream" )
			{
				//try again with a job
				QString mime = KIO::NetAccess::mimetype ( t->url() , 0);
				if ( mime != "application/octet-stream" )
				{
					mode_t mod = t->mode();
					KURL  url = t->url();
					delete t;
					t =  new KFileItem ( url, mime, mod);
				}
			}
			
			itemList.append(new KFileItem( *t ));
			delete t;
			it++;
		}
		else
		{
			kdDebug()<< "KCLuceneIndexerDaemon::stringListToKFileItemList URL DOES NOT EXISTS"  << endl;

			//remove from the list of urls
			it =includeURLs.remove(it);
			kdDebug()<< "KCLuceneIndexerDaemon::stringListToKFileItemList URL REMOVED" << endl;
		}
	}
}

//do index exist ?
bool KCLuceneIndexerDaemon::checkIfIndexExists(QString indexLocation)
{
	char * target = (char*) indexLocation.latin1();
	return IndexReader::indexExists( target );
}

//init index directory
void KCLuceneIndexerDaemon::initIndexDirectory(const QString & indexLocation, const bool & erase)
{
	//need to know if the index exists
	char * target = (char*) indexLocation.latin1();
	
	bool indexExists = checkIfIndexExists(indexLocation);
	
	if ( indexExists )
	{
		kdDebug() << "KCLuceneIndexerDaemon::initIndex INDEX exists"<< endl;
		
		// APPEND, SYNC or UPDATE mode when index already exists
		if ( !erase)
		{
			//init index directory in "APPEND" mode
			m_d = FSDirectory::getDirectory( target,FALSE);
			
			//HACK: check if index locked and force unlock	Dangerous
			if ( IndexReader::isLocked(m_d) )
			{
				kdDebug() << "KCLuceneIndexerDaemon::initIndex Index was locked... unlocking it."<<endl;
				IndexReader::unlock(m_d);
				m_d->close();
				delete m_d;
				initIndexDirectory (indexLocation, erase);
			}
			
		}
		else
		{
			m_d = FSDirectory::getDirectory(target,TRUE);

		}
		
	}
	else
	{
		m_d = FSDirectory::getDirectory(target,	TRUE );
	}
}

//init index writer
void KCLuceneIndexerDaemon::initIndexWriter(Directory * d, lucene::analysis::standard::StandardAnalyzer * an, bool erase)
{
	kdDebug() << "KCLuceneIndexerDaemon::initIndexWriter" << endl;
	m_writer = new IndexWriter( d, an, erase);
}

//this will index the given list of items. Cleanup is unused
void KCLuceneIndexerDaemon::indexItems( KFileItemList & items)
{
	kdDebug() << " KCLuceneIndexerDaemon::indexItems. "<<endl;
	
	m_countIndexedEntries = 0;
	
	//start indexing	

	KFileItem *item;
	for (item = items.first(); item; item = items.next())
	{
		kdDebug() << "KCLuceneIndexerDaemon::indexItems. before write:  " << item->url() << endl;

		if ( m_doc == 0 )
			m_doc  =  new Document();
		else
		{
			kdDebug() << "KCLuceneIndexerDaemon::indexItems.Doc should have been NULL but wasn't:  " << endl;

			m_doc  =  new Document();
		}

		/*bool entryHasBeenWritten = */ 
		indexEntry(item);
		kdDebug() << "KCLuceneIndexerDaemon::indexItems. before write:  " << endl;
		m_writer->addDocument( m_doc );
		kdDebug() << "KCLuceneIndexerDaemon::indexItems. after write:  " << endl;
		
		if ( m_doc != 0 )
		{
			delete m_doc;
			m_doc = 0;
		}
		kdDebug() << "KCLuceneIndexerDaemon::indexItems. delete doc:  " << endl;

		m_countIndexedEntries ++;
		kdDebug() << "KCLuceneIndexerDaemon::indexItems. INDEXEDENTRIES:  " << m_countIndexedEntries<<endl;
		
		
		
	}
	for (item = items.first(); item; item = items.next())
	{
		kdDebug() << "KCLuceneIndexerDaemon::indexItems. before write:  " << item->url() << endl;
	}

	
	//digkamtags HACK do rewrite 
/*	openReaderAndSearcher();
	indexDigikamQMap();
	closeReaderAndSearcher();*/
	
}

void KCLuceneIndexerDaemon::flushAndCloseIndex(IndexWriter * writer)
{
	kdDebug() << "KCLuceneIndexerDaemon::flushAndCloseIndex" << endl;

	//closing only if writer is not empty
	writer->close();
	delete writer;
	writer = 0;
}

//index the given entry 
bool KCLuceneIndexerDaemon::indexEntry(KFileItem * item)
{
	kdDebug() << "KCLuceneIndexerDaemon::indexEntry. STARTS:  " << item->url()<< " with Mime:" << item->mimetype() <<endl;

	//FIXME: should there be an option to INDEX non-readable FILES
	
	/*Help needed*
	There is 
	- MimetypeJob ioslave
	- MetaInfoJob ioslave
	- PreviewJob  ioslaves
	- Kat content ioslaves
	
	Then there is :
	- KFileItem->mimetype() it uses  KMimeType::findByURL()
	- KMimeType::findByURL(). It looks at mode_t which should give the filetype. does not work for cgi scripts
	"If no extension matches, then the file will be examined if the URL is a local file or "application/octet-stream" is returned otherwise."
	So I could do: ->mimetype(TRUE), and if octet-stream try again with a sync MimeTypeSlave
	- KMimeType::findByContent(),....
	
	Problem is: one always needs the Mimetype. And it can be found on the fileSystem or in the file 
	
	//case of files to be treated as archives
	if (isContentWanted(item) && isArchive(item) && isArchiveWanted(item))
	{
	copy the file locally;
	}
	*/
	
	//if true data are written to index
	bool isToBeAdded = FALSE;		
	QString anyStringExeptContent;
	if ( isContentWanted(item) )
	{
		kdDebug() << "KCLuceneIndexerDaemon::indexEntry ITEM READABLE ? :" << item->isReadable() << endl;

		if (item->isReadable())	
		{
			//must copy non-local files or use Kat ioslaves
			//get local copy and tmp file with the correct file extension
			KTempFile * localCopy = 0;
			KFileItem * localItem = 0;
			KFileItem * selectedItem = 0;
			QString localItemURLStr;
			if (!item->isLocalFile())
			{
				QFileInfo fi( item->url().pathOrURL() );
				QString fileExtension	= fi.extension(TRUE); 
				localCopy = new KTempFile(QString::null,"."+ fileExtension,0600); 
				localCopy->setAutoDelete(FALSE);
				if (! (localCopy->status() == 0) ) 
				{
					kdDebug() <<"KCLuceneIndexerDaemon::indexEntry INPUT TMP FILE CREATION FAILED" << localCopy->name() <<endl;
					localCopy = 0;
				}
				else
				{
					localCopy->close();
					
					KIO::FileCopyJob *job =  KIO::file_copy
							(item->url(),localCopy->name(),-1,TRUE,
							FALSE,FALSE );
					if (! KIO::NetAccess::synchronousRun( job, 0 ) )
					{
						kdDebug() <<"KCLuceneIndexerDaemon::indexEntry local copy failed" <<endl;
						localItem = 0;
					}
					else
					{
						selectedItem = new KFileItem(*item);
						//dangerous HACK
						selectedItem->setURL(localCopy->name());
						selectedItem->refreshMimeType();
// 						localItem->refresh();
					}
				}
			}
			else
			{
				selectedItem =  new KFileItem(*item);
			}
			if ( selectedItem != 0 )
			{
				doContentIndexing(selectedItem);
				if ( m_restrictMetaData || m_allMetaData)
				{
					//get metaInfo from the local file
// 				if (this->metaInfoJob( selectedItem ) )
					doAllMetaDataIndexing(selectedItem, anyStringExeptContent);
				}
				//get Mod Time from the local file FIXME:creation time ???
				doTimesIndexing(selectedItem);
			}				
			//get Media and other basic keys
			if (item !=0)
			{
				doURLandMediaAndMimetypeIndexing(item, anyStringExeptContent);
			}
			
			//FIXME:how else could I do ? 
			//get preview for all images and store it on disk
			//the preview will be the icon of the item. How else could I do
			if (item !=0)
			{
				KURL realURL = item->url();
				//FIXME: always run doStorePreview if doURL is run 
				if (selectedItem !=0)
				{
					doStorePreview( selectedItem, realURL);
				}
			}
			
			//cleanup 
			//FIXME:item ?????
			if ((selectedItem !=0) )
			{
				delete selectedItem;
				selectedItem = 0;
			}
			if (!localItem == 0)
				delete localItem;
			if (!localCopy == 0)
			{
				localCopy->unlink();
				delete localCopy;
			}

		}
	}
	// user doesn't want to index the content of this file but the metainfo
	else if (isMetaDataWanted( item ))
	{
		//only want metadata and File-info
		//use METAINFO IOSLAVE only if non local ?
// 		if (this->metaInfoJob(item))
// 		{
			isToBeAdded = TRUE;
			
			doAllMetaDataIndexing(item, anyStringExeptContent);
			//will get Mod Time from the local file only 
			//what to do with remote file. I don't want to copy them 
			//what about archives: should work with KDE supported archives
			doTimesIndexing(item);
			
			//store Previews : FIXME: if non-local ???
			KURL realURL = item->url();
			doStorePreview( item, realURL);

					
			//get Media and other basic keys
			doURLandMediaAndMimetypeIndexing(item, anyStringExeptContent);
			
			//dump fileitem to disk
			QString pathToItem = m_indexLocation + "/." + m_mediaType + "/" +
					m_mediaName + "/" + realURL.pathOrURL() + "_item.png";
			QFile file(pathToItem);
			file.open( IO_WriteOnly );
			QDataStream in(&file);
			in << item;
// 			TCHAR tf[CL_MAX_DIR];
//    STRCPY_AtoT(tf,f,CL_MAX_DIR);
			m_doc->add( *Field::Keyword(_T("KFileItem"),  (const wchar_t*) pathToItem.ucs2() ) );
			
// 		}
		
		//cleanup
	}

	//write file-info fields. not here, but above, only when indexing has been successfull
	if (!anyStringExeptContent.isEmpty())
	{
		m_doc->add( *Field::Text(_T("anystring"), (wchar_t*)  anyStringExeptContent.ucs2() ) );
	}
	bool entryHasBeenWritten=TRUE;
	
	return entryHasBeenWritten;
}		

//check if the URL is allowed by the User and the system
bool KCLuceneIndexerDaemon::isURLExcluded(const KFileItem * item)
{
	bool excludeURL;
	KURL url = item->url(); 
	if (item->isLink())
	{
		kdDebug()<< "KCLuceneIndexerDaemon::isURLExcluded LINK" << endl;
		excludeURL = TRUE;
	}
	else if ( url.filename() == "..")
		excludeURL = TRUE;
	else if ( url.filename() == ".")
		excludeURL = TRUE;
	//very crude check
	else if (!( m_excludeItems.find(url.pathOrURL()) == m_excludeItems.end() ) )
	{
		kdDebug()<< "KCLuceneIndexerDaemon::isURLExcluded ON LIST" << endl;
		excludeURL = TRUE; 
	}
	else
		excludeURL = FALSE;

	kdDebug()<< "KCLuceneIndexerDaemon::isURLExcluded" << url.pathOrURL() << " excluded answer :" << excludeURL<< "from urls :" << m_excludeItems << endl;
	
	return excludeURL;
}

//return true if indexing the directory content (listdir) is allowed and possible
//callFromUserInput allow to consider the special case, which is when one considers the urls
//the user wants to include or exclude. In this case, urls are not coming from a listDir call
//
//so far, always recurse inside archives when working on archives
//	
bool KCLuceneIndexerDaemon::isIndexingDirContentAllowed(KFileItem * item, bool gotURLFromListDir)
{
	bool excludeSubURL;
	if ( m_includeSubFolders == FALSE) 
	{
		if ( gotURLFromListDir == FALSE )
		{
			//always recurse inside archive
			//not perfect because it will recurse if user enters tar:lalilala
			//and there it is clear that the user doesn't always want to recurse
			if (! ( item->url().protocol() == "tar" || item->url().protocol() == "zip" ) )
			{
				excludeSubURL = TRUE;
			}
		}
	}
	else 
	{
		KURL url = item->url().pathOrURL();
		//simple check here url is on list of urls to exclude
		if (!( m_excludeItems.find(url.pathOrURL()) == m_excludeItems.end() ) )
		{
			//subUrl should be excluded 
			if (m_excludeSubFolders)
			{
				excludeSubURL = TRUE;
			}
			else
			{
				excludeSubURL = FALSE;				
			}
		}
		else
		{
			//url is not on list of urls to exclude
			excludeSubURL = FALSE;				
		}
	}
	kdDebug()<< "KCLuceneIndexerDaemon::isIndexingDirContentAllowed(" << item->url() << " excluded sub dirs answer :" << !excludeSubURL <<endl;

	return (!excludeSubURL);
}

//this loops through all index-entries and removes entries with no corresponding urls
//and updates entries with older Mod.Time than the one of the url they are pointing to.
//FIXME:should I do that ? This checks for changes of converter/metada indexing. This removes all entry whom converter
//isn't found anymore 
bool KCLuceneIndexerDaemon::syncEntry(Document & /*doc*/, int docNumber, const QString &urlString, KFileItem * item)
{
	bool update;
	
	KURL u = KURL::fromPathOrURL(urlString);
	
	//FIXME:only for local files ? Archives ?
	m_entry.clear();
	bool itemExists = KIO::NetAccess::stat(u ,m_entry,0 );
	
	kdDebug() << "KCLuceneIndexerDaemon::syncEntry STATURL :" << u <<endl;
		
	if (!itemExists)
	{
		item = 0;
		kdDebug() << "KCLuceneIndexerDaemon::syncEntry URL NOT FOUND"  <<endl;
		
		//we are in sync mode and stat failed! Remove this entry from index
		if (! m_idReader->isDeleted(docNumber))
		{
			m_idReader->deleteDoc(docNumber);
			update = FALSE;
		}
		else
		{
			//strange, it should not be
			kdDebug() << "KCLuceneIndexerDaemon::syncEntry ERROR:"  <<endl;
			update = FALSE;
		}
	}
	else
	{
		//item exists
		item = new KFileItem(m_entry , u , FALSE , FALSE); 
		kdDebug() << "KCLuceneIndexerDaemon::syncEntry URL FOUND:" << item->url() <<endl;
		//check if mod time changed XXX
		if (! m_idReader->isDeleted(docNumber))
		{
			Document * doc =  m_idReader->document(docNumber);
			//check Mod.Time	
			if (hasModTimeChanged(*doc, item)) 
			{
				update = TRUE;
			}
			//FIXME check config changed (cannot check for change of converters) ""
			else if (hasConfigChanged(*doc, item))
			{
				update = TRUE;
			}

			else
			{
				update = FALSE;
			}
			delete &doc;
		}
		else
		{
			//strange
			kdDebug() << "KCLuceneIndexerDaemon::syncEntry() ERROR2:" << item->url() <<endl;
			update = FALSE;
		}
	}
	return update;
}


//TODO: this should help to get update works correctly. But does not take care of a change of converter/...
//if doc has been indexed with metadata only and I now want to index its contents
//then update should notice this situation and allow for the update.
//this is rather fuzzy. Sometimes only field removal would be needed, but this is not possible with LUCENE
QString KCLuceneIndexerDaemon::getConfigNumber(KFileItem * item)
{
	kdDebug() << "KCLuceneIndexerDaemon::getConfigNumber:" << item->url() <<endl;

	QString str;
	//append content wanted YES OR NO
	bool b;
	if (isContentWanted(item))	
	{
		b = TRUE;
	}
	else
	{
		b = FALSE;
	}
	str.append( QString::number( (int) b ) );
	
	if (isMetaDataWanted(item))
	{
		b = TRUE;
	}
	else
	{
		b = FALSE;
	}
	//append metadatawanted
	str.append( QString::number( (int) b ) ) ;
			
	kdDebug() << "KCLuceneIndexerDaemon::getConfigNumber: " << str <<endl;
	return str;
}

//guess if the indexing of doc was done differently as now wished by the user
bool KCLuceneIndexerDaemon::hasConfigChanged(Document & doc,KFileItem * item)
{
	//get magic number
	//get modtime integer string
	// Document::get clucene api expects a char * and lucene api a java string
	//QString(const char *) makes a latin1 cast. USe 
	//FIXME TODO non latin ? UTF8 unicode support
	QString results;
	results.append( QString::fromUcs2( (const unsigned short  *) doc.get(_T("ConfigNumber") ) ));
	if (results == getConfigNumber(item))
		return FALSE;
	else
		return TRUE;
}

// yes if the Mod.Time of the index-item is not the Mod.Time of the url it points to
bool KCLuceneIndexerDaemon::hasModTimeChanged( Document & doc,  KFileItem * item)
{
	kdDebug() << "KCLuceneIndexerDaemon::hasModTimeChanged(() START:"  <<endl;

	QString results;
	//get modtime integer string
	// Document::get clucene api expects a char * and lucene api a java string
	//QString(const char *) makes a latin1 cast. USe 
	//FIXME TODO non latin ? UTF8 unicode support
	results.append( QString::fromUcs2((const unsigned short *) doc.get(_T("ModTimeIntKey") )  )); 
			
	//compare item time and stored time
	uint oldModTime    = results.toUInt();
	uint actualModTime = item->time(KIO::UDS_MODIFICATION_TIME);
			
	if (actualModTime != oldModTime )
	{
		kdDebug() << "KCLuceneIndexerDaemon::hasModTimeChanged CHANGED : " << oldModTime << " | " <<actualModTime << endl;	
		return TRUE;
	}
	else 
	{
		kdDebug() << "KCLuceneIndexerDaemon::hasModTimeChanged NOT CHANGED : " << actualModTime<< endl;	
		return FALSE;
	}
}

// deletes index-entries if they must be updated.
//return yes to update ? if the Mod.Time of the index-item is not the Mod.Time of the url it points to
bool KCLuceneIndexerDaemon::updateEntry( KFileItem * item)
{
	bool isEntryUpToDate;
	//URLKEY is a keyword, so no need to bracket the "query"
	QString query =URLToURLKey (item->url() );
	
	kdDebug()<< "KCLuceneIndexerDaemon::updateEntry() CALLLLLLLLLLLLLLLLLLLLLLLLLLED" << query << endl;

	//parse query  see http://java2.5341.com/msg/95918.html
	TCHAR * input = (wchar_t*) query.ucs2();
		
	Term * t = new Term(_T("URLKey"), input);
	
	lucene::search::Query * q = new lucene::search::TermQuery(t);
		
	const TCHAR* buf;
	buf = q->toString(_T("contents"));
	kdDebug() << "KCLuceneIndexerDaemon::updateEntry()" << buf << endl;
	delete[] buf;
	
	lucene::search::Hits * h = m_idSearcher->search(q);
	delete &t;
	delete &q;

	int n = h->length();

	if (n>0)
	{
		//entry found
		kdDebug() << "KCLuceneIndexerDaemon::checkUpdateDoc SAME URL ALREADY INDEXED:"  << endl;
		
		//FIXME look at the first result only !!
		lucene::document::Document & doc = h->doc(0);
		kdDebug() << "KCLuceneIndexerDaemon::updateEntry() after doc "  <<  n << endl;

		bool deleteEntry = FALSE;
		
		//check Mod.Time	
		if (hasModTimeChanged(doc, item)) 
		{
			//delete Hits entries
			deleteEntry = TRUE;
			isEntryUpToDate =FALSE;
		}
		//FIXME check config changed (cannot check for change of converters) ""
		else if (hasConfigChanged(doc, item))
		{
			//delete Hits entries
			deleteEntry = TRUE;
			isEntryUpToDate =FALSE;
		}
		else
		{
			kdDebug() << "KCLuceneIndexerDaemon::checkUpdateDoc Doc HAS NOT BEEn MODIFIED" <<endl;	
			deleteEntry = FALSE;
			isEntryUpToDate =TRUE;
		}
		
		//really delete now
		if ( deleteEntry == TRUE )
		{
			for ( int32_t i=0;i<h->length();i++ )
			{
				kdDebug() << "KCLuceneIndexerDaemon::checkUpdateDoc DELETE item " << i << endl;	
				m_idReader->deleteDoc(h->id(i));
			}
		}

	}
	else
	{
		kdDebug() << "KCLuceneIndexerDaemon::checkUpdateDoc Doc NOT FOUND, DO UPDATE" <<endl;
		isEntryUpToDate =FALSE;
	}
	delete &h;
	return (!isEntryUpToDate);
}

//should not be used. It doesn't work
// Hits & KCLuceneIndexerDaemon::fromURLToIndexHits(QString urlString)
// {
// 	//parse query  see http://java2.5341.com/msg/95918.html
// 	TCHAR * input = (char*) (const char*) (urlString).latin1();
// 		
// 	Term * t = new Term("URLKey", input);
// 	lucene::search::Query * q = new TermQuery(*t);
// 		
// 	const TCHAR* buf;
// 	buf = q->toString(_T("contents"));
// 	kdDebug() << "KCLuceneIndexerDaemon::fromURLToIndexHits()" << buf << endl;
// 	
// 	//FIXME: does it cost time ???
// 	static Hits & h = m_idSearcher->search(*q);
// 	delete[] buf;
// 	delete t;
// // 	delete q;
// 	return h;
// }

QString KCLuceneIndexerDaemon::URLToURLKey(const QString &URLstr)
{
	QString URLKey = "\"" + m_mediaType + "/" + m_mediaName + "/" +URLstr + "\"";
	return URLKey;
}

QString KCLuceneIndexerDaemon::URLKeyToURL(const QString &URLKeyStr)
{
	QStringList docList = QStringList::split("/",URLKeyStr,FALSE);
			//FIXME check the list is long enough: it MUST BE
	QString removeStr = docList.first() + "/" + *(docList.at(1)) + "/";
	QString docURLAndMedia = URLKeyStr;
	docURLAndMedia.remove(0,removeStr.length());
	return docURLAndMedia;
}


QString KCLuceneIndexerDaemon::URLToURLKey(const KURL & URL)
{
	QString URLKey = m_mediaType + "/" + m_mediaName + "/" + URL.pathOrURL();
	return URLKey;
}

//REMOVE ALL funny HACK to index digikamtags
void KCLuceneIndexerDaemon::indexDigikamQMap()
{
	kdDebug()<< "KCLuceneIndexerDaemon::indexDigikamQMap() CALLED SIIIZE" << m_digikamListTags.count() << endl;

	QStringList::Iterator it2 =  m_digikamListTags.begin();
	QStringList::Iterator it1;
	
	for ( it1 = m_digikamListURLs.begin(); it1 != m_digikamListURLs.end(); ++it1 )
	{
		QString query =URLToURLKey (*it1 );
	
		kdDebug()<< "KCLuceneIndexerDaemon::indexDigikamQMap() CALLED" << *it2 << endl;

		//parse query  see http://java2.5341.com/msg/95918.html
		TCHAR * input = (wchar_t *) query.ucs2();
		
		Term * t = new Term(_T("URLKey"), input);
		lucene::search::Query * q = new lucene::search::TermQuery(t);
		
		const TCHAR* buf;
		buf = q->toString(_T("contents"));
		kdDebug() << "KCLuceneIndexerDaemon::indexDigikamQMap()" << buf << endl;
		delete[] buf;
	
		lucene::search::Hits * h = m_idSearcher->search(q);
		delete t;
		delete q;
		
		
		int n = h->length();
		
		if (n>0)
		{
			//entry found
			//FIXME look at the first result only !!
			lucene::document::Document & doc = h->doc(0);
			QString field("digikam/");
			field.append(*it2);
			doc.add(* Field::Text(_T("digikamtags"), (TCHAR *) field.ucs2() ) );
			m_writer->addDocument( &doc );
			kdDebug() << "KCLuceneIndexerDaemon::indexDigikamQMap(): FIELD "  << field << endl;

			delete &doc;
			delete &h;
		}
		else
		{
			//FIXME:leak ? 
// 			delete &h;

		}
		
		it2++;
	}
	
	m_digikamListURLs.clear();
	m_digikamListTags.clear();

}

//this is the scan Loop. It is completely BLOCKING and RECURSIVE.
//but I could emit some signals from time to time and send some statistics.
void KCLuceneIndexerDaemon::indexingLoopOverItems( KFileItemList & itemsList, bool updateMode, bool cleanUp , QString visualPathName)
{
	//----------------------- "NON-RECURSIVE" PART ----------------------------//
	
	kdDebug() << "KCLuceneIndexerDaemon::indexingLoopOverItems START with SIZE:" <<  itemsList.count() <<endl;

	//this a list of directories taken from the input itemsList.
	KFileItemList dirItemsList;
	dirItemsList.setAutoDelete(TRUE);
	dirItemsList.clear();
	
	//this a list of archives taken from the input itemsList.
	KFileItemList archiveItems;
	archiveItems.setAutoDelete(TRUE);
	archiveItems.clear();
	
	
	itemsList.setAutoDelete(TRUE);
	
	
	//multiuse item
	KFileItem * itemURL;
	
	//main loop over all itemsList items. It is not recursive, and will index the items if ok
// 	QPtrListIterator<KFileItem> it( itemsList);
	while ( (itemURL = itemsList.first() )  != 0)
	{
		itemURL = itemsList.first();
		kdDebug() << "KCLuceneIndexerDaemon::indexingLoopOverItems. inLoop:" <<  itemURL->url() << "   itemLists size : "<< itemsList.count() << endl;
		

		
		//check if URL is not user or system excluded 
		//in particular exclude "." and ".." urls
		bool excludeURL = isURLExcluded(itemURL);

		if (excludeURL)
		{
			itemsList.remove();
// 			itemURL = it.current();
		}		
		else 
		{
			
		//check for directories, if it is allowed to recurse in it.. and add to list
		//it always allow recursion inside archive exept if user input ?
		if ( (!itemURL->isLink() ) && itemURL->isDir() )
		{
			//check if user allows to list into this dir and add to dirslist
			if ( isIndexingDirContentAllowed(itemURL , cleanUp) )
			{
				kdDebug() << "KCLuceneIndexerDaemon::indexingLoopOverItemsDIR FOUND" <<endl;
				dirItemsList.append(new KFileItem(*itemURL));
			}
			
		}

			
		//check if it is an archive and check for indexing into archives here
		//or if file type is based on archives
		URLDispatcher * l_dispatcher = new URLDispatcher();
		CLuceneArchType aType=l_dispatcher->findArchiveType(itemURL->mimetype());
		
		if  ( ( !(aType == UNKNOWN_FORMAT) ) && m_insideArchive )
		{
			//create a new item for the archive
			KFileItem * item = new  KFileItem(*itemURL);
			
			QString urlString;
			//FIXME isLocal ??
			if ((itemURL->url().protocol() == "file" ))
			{
				//get the new tar:lalala URI if archive is local
				urlString = l_dispatcher-> archiveFactory(aType  , item->url().path()) ;
				item->setURL(urlString);
				archiveItems.append( new KFileItem(*item) );
				
// 				KURL u_t = itemURL->url(); //the old url
// 				u_t.setRef(item->url().protocol() + ":/");
// 				archiveItems.current()->setExtraData ("realURL", new KURL(u_t) );
				delete item;
				//check if it is ok too
// 				archiveItems.append( item );	
			}
			else    // itemURL is not LOCAL
			{
				
				
				kdDebug() <<"KCLuceneIndexerDaemon::ARCHIVE IS A REMOTE ARCHIVE" <<endl;
				delete item;
// 				if (!((KURL*)itemURL->extraData("realURL")))
// 				{
// 					kdDebug() <<"KCLuceneIndexerDaemon::ARCHIVE IS NOT A SUB ARCHIVE" <<endl;
// 					
// 					//the archive is not a SUBARCHIVE
// 					//set its real URL now
// 					item->setExtraData ("realURL" , new KURL(item->url()) );
// 				}
// 				else
// 				{
// 					kdDebug() <<"KCLuceneIndexerDaemon::ARCHIVE IS A SUB ARCHIVE" <<endl;
// 					
// 					//the archive is A SUBARCHIVE
// 					KURL u_t = *(KURL*) itemURL->extraData("realURL") ;
// 					u_t.addPath( item->url().fileName() );
// 					item->setExtraData ("realURL" , new KURL(u_t));
// 				}
// 				//add a tmpFile to the item
// 				QFileInfo fi( item->url().pathOrURL() );
// 				QString fileExtension	= fi.extension(TRUE); 
// 				//FIXME could it already be a tempFile ???
// 				item->setExtraData ("tempFile", new  KTempFile(QString::null,"."+fileExtension,0600) );
// 				((KTempFile * ) item->extraData("tempFile"))->setAutoDelete(FALSE);
// 				((KTempFile * ) item->extraData("tempFile"))->close();
// 				//copy archive to local tmpfile
// 				//FIXME check for success
// 				KIO::FileCopyJob *job =  KIO::file_copy (
// 						item->url(),
// 				((KTempFile *) item->extraData("tempFile"))->name(),-1,TRUE, FALSE,FALSE );
// 				if (! KIO::NetAccess::synchronousRun( job, 0 ) )
// 				{
// 					kdDebug() <<"KCLuceneIndexerDaemon::REMOTE ARCHIVE COPY INPUT COPY FAILED I will NOT INDEX the ARCHIVE" <<endl;
// 					((KTempFile * ) item->extraData("tempFile"))->unlink();
// 					delete ((KTempFile * )item->extraData("tempFile"));
// 					delete ((KURL *)item->extraData("realURL"));
// 					delete item;
// 				}
// 				else
// 				{
// 																		//get the new URLtar:lalala from the tempURL
// 					urlString = l_dispatcher-> archiveFactory(aType, 
// 							(*((KURL*)item->extraData("realURL"))).path()) ;
// 					item->setURL(urlString);
// 					((KURL*)item->extraData("realURL"))->setRef(item->url().protocol() +":/");
// 					kdDebug() <<"ARCHIVE-REALURL IS" << *((KURL*)item->extraData("realURL")) << endl;
// 
// 					//FIXME: do a refresh now or later ?
// 					//FIXME leaks, what to delete ??
// 					archiveItems.append(item );	
// 					//archiveItems.append( (QString) urlString);	
// 				}
			}
		}
			//cleanup
			delete l_dispatcher;

			
			bool itemIsToBeIndexed;
			//second check if we want to look at this file at all (execpt archives, see above)
			if (isFileTypeWanted(itemURL))
			{	
				//then check if this file ist to be updated or not
				if ( updateMode)
				{
					//then do the update checks on Mod.Time
					itemIsToBeIndexed =  updateEntry(itemURL);
				}
				else
				{
					itemIsToBeIndexed = TRUE;
				}
			}
			else
			{
				//we don't want this item
				itemIsToBeIndexed = FALSE;
			}
			
			if (!itemIsToBeIndexed)
			{
				itemsList.remove();
// 				itemURL = it.current();
			}
			//we check if one can work with this doc as it is or if we need to do preprocessing
			//TODO TODO TODO
/*			else if ( isMultipleDocFile(itemURL) )
			{
				//we can't know what is inside, we should always copy it and see
				//but this is not good. Let make a test
				if ( isContentWanted(itemURL) )
				{
					itemIsToBeIndexed = FALSE;
					//run the preprocessor
					runPreprocessor(itemURL);
					//remove the item ?
					itemsList.remove();		
				}
			}*/
				//now, is it local or remote ?
				//if remote, we will copy it in the indexEntry routine. FIXME: Should be done here in fact
				//but there is a problem if the file is contqaining other files, 
				//we will ask the preprocessor to create an archive.
			if (itemIsToBeIndexed)
			{
				//we want to index this file.
				m_countEntriesToIndex++;
				//maybe slow to recreate the list of items here, but it must survive
				//recursion and be accessible from recursived calls. But it should already
				m_itemsToIndex.append(new KFileItem(*itemURL));
				kdDebug() << "KCLuceneIndexerDaemon::indexingLoopOverItems INDEX url ADDED FOR INDEXING" << itemURL->url() << endl;

				//delete item
				kdDebug() << "KCLuceneIndexerDaemon::indexingLoopOverItems  remove:" << itemURL->url() << "   SIZE: " << itemsList.count() <<endl;
				itemsList.remove();

				
	// 			itemURL = it.current();
			
				
				//really do the indexing in chunks of m_maxEntriesToIndex entries
				//we can't delete and write at the same time
				if (m_countEntriesToIndex > m_maxEntriesToIndex)
				{
					kdDebug()<< "KCLuceneIndexerDaemon::indexingLoopOverItems() WRITTING START: COUNTER :" << m_countEntriesToIndex << 
							"  MAX: " << m_maxEntriesToIndex <<endl;

					//reset the chunk-counter
					m_countEntriesToIndex = 0;
					 
					if (updateMode)
					{
						closeReaderAndSearcher();
						initIndexWriter(m_d, m_an, FALSE);
					}
					
					indexItems( m_itemsToIndex);
					m_counterRunOptimize++;
					if (m_counterRunOptimize >10)
					{
						m_counterRunOptimize = 0;
						m_writer->optimize();
					}
					m_itemsToIndex.clear();
					
// 					openReaderAndSearcher();
// 					indexDigikamQMap();
// 					closeReaderAndSearcher();

					flushAndCloseIndex(m_writer);
					
					if (updateMode)
					{
						openReaderAndSearcher();
					}
					else
					{
						initIndexWriter(m_d, m_an, FALSE);
					}
				}
			}
		}
		
	}
	
	//------------------------- "RECURSIVE" PART ------------------------------//
	
	//------------------first prepare recursion over archives-------------------
	
	kdDebug()<< "KCLuceneIndexerDaemon::indexingLoopOverItems() ARCHIVES START SIZE: " << archiveItems.count() << endl;
	
	itemURL = archiveItems.first();
	while ( itemURL   != 0 )
	{
		kdDebug()<<"KCLuceneIndexerDaemon::indexingLoopOverItems() ARCH URL: " << 
				itemURL->url() << endl;
		//listdir on archive protocol
		m_tmpUDSEntryList.clear();
		if ( listDir(itemURL->url()) )	
		{
			KIO::UDSEntryList listUDSentries = m_tmpUDSEntryList;
				//build a list of fileItems.
			KFileItemList newFileItemList;
			newFileItemList.setAutoDelete(TRUE);
			newFileItemList.clear();
			listDirEntriesToKFileItemList(listUDSentries, newFileItemList, itemURL->url(), (KURL*)itemURL->extraData("realURL"));
			listUDSentries.clear();

			indexingLoopOverItems( newFileItemList, updateMode, FALSE);
		}
		//archiving of this archive finished
		//check if it is a tmpFile
		if ( (KTempFile*) itemURL->extraData("tmpFile"))
		{
			//close and delete temp file
			( (KTempFile*) itemURL->extraData("tmpFile"))->unlink();
			delete ( (KTempFile*) itemURL->extraData("tmpFile"));
		}
		if ( (KURL*) itemURL->extraData("realURL") )
		{
			delete ((KURL*) itemURL->extraData("realURL"));
		}
		archiveItems.remove();
		itemURL = archiveItems.first();
	}
	
		
	//-------------------------then recursion over selected dirs-------------------
	
	kdDebug()<<"KCLuceneIndexerDaemon::indexingLoopOverItems() DIRECTORY START SIZE: " << 
			dirItemsList.count() << endl;

	//recheck if recursion is allowed
	itemURL = dirItemsList.first();
	while ( itemURL   != 0)
	{
		kdDebug()<<"KCLuceneIndexerDaemon::indexingLoopOverItems() DIRECTORY LOOP: "<< endl;

		m_tmpUDSEntryList.clear();
		bool test = listDir(itemURL->url());
		visualPathName = "";
		if ( test )	
		{
			kdDebug()<<"KCLuceneIndexerDaemon::indexingLoopOverItems() DIRECTORY LISTDIR : "<< itemURL->url() << endl;

			KIO::UDSEntryList listUDSentries = m_tmpUDSEntryList;
			//build a list of fileItems.
			KFileItemList newFileItemList;
			newFileItemList.setAutoDelete(TRUE);

			newFileItemList.clear();
			listDirEntriesToKFileItemList(listUDSentries, newFileItemList, itemURL->url() , (KURL*) itemURL->extraData("realURL"));
			listUDSentries.clear();
			
			//this should give the "visual" parent path name for newFileItemList
			//HACKY because it does not go UP do recreate the complete PATH
			//this will add the same entries multiple times because 
			//one pic will be found at 4 different tags_path. Consequence BAD TASTE :-(
			//This means one need more than just such a quick and dirtry hack
			//one should go in update mode.
			visualPathName.append(itemURL->name() + "/");	
			//digikam HACK. cannot work nicely on update or sync modes
			//to avoid image duplicates one should switch to UPDATE mode
			//one would need a MODE AVOID_DUPLICATES
			if ( itemURL->url().protocol() == "digikamtags")
			{
				//on Write mode, check the digikam map and 0 it after
				buildDigikamQMap(visualPathName, newFileItemList);
				
				indexingLoopOverItems( newFileItemList, updateMode, FALSE, visualPathName);

			}
			else
			{
			
			kdDebug()<<"KCLuceneIndexerDaemon::indexingLoopOverItems() DIRECTORY LIST SIZE : "<< newFileItemList.count() << endl;
			
			kdDebug()<<"KCLuceneIndexerDaemon::indexingLoopOverItems() VISUAL : "<<visualPathName << endl;
			indexingLoopOverItems( newFileItemList, updateMode, FALSE, visualPathName);
					    
			kdDebug() << "KCLuceneIndexerDaemon::indexingLoopOverItems: A RECURSION JUST FINISHED...:" << endl;
			}
		}
		//FIXME should be tmpDir
		//this deletes all data in the dir
		if (itemURL->extraData("tmpFile"))
		{
			( (KTempFile*) itemURL->extraData("tmpFile"))->unlink();
			delete (KTempFile*) itemURL->extraData("tmpFile");
		}
		if (itemURL->extraData("realURL"))
		{
			delete ((KURL*) itemURL->extraData("realURL"));
		}
		dirItemsList.remove();
		itemURL = dirItemsList.first();
		kdDebug() << "KCLuceneIndexerDaemon::indexingLoopOverItems: REMOVED DIR.. . ...:" << endl;
	}	
	
	
	//-----------------------------final cleanup and indexing pass------------------
	
	//cleanup is true only from inside the first call to indexingLoopOverItems.
	//Being here, means we are getting close to the end
	if ( cleanUp)
	{
		kdDebug()<<"KCLuceneIndexerDaemon::indexingLoopOverItems() CLEANUP entries: "<< m_countEntriesToIndex << endl;
		
		if (m_countEntriesToIndex >= 0)
		{
			m_countEntriesToIndex = 0;
			//only allow to write those urls. No recursion.
			if (updateMode)
			{
				closeReaderAndSearcher();
				initIndexWriter(m_d, m_an, FALSE);
			}

			indexItems( m_itemsToIndex );
			m_itemsToIndex.clear();

			m_writer->optimize();
			flushAndCloseIndex(m_writer);
		}
		else 	if (updateMode)
		{
			closeReaderAndSearcher();
		}
	}
	kdDebug() << "KCLuceneIndexerDaemon::END OF SUBRECURSION:"   << endl;
}

void KCLuceneIndexerDaemon::openReaderAndSearcher()
{
	//init indexReader
	if (m_idReader == 0)
		m_idReader = IndexReader::open(m_d, FALSE);
	else
	{
		kdDebug() << "KCLuceneIndexerDaemon::openReaderAndSearcher READER             EEEEEEEEEEEEEERRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR"   << endl;
	}
		
	//create searcher
	if ( m_idSearcher== 0)
	{
		m_idSearcher = new lucene::search::IndexSearcher(m_idReader) ;
	}
	else
	{
		kdDebug() << "KCLuceneIndexerDaemon::openReaderAndSearcher WRITER             EEEEEEEEEEEEEERRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR"   << endl;
	}
}

void KCLuceneIndexerDaemon::closeReaderAndSearcher()
{
	//close reader and searchers
	kdDebug()<<"KCLuceneIndexerDaemon::closeReaderAndSearcher FIXME: close reader out of order"<<endl;

	m_idSearcher->close();
	delete m_idSearcher;
	m_idSearcher = 0;
	
	m_idReader->close();
	delete m_idReader;
	m_idReader = 0;
	
	//closing reader closes all index files. So reopen a new directory ?
// 	m_d->close();
// 	delete m_d;
	initIndexDirectory(m_indexLocation, FALSE);

}

bool KCLuceneIndexerDaemon::isFileTypeWanted(KFileItem * item)
{
	bool b;
	if (isMetaDataWanted(item))
	{
		b = TRUE;
	}
	else if (item->isReadable())
	{
		if (isContentWanted(item))	
			b = TRUE;
		else
			b = FALSE;
	}
	kdDebug()<<"KCLuceneIndexerDaemon::isFileTypeWanted" << b <<endl;
	return b;
}

//FIXME: BASIC answer. Does not check for existence
bool KCLuceneIndexerDaemon::isMetaDataWanted(KFileItem * /*item*/)
{
	if ( m_allMetaData )
	{
		//even if no KInfoPlugin exists, we want at least the file name and so on.
		return TRUE;
	}
	else
	{
		//the test on converters will tell what we want
		// m_restrictMetaData have no influence here
		return FALSE;
	}
}

bool KCLuceneIndexerDaemon::isContentWanted(KFileItem * item)
{
	bool b;
	if (m_content)
	{
		b =  hasFileTypeAConverter(item);
	}
	else
		b = FALSE;
	kdDebug()<<"KCLuceneIndexerDaemon::isContentWanted: " << b <<endl;

	return b;
}

//TODO: build a QMap mimetype <-> convewrter
bool KCLuceneIndexerDaemon::hasFileTypeAConverter(KFileItem * item)
{
	QString mimeType = item->mimetype();
	
	kdDebug()<<"KCLuceneIndexerDaemon::hasFileTypeAConverter " << mimeType <<endl;

	if (m_readerHtml)
	{
		kdDebug()<<"KCLuceneIndexerDaemon::hasFileTypeAConverter m_readerHtml" << m_readerHtml <<endl;

		if ( mimeType.startsWith("text/html"))
			return  TRUE;
	}
	if (m_readerXml)
	{	
		if ( mimeType.startsWith("application/xml"))
			return TRUE;
	}
	if (m_readerHtml)
	{
		if ( mimeType.startsWith("text/xhtml+html"))
			return  TRUE;
	}
	if (m_readerText)
	{
		kdDebug()<<"KCLuceneIndexerDaemon::hasFileTypeAConverter m_readerText" << m_readerText <<endl;

		if ( mimeType.startsWith("text/"))
			return  TRUE;
	}
	if (hasExternalConverter(item))
	{
		return TRUE;
	}
	
	return FALSE;
}

bool KCLuceneIndexerDaemon::hasExternalConverter(KFileItem * item)
{
	kdDebug() << "KCLuceneIndexerDaemon::hasExternalConverter() START"   << endl;

	QFileInfo fi( item->url().pathOrURL() );
	QString l_extension	= fi.extension(TRUE);  //returns tar.gz FALSE would return gz
	int mimePos 		= m_convert->mimePos(item->mimetype());
	int patternPos 		= m_convert->patternPos(l_extension); //parse filename inside function
	
	if ((patternPos <0) && (mimePos <0))
	{
		kdDebug() << "KCLuceneIndexerDaemon::hasExternalConverter() : NO"   << endl;
		return FALSE;
	}
	else 
	{
		kdDebug() << "KCLuceneIndexerDaemon::hasExternalConverter() : YES"   << endl;
		return TRUE;
	}
}

//from kdelibs/kio/tests/jobtest and ktartest
void KCLuceneIndexerDaemon::slotEntries( KIO::Job*, const KIO::UDSEntryList& lst )
{
	m_tmpUDSEntryList = lst;
}

bool KCLuceneIndexerDaemon::listDir(KURL u)
{
	u.adjustPath(+1);
	
	kdDebug() << "KCLuceneIndexerDaemon::indexDocs AFTERJOB CONNECT:"  << u.url() << endl;
	kdDebug() << "KCLuceneIndexerDaemon::indexDocs AFTER LISTDIRJOB INIT:"  << u.path() << endl;

	KIO::ListJob* job = KIO::listDir( u , false , true);
	connect( job,  SIGNAL( entries( KIO::Job*, const KIO::UDSEntryList& ) ), 
		 this, SLOT( slotEntries( KIO::Job*, const KIO::UDSEntryList& ) ) );

	bool ok = KIO::NetAccess::synchronousRun( job, 0 );
	return ok;
}

void KCLuceneIndexerDaemon::indexingLoopOverIndexEntries()
{
	int maxDocNumber = m_idReader->maxDoc();
	int currentDocNumber = 0;
	int countEntriesToIndex = 0;
// 	m_itemsToIndex.clear();
	
	while ( currentDocNumber < maxDocNumber )
	{
		//check Doc if not deleted
		KFileItem * item = 0;
		if (! m_idReader->isDeleted(currentDocNumber))
		{
			Document & doc = *m_idReader->document(currentDocNumber);
			QString docURLAndMedia = QString::fromUcs2( (const unsigned short  *) doc.get(_T("URLKey")));
			QString docURL = URLKeyToURL(docURLAndMedia);
			kdDebug() << "indexingLoopOverIndexEntries:        URLKEY : " << docURLAndMedia << endl;                
			kdDebug() << "indexingLoopOverIndexEntries: parsed URLKEY :" << docURL << endl;                

			//FIXME:how to get the item back to me ?
			
			
			bool sync;
			KURL u = KURL::fromPathOrURL(docURL);
			m_entry.clear();
			bool itemExists = KIO::NetAccess::stat(u ,m_entry,0 );
			kdDebug() << "KCLuceneIndexerDaemon::syncEntry STATURL :" << u <<endl;
			if (!itemExists)
			{
				item = 0;
				kdDebug() << "KCLuceneIndexerDaemon::syncEntry URL NOT FOUND"  <<endl;
				//we are in sync mode and stat failed! Remove this entry from index
				m_idReader->deleteDoc(currentDocNumber);
				delete &doc;
				sync = FALSE;
			}
			else
			{
				//item exists
				//I do not need the Mimetyes
				item = new KFileItem(m_entry , u , FALSE , FALSE); 
				kdDebug() << "KCLuceneIndexerDaemon::syncEntry URL FOUND:" << item->url() <<endl;
				//check if mod time changed XXX
				if (hasModTimeChanged(doc, item)) 
				{
					delete &doc;	
					m_idReader->deleteDoc(currentDocNumber);
					sync = TRUE;
				}
				else if ( hasConfigChanged(doc, item) )
				{
					delete &doc;	
					m_idReader->deleteDoc(currentDocNumber);
					sync = TRUE;
				}
				else
				{
					delete &doc;
					sync = FALSE;
					delete item;
				}
			}
			if ( sync )
			{
				//FIXME syncEntryy should return the KFILEITEN and you should had it to m_UDSentriesToUpdate
					m_itemsToIndex.append(new KFileItem(*item));
					delete item;
					countEntriesToIndex++;
			}
			
			if ( countEntriesToIndex > m_maxEntriesToIndex)
			{
				countEntriesToIndex = 0;
				
				//close reader and searchers
				closeReaderAndSearcher();
					
				//init writer
				initIndexWriter(m_d, m_an, FALSE);
					
				//only allow to write those urls. no cleanup here
				indexItems(m_itemsToIndex);
				m_itemsToIndex.clear();
					
				//close writer
				flushAndCloseIndex(m_writer);
				//reopen an indexReader and searcher
				openReaderAndSearcher();
			
				//recalc maxDocNumber
				maxDocNumber = m_idReader->maxDoc();
			}	
			
		}
		currentDocNumber++;
	}
	//write remainig entries
	if ( countEntriesToIndex >= 0)
	{
		countEntriesToIndex = 0;
		
		closeReaderAndSearcher();
		initIndexWriter(m_d, m_an, FALSE);
	
		//only allow to write those urls. cleanup
		indexItems(m_itemsToIndex);
		m_itemsToIndex.clear();
		m_writer->optimize();

		//close writer
		flushAndCloseIndex(m_writer);
		

	}
	
	//FIXME: for ARCHIVES, one WOULD LIKE TO CHECK THE ROOT OF THE ARCHIVE FOR CHANGES
}

//dcop call here. Inits everything.
int KCLuceneIndexerDaemon::deleteHits(
					const QString & callingClientId,
					const QString & indexName,
					const QString & indexLocation,
					const QString &	query
				     )
{
	m_callingClient = 	callingClientId;
	m_indexName	=	indexName;
	m_indexLocation	=	indexLocation;
	m_query		= 	query;
	
	emit deleteHitsCallReceived();
	return 1;
}

//not yet implemented
void KCLuceneIndexerDaemon::startDeleteHits()
{
}

QString KCLuceneIndexerDaemon::stripMetaData(QString str)
{
	QString myStrip;
	myStrip = str.stripWhiteSpace();
	myStrip = myStrip.lower();
	myStrip = myStrip.replace(" ","_");
	myStrip = myStrip.replace(":","_");
	myStrip = myStrip.replace("/","_");
	kdDebug() << "KCLuceneIndexerDaemon::stripMetaData() FINISHED"  << str << " | " << myStrip << endl;

	return myStrip;
}


void KCLuceneIndexerDaemon::builtinReader(KFileItem * item)
{
	kdDebug() << "KCluceneIndexerDaemon:: builtinReader INDEXING:"  <<  item->url().path() << endl;
	
	m_doc->add( * Field::Text(_T("contents"),new FileReader( (char*) (const char*) item->url().path() ) ) );
}

bool KCLuceneIndexerDaemon::getContentToTmpFile(KFileItem * item, KTempFile * tmpFile)
{
	kdDebug() << "KCLuceneIndexerDaemon::getContentToTmpFile CALLED:"  << item->url() << endl;

	// to be converted file must be local or USE a KAT IOSLAVE but they might kill memory
	// assume file is local.
		
	//prepare tmp file
// 	tmpFile = new KTempFile(QString::null,".txt",0600);
	tmpFile->setAutoDelete(FALSE);
	if (! (tmpFile->status() == 0) ) 
	{
		kdDebug() <<"OUPTPUT TMP FILE CREATION FAILED" << endl;
		return FALSE;
	}
	tmpFile->close();//ready for transfer
	
	//decide if MimeType or Pattern based conversion mode will be used
	//use path-> must be local file !
	QFileInfo fi( item->url().path() );
	QString l_extension	= fi.extension(TRUE);  //returns tar.gz FALSE would return gz
	int mimePos 		= m_convert->mimePos(item->mimetype());
	int patternPos 		= m_convert->patternPos(l_extension); //parse filename inside function
	int l_pos;
	bool isMime;
	
	//prefer PATTERN MATCHING OVER OF MIMETYPE MATCHING: DANGEROUS
	if ((patternPos <0) && (mimePos <0))
		return FALSE;
	else if (patternPos > -1)
		isMime=FALSE;
	else 
		isMime=TRUE;
	if (isMime)
		l_pos = mimePos;
	else
		l_pos = patternPos; 
	
	//determine if one can set the converted file URL to the URL of the tmpFile
	//if not, one will need to copy the file again.
	bool acceptTmpFile 	= m_convert->acceptOutputTmpFile(isMime , l_pos);

	//some needed vaiables
	QString l_proc;
	QString l_args;
	KTempFile * 	l_tmpInputFile;
	QString 	l_tmpOutputFileString;
	

	//$$o found in args
	if (acceptTmpFile)
	{
		l_proc = m_convert->getProc(isMime , l_pos);
		l_args = m_convert->getArgs(isMime , l_pos , item->url().path() ,tmpFile->name());
	}
	else
	{
		//get local copy and tmp file with the correct file extension
		QFileInfo fi( item->url().pathOrURL() );
		QString fileExtension	= fi.extension(TRUE); 
		l_tmpInputFile = new KTempFile(QString::null,"."+ fileExtension,0600); 
		l_tmpInputFile->setAutoDelete(FALSE);		
		l_tmpInputFile->close();
		//FIXME check for success
		KIO::FileCopyJob *job =  KIO::file_copy (item->url(),l_tmpInputFile->name(),-1,TRUE, FALSE,FALSE );
		if (! KIO::NetAccess::synchronousRun( job, 0 ) )
		{
			kdDebug() <<"KCLuceneIndexerDaemon::getContentToTmpFile $$i INPUT COPY FAILED"  <<endl;
			return FALSE;
		}
		
		//FIXME: SECURITY ?: we have no control on the output file !!!!
		l_tmpOutputFileString = m_convert->getOutput(isMime ,l_pos , l_tmpInputFile->name());
		kdDebug() <<"KCLuceneIndexerDaemon::getContentToTmpFile $$i OUTPUT TMP FILE name " << l_tmpOutputFileString <<endl;

		//setup args
		l_proc = m_convert->getProc(isMime ,l_pos);
		l_args = m_convert->getArgs(isMime ,l_pos, l_tmpInputFile->name());
	}
				

	//now we are ready to convert the file, using a blocking-kprocess	
	kdDebug() <<"startSyncDocToTextConversion: PROC" << l_proc <<endl;
	kdDebug() <<"startSyncDocToTextConversion: ARGS" << l_args <<endl;
	KProcess *proc = new KProcess;
	proc->setUseShell(TRUE);
	*proc << l_proc;
	*proc << l_args;
	connect(proc, SIGNAL(processExited(KProcess *)), this, SLOT(slotSyncTextConversionFinished(KProcess *)));
	
	//if conversion is successfull according to kprocess
	//FIXME: it return success even when proc is not found, useless here
	//check if file is empty would be better
	bool goodConversion = FALSE;
	if ( proc->start(KProcess::Block) )
	{
		//move the tmp output file to a real tmp-file referenced by m_tmpPath.
		if (acceptTmpFile)
		{
			goodConversion = TRUE;
		}
		else
		{
			KURL tmpURL;
			tmpURL.setPath(l_tmpOutputFileString);
			KURL tmpPathURL;
			tmpPathURL.setPath(tmpFile->name());

			KIO::FileCopyJob *job =  KIO::file_move (tmpURL, tmpPathURL ,-1,TRUE, FALSE,FALSE );
			if ( KIO::NetAccess::synchronousRun( job, 0 ))
			{
				goodConversion = TRUE;
			}
		}

	}
	
	//clean a bit.
	if (!acceptTmpFile)
	{
		l_tmpInputFile->unlink();
		delete l_tmpInputFile;
		//leak ??
// 		delete tmpItem; //useless item :-)
	}
	delete proc;
	
	kdDebug() <<"startSyncDocToTextConversion: CONVERSION IS SUCCESS ?" <<goodConversion <<endl;
	return goodConversion;
}


//this get a preview of the fileitem and stores it on disk. 
//Because I don't know how to tell KDE/Konqueror to look for a preview somewhere
//I do tell the preview is the icon. It is better than no preview of files at all
bool KCLuceneIndexerDaemon::doStorePreview( KFileItem * item, KURL & realURL)
{
	kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview  CALLED: " << realURL << endl;

	bool success = FALSE;
	QString pathToPreview;
	if (! (item->mimetype().startsWith("image") ||
		      item->mimetype().startsWith("video") )) 
	{
		//do not run preview
		success = FALSE;
	}
	else
	{
	
	
	//create a path where to store pixmap
	//FIXME: using the name m_indexLocation too, allow to always know where to store 
	//preview, but this means more traffic when sending results: at least the name of
	//
		
	pathToPreview = m_indexLocation + "/." + m_mediaType + "/" +
			m_mediaName + "/" + realURL.pathOrURL() + "_preview.png";
	
	//FIXME: help :One can add the path as extraData to the item, or declare it "globaly"
	//can I create a KFileIconViewItem or something like that ans setExtraData
// 	item->setExtraData("StoredPreview", new QString(pathToPreview));
	//forget if failure to get get preview
	//delete extra data
// 	item->removeExtraData(QString("StoredPreview") );
	
	
	//the file is assumed to be a local file
		KFileItemList items;
		items.append( item );
		KIO::PreviewJob * job = KIO::filePreview( items, 256, 256, 0, 0, FALSE, FALSE, NULL);

		connect( job,  SIGNAL( gotPreview (const KFileItem *, const QPixmap &) ),
			 this, SLOT  ( slotPreviewGot(const KFileItem *, const QPixmap & ) ) );
		
	
		bool ok = KIO::NetAccess::synchronousRun( job, 0 );
		if (ok)
		{
			kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview RUN OK : " << endl;

			if (!m_pixmap.isNull())
			{
				kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview PIXMAP EXISTS: " << endl;

				KURL pathToDirURL = KURL::fromPathOrURL(pathToPreview);
				pathToDirURL.setDirectory(pathToDirURL.directory());
				if (! KIO::NetAccess::exists(pathToDirURL, TRUE, 0))
				{
					kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview DIR DOES NOT EXIST: " << endl;

// 					QDir q(pathToDirURL.pathOrURL());
// 					q.mkdir(pathToDirURL.fileName(TRUE), TRUE);
					KStandardDirs::makeDir(pathToDirURL.pathOrURL());
// 					KIO::NetAccess::mkdir(pathToDirURL,0,-1);
					kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview  MAKKE DIR name"<< pathToDirURL.fileName(TRUE) <<endl;
// 					else
					kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview  MAKKE DIR path"<< pathToDirURL.pathOrURL() << endl;
						
					kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview  : " << pathToDirURL << endl;

				}
				if (KIO::NetAccess::exists(pathToDirURL, TRUE, 0))
				{
					kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview DIR EXISTS: " << endl;

					success = m_pixmap.save( pathToPreview, "PNG", -1);
				}
				m_pixmap.resize(0,0);
			}
		
		}
	}
	if (success)
	{
		m_doc->add( *Field::Keyword(_T( "StoredPreview"), (TCHAR*) pathToPreview.ucs2() ));
		kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview SUCCESS : " << endl;
	}
	else
	{		
		m_doc->add( *Field::Keyword(_T( "StoredPreview"),  _T("empty") ));

		kdDebug() <<"KCLuceneIndexerDaemon::doStorePreview FAILED : " << endl;
	}

	
	//cleanup
	
	return success;
}

void KCLuceneIndexerDaemon::slotPreviewGot(const KFileItem * /*item*/, const QPixmap & pixmap)
{
	kdDebug() <<"KCLuceneIndexerDaemon::slotPreviewGot RealURL : " << endl;

	m_pixmap = pixmap;
}


void KCLuceneIndexerDaemon::doTimesIndexing(KFileItem * item)
{
	//get modification times 
	//TODO HACK FIXME only works for localfiles, good I copy them for fulltext anyway
		//times are read from UDSEntry info, if not found, then from stat but only for local files
	time_t timeT = item->time(KIO::UDS_MODIFICATION_TIME);
			
	QString timeModInt;
	timeModInt = QString::number(timeT );
	QDateTime * qDT = new QDateTime;
	qDT->setTime_t(timeT);
	QString timeMod;
	timeMod = qDT->toString("yyyyMMddhhmm");
	delete qDT;
	kdDebug() << "KCluceneIndexerDaemon::doTimesIndexing  DATE TIME MOD="    << timeMod << endl;
	kdDebug() << "KCluceneIndexerDaemon::doTimesIndexing  DATE TIME INT="    << timeModInt << endl;
			
	// time modification KEY user-searchable
	
	m_doc->add( *Field::Keyword(_T( "dateKey"), (TCHAR*) timeMod.ucs2()) );
// 			same but with tokens ???			
	m_doc->add( *Field::Text(_T( "KeyModtimeText"),   (TCHAR*) timeMod.ucs2()) );
			
// 	time modification KEY used internaly to find mod files when updating
	m_doc->add( *Field::Keyword(_T( "ModTimeIntKey"),  (TCHAR*) timeModInt.ucs2()) );
}

int KCLuceneIndexerDaemon::doURLandMediaAndMimetypeIndexing(KFileItem * item , QString & anyStringExeptContent)
{
	kdDebug() << "KCluceneIndexerDaemon::doURLandMediaAndMimetypeIndexing "  << endl;

	if (item->url().path().isEmpty())
	{
		return -100;
	}
	
	//check if names are empty.
	//QCString m_mediaType and m_mediaName are not allowed to have "/" or be empty
	if (m_mediaType.isEmpty())
	{
		m_mediaType = "unknown";
	}
	if (m_mediaName.isEmpty())
	{
		m_mediaName = "unknown";
	}
	m_mediaType = m_mediaType.replace("/", "_");
	m_mediaName = m_mediaName.replace("/", "_");
	
	QString mediaKey = m_mediaType + "/" + m_mediaName;
	QString URLKey = URLToURLKey( item->url() );
		
	m_doc->add( *Field::Keyword(_T( "URLKey")  , (TCHAR*) URLKey.ucs2() ));
	m_doc->add( *Field::Keyword(_T( "MediaKey"), (TCHAR*) mediaKey.ucs2() ));

	//add configNumber. it is used 
	QString cfgNumber = getConfigNumber(item);
	if (!cfgNumber.isEmpty())
		m_doc->add( *Field::Keyword(_T( "ConfigNumber")  , (TCHAR*)  cfgNumber.ucs2()));
	else
	{
		m_doc->add( *Field::Keyword(_T( "ConfigNumber")  ,   _T("00")));
	}
	
	//CLBU po

	QString recordedURL = "CLBU " + item->url().pathOrURL();
	TCHAR* myTCHAR = (TCHAR*) recordedURL.ucs2();
	TCHAR * myTCHAR2 = new TCHAR ( *myTCHAR);
	
	m_doc->add( *Field::Text(_T("path"), myTCHAR2 ) );
// 	m_doc->add( *Field::Text(_T("path"), (TCHAR*) recordedURL.ucs2() ) );
	//FIXME:is this all necessary ?
	m_doc->add( *Field::Text(_T("mediaType"), (TCHAR*) m_mediaType.ucs2() ) );
	m_doc->add( *Field::Text(_T("mediaName"), (TCHAR*) m_mediaName.ucs2() ) );

	kdDebug() << "KCluceneIndexerDaemon::doURLandMediaAndMimetypeIndexing URLKEY"    << URLKey << endl;

	if (!item->mimetype().isEmpty())
	{
	       QString str = item->mimetype();
		m_doc->add( *Field::Text(_T( "mimetype"), (TCHAR*) str.ucs2() ) );
	}
	else
	{
		m_doc->add( *Field::Text(_T( "mimetype"), _T("unknown") ) );
	}
	
	QString numb = QString::number(item->mode());
	m_doc->add( *Field::Text(_T( "Mode_T"), (TCHAR*) numb.ucs2() ) );
	numb = KIO::number(item->size());
	m_doc->add( *Field::Text(_T( "FileSize"), (TCHAR*) numb.ucs2() ) );
	
	//append some strings to this 
	anyStringExeptContent.append( item->url().pathOrURL() + " " + m_mediaType + " " 
			+ m_mediaName + " " + item->mimetype()  );
	kdDebug() << "KCluceneIndexerDaemon::doURLandMediaAndMimetypeIndexing FINISHED"    << URLKey << endl;
	
	return 0;
}

void KCLuceneIndexerDaemon::slotMetaInfoGot(const KFileItem * item)
{	
	kdDebug() << "KCluceneIndexerDaemon::slotMetaInfoGot "    << item->url() << endl;
	kdDebug() << "KCluceneIndexerDaemon::slotMetaInfoGot "    << item->mimetype() << endl;


	m_metaInfo = TRUE;
}

void KCLuceneIndexerDaemon::slotMetaInfoFailed(const KFileItem * item)
{	
	kdDebug() << "KCluceneIndexerDaemon::slotMetaInfoFailed "    << item->url() << endl;

	m_metaInfo = FALSE;
}

//here this is the only place where metadata should be extracted from KFileitems
//all other function will only read metadata
//is there a difference between using a MetaInfoJob in sync mode and KFileMetaInfo constructor ?
bool KCLuceneIndexerDaemon::metaInfoJob(KFileItem * item)
{
// 	if (item->isLocalFile())
// 	{
		//FIXME: second arg in item->metainfo is IGNORED !!!, always fastest
		if (item->mimetype() == "inode/directory")
		{
			item->setMetaInfo(KFileMetaInfo(item->url(), 
				item->mimetype(), KFileMetaInfo::Fastest) );
		}
		else
		{
			item->setMetaInfo(KFileMetaInfo(item->url(),
				item->mimetype(), KFileMetaInfo::Preferred) );
		}
		if (item->metaInfo(FALSE).isEmpty())
		{
			item->setMetaInfo( KFileMetaInfo()) ;
			kdDebug() << "KCluceneIndexerDaemon::metaInfoJob isValid" << item->metaInfo().isValid() << endl;
			//try again using a job
			KFileItemList l_items;
			l_items.append(item);
			KIO::MetaInfoJob * job = new KIO::MetaInfoJob( l_items , FALSE);
			connect( job,  SIGNAL( gotMetaInfo   (   const KFileItem *     ) ),
			 	this, SLOT  ( slotMetaInfoGot(const KFileItem * ) ) );
			connect( job,  SIGNAL( failed   (   const KFileItem *    ) ), 
			 	this, SLOT  ( slotMetaInfoFailed(const KFileItem *  ) ) );
	
			bool ok = KIO::NetAccess::synchronousRun( job, 0 );
// 			delete job;
			if ( !ok)
				return FALSE;
			else
				if (item->metaInfo(FALSE).isEmpty())
					return FALSE;
				else
					return TRUE;
			}
		else
		{
			kdDebug() << "KCluceneIndexerDaemon::metaInfoJob SUCCESS" << endl;
			return TRUE;
		}
// 	}
// 	else
// 	{
// 		KFileItemList items;
// 		items.append(item);
	
// 		kdDebug() << "KCLuceneIndexerDaemon::metaInfoJob AFTERJOB CONNECT:"  << item->url() << endl;

// 		KIO::MetaInfoJob * job = new KIO::MetaInfoJob( items , FALSE);
// 		connect( job,  SIGNAL( gotMetaInfo   (   const KFileItem *     ) ),
// 			 this, SLOT  ( slotMetaInfoGot(const KFileItem * ) ) );
// 		connect( job,  SIGNAL( failed   (   const KFileItem *    ) ), 
// 			 this, SLOT  ( slotMetaInfoFailed(const KFileItem *  ) ) );
	
// 		bool ok = KIO::NetAccess::synchronousRun( job, 0 );
// 		delete job;
// 		return ok;
// 	}
}

void KCLuceneIndexerDaemon::doContentIndexing(KFileItem * item)
{
	kdDebug() << "KCluceneIndexerDaemon::doContentIndexing "    << item->url() << endl;

	//convert item to a text file or html file or whatever is good for clucene
	if (hasExternalConverter(item))
	{
		kdDebug() << "KCluceneIndexerDaemon::doContentIndexing  HAS EXTENAL ONE"    << endl;

		KTempFile * outputTmpFile;
		outputTmpFile = new KTempFile(QString::null,".txt",0600);
		getContentToTmpFile(item,  outputTmpFile);

		//get path of the new tmp text-file
		const char* f =  (char*) (const char*) outputTmpFile->name();
		m_doc->add( *Field::Text(_T("contents"),new FileReader(f)) );
	
		//cleanup
		outputTmpFile->unlink();
		delete outputTmpFile;
		kdDebug() << "MYDOC::::doIndexing slotWriteDoc AFTER del tmp :"  << endl;
	
	}
	//use LUCENE internal converter if exists
	else if ( item->mimetype().startsWith("text/html"))
	{
		if (m_readerHtml)
		{
			this->builtinReader(item);
			kdDebug() << "KCluceneIndexerDaemon::doContentIndexing m_readerHtml :"  << m_readerHtml << endl;
			return;
		}
	}
	else if ( item->mimetype().startsWith("application/xml"))
	{
		if (m_readerXml)
		{
			this->builtinReader(item);
			kdDebug() << "KCluceneIndexerDaemon::doContentIndexingm_readerXml:"  << m_readerXml<< endl;
			return;
		}
	}
	else if ( item->mimetype().startsWith("text/xhtml+html"))
	{
		if (m_readerHtml)
		{
			this->builtinReader(item);
			kdDebug() << "KCluceneIndexerDaemon::doContentIndexing m_readerHtmlXHTML:"  << m_readerHtml << endl;
			return;
		}
	}
	else if ( item->mimetype().startsWith("text/"))
	{
		if (m_readerText)
		{
			this->builtinReader(item);
			kdDebug() << "KCluceneIndexerDaemon::doContentIndexing m_readerText:"  <<  m_readerText<< endl;
			return;
		}
	}
}

int KCLuceneIndexerDaemon::doAllMetaDataIndexing( KFileItem * item, QString & anyStringExeptContent)
{
	kdDebug() << "KCluceneIndexerDaemon:: doAllMetaDataIndexing() CALLED"  << endl;
	KFileMetaInfo info;
	if (item->mimetype() == "inode/directory")
	{
		info = KFileMetaInfo(item->url(),item->mimetype(),KFileMetaInfo::Fastest) ;
	}
	else
	{
// 		info = KFileMetaInfo(item->url(),item->mimetype(), KFileMetaInfo::DontCare);
		info = KFileMetaInfo(item->url(),item->mimetype(), KFileMetaInfo::Everything);

	}

	bool doAll;
	if (info.isEmpty())
	{
		//TODO FIXME: metadata are not extracted when inside a ZIP archive 
		//this is the case of mimetype too, but with mimeTypeJob, I can get the good mime
		//I get them by copying the file as it is done when content indexing is required
		//when a URL is not local, one should always copy its content.	
		kdDebug() << "KCluceneIndexerDaemon::doAllMetaDataIndexing first trial has FAILED" << endl;
		kdDebug() << "KCluceneIndexerDaemon::doAllMetaDataIndexingfirst trial has FAILED" << item->mimetype() << endl;

			//reset to invalid
			item->setMetaInfo( KFileMetaInfo()) ;
			kdDebug() << "KCluceneIndexerDaemon::doAllMetaDataIndexing first trial has FAILED" << item->mimetype() << endl;

			//try again using a job
			KFileItemList l_items;
			l_items.append(item);
			KIO::MetaInfoJob * job = new KIO::MetaInfoJob( l_items , FALSE);
			connect( job,  SIGNAL( gotMetaInfo   (   const KFileItem *     ) ),
			 	this, SLOT  ( slotMetaInfoGot(const KFileItem * ) ) );
			connect( job,  SIGNAL( failed   (   const KFileItem *    ) ), 
			 	this, SLOT  ( slotMetaInfoFailed(const KFileItem *  ) ) );
	
			bool ok = KIO::NetAccess::synchronousRun( job, 0 );
			kdDebug() << "KCluceneIndexerDaemon::metaInfoJob doinmgssssss trial OK ? " << ok << endl;

// 			delete job;
			if ( !ok)
				doAll = FALSE;
			else
			{
			 	info  = item->metaInfo(FALSE);
				if (info.isEmpty())
					doAll = FALSE;
				else
				{
					doAll = TRUE;
					
				}
			}
	}
	else
	{
		kdDebug() << "KCluceneIndexerDaemon::metaInfoJob SUCCESS" << endl;
		doAll = TRUE;
	}

	if (doAll)
	{
	QStringList glist = info.groups();  //list of groups
	kdDebug() << "KCluceneIndexerDaemon:: doAllMetaDataIndexing() After GRPOUPO"  << endl;

	for (QStringList::Iterator itG = glist.begin(); itG != glist.end(); ++itG)
	{
		KFileMetaInfoGroup g = info.group(*itG);
		QStringList listKeys = g.keys();

		for (QStringList::Iterator itK = listKeys.begin(); itK != listKeys.end(); ++itK)
		{
			KFileMetaInfoItem item = g.item(*itK);
			if (item.string() )
			{
				
				//FIXME : In the future, don't strip anymore please,
				//so that one can reconstruct the whole metadatainfo plugin,...
				QString myStrip = stripMetaData (*itK );
				
				kdDebug() << "KCLuceneIndexerDaemon:: field        :"  << myStrip << endl;
				kdDebug() << "KCLuceneIndexerDaemon:: field content:"  <<  item.string() << endl;
// 				kdDebug() << "KCLuceneIndexerDaemon:: field content:"  << _T( (char*) (const char*)  myStrip.latin1()) << endl;

				QString cont = item.string().latin1();
				m_doc->add( *Field::Text((TCHAR*) myStrip.ucs2(),  (TCHAR*) cont.ucs2() ) );

				
				anyStringExeptContent.append(*itK + " " + cont);
			}
		}
	}
}
	return 0;
}

//dcop calls here. Inits everything!!!
int KCLuceneIndexerDaemon::doIndex(
		const QString & 		callingClientId,
		const QString &  		indexName,
		const QString &      		indexLocation,
		const bool & 			boolMetaData,		//ok
		const bool &			boolContent,		//ok
		const bool &			boolArchives,		//ok
		const bool &			includeSubFolders,	//TODO
		const bool &			excludeSubFolders,	//TODO
		const QStringList & 		includeItems,		//TODO
		const QStringList & 		excludeItems,		//TODO
		const bool &			addToIndex,		//ok with erase
		const bool &			eraseIndex,		//ok 
		const bool &			eraseMedia,		//ok 
		const bool &			updateIndex,		//TODO
		const bool &			syncIndex,		//TODO
		const QStringList &		tmpMimeList,
		const QStringList &		tmpMimeProc, 
		const QStringList &		tmpMimeArgs,
		const QStringList &		tmpMimeOutput,
		const QStringList &		tmpPatternList,
		const QStringList &		tmpPatternProc,
		const QStringList &		tmpPatternArgs,
		const QStringList &		tmpPatternOutput,
		const bool & 			tmpText,
		const bool & 			tmpHtml,
		const bool & 			tmpXml,
		const bool &			tmpRestrictMetaData,
		const QString &      		mediaType,
		const QString &      		mediaName)
{
	//stop timeOut-timer 
	if ( m_timer->isActive() )
	{
		m_timer->stop();
	}
	
	//get ID of the calling app
	m_callingClient 	= 	callingClientId;
	kdDebug() << "INDEXING SETUP CALLING app        :" << m_callingClient << endl;

	//init variables
	m_indexName		=	indexName;
	m_indexLocation		=	indexLocation;
	m_insideArchive		=	boolArchives;
	m_includeSubFolders	=	includeSubFolders;
	m_excludeSubFolders	=	excludeSubFolders;
	m_overWriteIndex	=	eraseIndex;   
	m_overWriteMedia	=	eraseMedia;   
	m_appendIndex		=	addToIndex;   
// 	m_appendIndex		=	!eraseIndex;   
	m_updateIndex		=	updateIndex;		//TODO:update not implemented
	m_syncIndex		=	syncIndex;
	m_includeItems		=       includeItems;
	m_excludeItems		=  	excludeItems;		//TODO:not implemented
	
	//init media stuff
	m_mediaName		=	mediaName;
	m_mediaType		=	mediaType;
	
	//converter setup
	m_convert->m_mimeList		=	tmpMimeList;
	m_convert->m_mimeProc		=	tmpMimeProc; 
	m_convert->m_mimeArgs		=	tmpMimeArgs;	
	m_convert->m_mimeOutput		=	tmpMimeOutput;
	m_convert->m_patternList	=	tmpPatternList;
	m_convert->m_patternProc	=	tmpPatternProc;
	m_convert->m_patternArgs	=	tmpPatternArgs;
	m_convert->m_patternOutput	=	tmpPatternOutput;
	
	// document indexing setup
	m_allMetaData	=	boolMetaData;
	m_content	=	boolContent;
	m_readerText 	=       tmpText;
	m_readerHtml	=       tmpHtml;		
	m_readerXml	=       tmpXml;
	m_restrictMetaData=	tmpRestrictMetaData;
	
	//debug
	kdDebug() << "INDEXING SETUP            :" <<  endl;
	kdDebug() << "indexName            	:" <<  m_indexName  << endl;
	kdDebug() << "m_indexLocation		:" <<  m_indexLocation<< endl;
	kdDebug() << "m_insideArchive		:" <<  m_insideArchive << endl;
	kdDebug() << "m_overwrite		:" <<  this->m_overWriteIndex << endl;
	kdDebug() << "m_overwriteMedia		:" <<  this->m_overWriteMedia << endl;
	kdDebug() << "m_updateIndex		:" <<  this->m_updateIndex<< endl;
	kdDebug() << "m_sync		        :" <<  this->m_syncIndex << endl;
	kdDebug() << "m_append		        :" <<  this->m_appendIndex << endl;
	kdDebug() << "m_includeSubFolders	:" <<  m_includeSubFolders << endl;
	kdDebug() << "m_excludeSubFolders	:" <<  m_excludeSubFolders << endl;
	kdDebug() << "m_includeItems		:" <<  m_includeItems << endl;
	kdDebug() << "m_excludeItems		:" <<  m_excludeItems  << endl;
	
	kdDebug() << "CONVERTERS SETUP          :" << endl;
	kdDebug() << "m_mimeList		:" << m_convert-> m_mimeList  << endl;
	kdDebug() << "m_mimeProc		:" << m_convert->m_mimeProc  << endl;
	kdDebug() << "m_mimeArgs		:" << m_convert->m_mimeArgs << endl;
	kdDebug() << "m_mimeOutput		:" << m_convert->m_mimeOutput << endl;
	kdDebug() << "m_patternList		:" << m_convert->m_patternList<< endl;
	kdDebug() << "m_patternProc		:" << m_convert->m_patternProc << endl;
	kdDebug() << "m_patternArgs		:" << m_convert->m_patternArgs << endl;
	kdDebug() << "m_patternOutput		:" << m_convert->m_patternOutput  << endl;
	
	kdDebug() << "DOC SETUP                 :" << endl;
	kdDebug() << "m_allMetaData		:" << m_allMetaData << endl;
	kdDebug() << "m_content			:" << m_content << endl;
	kdDebug() << "m_readerText		:" << m_readerText  << endl;
	kdDebug() << "m_readerHtml		:" << m_readerHtml  << endl;
	kdDebug() << "m_readerXml		:" << m_readerXml  << endl;
	kdDebug() << "restrictMetaData		:" << m_restrictMetaData << endl;
	kdDebug() << "m_mediaName		:" << m_mediaName  << endl;
	kdDebug() << "m_mediaType		:" << m_mediaType  << endl;

	
	//rewrite excludeitems as correct KURLs strings
	QStringList tmp_m_excludeItems;
	for (QStringList::Iterator it = m_excludeItems.begin(); it != m_excludeItems.end(); ++it )
	{
		QString item = *it;
		KURL tmpURL = KURL::fromPathOrURL (item);
		//remove last /
		tmpURL.adjustPath(-1);
		tmp_m_excludeItems.append (tmpURL.pathOrURL());
	}
	m_excludeItems = tmp_m_excludeItems;
	kdDebug() << "m_excludeItems		:" <<  m_excludeItems  << endl;

	emit indexRequestReceived();
	return 1;
}

void KCLuceneIndexerDaemon::slotSyncTextConversionFinished(KProcess * /*proc*/)
{
}



#include "kcluceneindexerdaemon.moc"


