/**************************************************************************
 *
 * ivf.pass1.cpp -- Memory efficient pass 1 inversion
 * Copyright (C) 1999  Rodger McNab
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 **************************************************************************/

#include "UCArray.h"
#include "sysfuncs.h"
#include "mg_files.h"
#include "invf.h"
#include "mg.h"
#include "build.h"
#include "locallib.h"
#include "bitio_m_stdio.h"
#include "bitio_gen.h"
#include <stdio.h>
#include "words.h"
#include "messages.h"
#include "netorder.h"
#include "FIvfLevelInfo.h"

#include "longlong.h"

#if defined(GSDL_USE_OBJECTSPACE)
#  include <ospace\std\map>
#elif defined(GSDL_USE_STL_H)
#  include <map.h>
#else
#  include <map>
#endif

// a fragment corresponds to the index level (word-level is the
// minimum index level)


// structure to determine level information
struct LevelWorker {
  mg_u_long lastLevelDocNum;
  mg_u_long count;

  LevelWorker () {
    lastLevelDocNum = count = 0;
  }
};


// note: the word is stored in the map
struct IvfWordInfo {
  mg_u_long wordCount;      // word frequency
  mg_u_long fragCount;      // number of fragments that contain the word
  mg_u_long lastFragNum;    // last fragment to contain the word
  mg_u_long chunkWordCount; // word frequency within this chunk
  mg_u_long chunkFragCount; // number of fragments within this chunk that
                                //   contain the word

  LevelWorker *levels;          // level info for this word
  
  IvfWordInfo ();
  ~IvfWordInfo ();
  void Clear ();                // will delete levels
};

typedef map<UCArray, IvfWordInfo, DictLTUCArray> IvfWordInfoMap;
typedef vector<IvfWordInfoMap::iterator> IvfWordInfoItArray;

// tags don't require as much information
struct IvfTagInfo {
  mg_u_long tagCount;       // tag frequency
  mg_u_long fragCount;      // number of fragments that contain the tag
  mg_u_long lastFragNum;    // last fragment to contain the tag
  mg_u_long chunkFragCount; // number of fragments within this chunk that
                                //   contain the tag

  IvfTagInfo ();
  void Clear ();
};

typedef map<UCArray, IvfTagInfo, DictLTUCArray> IvfTagInfoMap;
typedef vector<IvfTagInfoMap::iterator> IvfTagInfoItArray;


#define INIT_CHECK_FRAC 0.10
#define CHECK_FRAC 0.75
#define CHECK_CLOSE 0.999
#define CHECK_DIV 1.5


static FILE *ic;		// the invf chunk file
static stdio_bitio_buffer icb;

IvfWordInfoMap ivfWordInfo;
IvfWordInfoItArray ivfWordInfoOccurOrder;
IvfTagInfoMap ivfTagInfo;
IvfTagInfoItArray ivfTagInfoOccurOrder;

static mg_u_long chunksWritten;
static mg_u_long maxMemNeeded;

static mg_u_long numDocs;
static mg_u_long numChunkDocs;

static mg_u_long numFrags;
static mg_u_long numChunkFrags;

static mg_u_long numWords;

// the number of document numbers in the inverted file
static mg_u_long numChunkEntries;

// next entry in the inverted file to check memory
// requirements for the current chunk
static mg_u_long entryCheckPoint;

// information about all the different levels
static FIvfLevel ivfLevel;



IvfWordInfo::IvfWordInfo () {
  levels = NULL;
  Clear();
}

IvfWordInfo::~IvfWordInfo () {
  if (levels != NULL) delete [] levels;
}

void IvfWordInfo::Clear () {
  wordCount = 0;
  chunkWordCount = 0;
  lastFragNum = 0;
  fragCount = 0;
  chunkFragCount = 0;

  if (levels != NULL) {
    delete [] levels;
    levels = NULL;
  }
}


IvfTagInfo::IvfTagInfo () {
  Clear();
}

void IvfTagInfo::Clear () {
  tagCount = 0;
  lastFragNum = 0;
  fragCount = 0;
  chunkFragCount = 0;
}



int init_ivf_1 (const TagInfo &tagInfo, char *file_name) {
  // set up the chunk file
  if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb",
			  MAGIC_CHUNK, MG_MESSAGE)))
    return COMPERROR;
  fwrite ("    ", sizeof (mg_u_long), 1, ic);	// Space for the maxmem
  icb.attachFile (ic);
  icb.encodeStart();

  // reset global variables
  ivfWordInfo.erase (ivfWordInfo.begin(), ivfWordInfo.end());
  ivfWordInfoOccurOrder.erase (ivfWordInfoOccurOrder.begin(), ivfWordInfoOccurOrder.end());
  ivfTagInfo.erase (ivfTagInfo.begin(), ivfTagInfo.end());
  ivfTagInfoOccurOrder.erase (ivfTagInfoOccurOrder.begin(), ivfTagInfoOccurOrder.end());
  
  chunksWritten = 0;
  maxMemNeeded = 0;
 
  numDocs = 0;
  numChunkDocs = 0;

  numFrags = 0;
  numChunkFrags = 0;

  numWords = 0;

  numChunkEntries = 0;
  entryCheckPoint = (mg_u_long) ((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);

  // init the level information
  ivfLevel.Clear();
  ivfLevel.docTag = tagInfo.docTag;
  ivfLevel.indexLevel = tagInfo.indexLevel;
  IvfLevelInfo blankLevel;
  UCArraySet::const_iterator levelHere = tagInfo.levelTags.begin();
  UCArraySet::const_iterator levelEnd = tagInfo.levelTags.end();
  while (levelHere != levelEnd) {
    blankLevel.levelTag = *levelHere;
    ivfLevel.levelInfo[*levelHere] = blankLevel;
    ++levelHere;
  }
  
  return COMPALLOK;
}

static void ProcessOpenTag (const TagInfo &tagInfo, const TextEl &el,
			    bool &inFrag) {
  bool wordLevelIndex = tagInfo.indexLevel.empty();

  // check for start of next fragment
  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
    ++numFrags;
    ++numChunkFrags;
    inFrag = true;
  }
  
  // update tag stats
  IvfTagInfo &i = ivfTagInfo[el.tagName];
  if (i.tagCount == 0) {
    // new tag, add to list of iterators
    IvfTagInfoMap::iterator iIt = ivfTagInfo.find (el.tagName);
    ivfTagInfoOccurOrder.push_back (iIt);
  }
  ++i.tagCount;

  // all open tags count as new tags
  ++numChunkEntries;
  ++i.fragCount;
  ++i.chunkFragCount;
  i.lastFragNum = numFrags;
  
  // update level information
  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
  if (levelIt != ivfLevel.levelInfo.end()) {
    // is a level tag
    ++levelIt->second.numEntries;
    levelIt->second.workInLevel = true;
  }
}

static void ProcessCloseTag (const TagInfo &tagInfo, const TextEl &el,
			     bool &inFrag) {
  bool wordLevelIndex = tagInfo.indexLevel.empty();

  // check for end of fragment
  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
    inFrag = false;
  }

  // update level information
  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
  if (levelIt != ivfLevel.levelInfo.end()) {
    // is a level tag
    (*levelIt).second.workInLevel = false;
  }
}

static void ProcessText (const TagInfo &tagInfo, const TextEl &el,
			 bool &inFrag) {
  bool wordLevelIndex = tagInfo.indexLevel.empty();

  // make sure this text is to be indexed
  if (!wordLevelIndex && !inFrag) return;

  const unsigned char *textHere = &(el.text[0]);
  const unsigned char *textEnd = &(el.text[el.text.size() - 1]);
  UCArray word;
  
  if (!inaword_mgpp (textHere, textEnd))
    ParseNonindexWord (textHere, textEnd);

  
  // Alternately parse off words and non-words from the input
  // Each token is then inserted into the set if it does
  // not exist or has it's frequency count incremented if it does.

  while (textHere <= textEnd) {
    textHere = ParseIndexWord (textHere, textEnd, word);
    textHere = ParseNonindexWord (textHere, textEnd);

    if (!word.empty()) {
      ++numWords;

      if (wordLevelIndex) {
	++numFrags;
	++numChunkFrags;
      }
      
      // update word stats

      IvfWordInfo &i = ivfWordInfo[word];
      if (i.wordCount == 0) {
	// new word
	// add to list of iterators
	IvfWordInfoMap::iterator iIt = ivfWordInfo.find (word);
	ivfWordInfoOccurOrder.push_back (iIt);

	// add level information array
	if (ivfLevel.levelInfo.size() > 0) 
	  i.levels = new LevelWorker [ivfLevel.levelInfo.size()];
      }

      ++i.wordCount;
      ++i.chunkWordCount;
      if (numFrags > i.lastFragNum) {
	++numChunkEntries;
	++i.fragCount;
	++i.chunkFragCount;
	i.lastFragNum = numFrags;
      }

      // update level information for this word
      if (i.levels != NULL) {
	IvfLevelInfoMap::iterator levelHere = ivfLevel.levelInfo.begin();
	IvfLevelInfoMap::iterator levelEnd = ivfLevel.levelInfo.end();
	LevelWorker *levelWorkerPtr = i.levels;
	while (levelHere != levelEnd) {

	  // check to make sure the level encompases this fragment
	  if (!(*levelHere).second.workInLevel) {
	    cerr << "Level tag <" << (*levelHere).first
		 << "> does not encompass all fragments\n";
	    exit (1);
	  }
	  
	  if ((*levelHere).second.numEntries > (*levelWorkerPtr).lastLevelDocNum) {
	    (*levelWorkerPtr).lastLevelDocNum = (*levelHere).second.numEntries;
	    ++levelWorkerPtr->count;
	  }
	  
	  ++levelHere;
	  ++levelWorkerPtr;
	}
      }
    }
  }
}


static mg_u_long MemoryRequired (bool wordLevelIndex) {
  register mg_u_long total = 0;

  // add memory required for word entries
  IvfWordInfoMap::const_iterator wordHere = ivfWordInfo.begin();
  IvfWordInfoMap::const_iterator wordEnd = ivfWordInfo.end();
  while (wordHere != wordEnd) {
    register const IvfWordInfo &info = (*wordHere).second;
    if (info.chunkFragCount > 0) {
      total += BIO_Bblock_Bound (numChunkFrags, info.chunkFragCount);
      if (!wordLevelIndex) {
	total += info.chunkWordCount;
      }
    }
    
    ++wordHere;
  }

  // add memory required for tag entries
  IvfTagInfoMap::const_iterator tagHere = ivfTagInfo.begin();
  IvfTagInfoMap::const_iterator tagEnd = ivfTagInfo.end();
  while (tagHere != tagEnd) {
    register const IvfTagInfo &info = (*tagHere).second;
    if (info.chunkFragCount > 0) {
      // two d entries for each frag entry
      mg_u_long pTag = info.chunkFragCount*2;
      total += BIO_Bblock_Bound (numChunkFrags+pTag, pTag);
    }
    
    ++tagHere;
  }
  
  total = (total + 7) >> 3;
  return total;
}


/*
static void PrintChunkInfo (mg_u_long mem) {
  cout << "Chunk Number: " << chunksWritten << "\n";
  cout << "numChunkDocs " << numChunkDocs << "\n";
  cout << "numChunkFrags " << numChunkFrags << "\n";
  cout << "mem " << mem << "\n";
  cout << "numWords " << ivfWordInfo.size() << "\n";
  cout << "numTags " << ivfTagInfo.size() << "\n\n";

  // output debug tag information in dictionary order
  IvfTagInfoMap::iterator tagMapHere = ivfTagInfo.begin();
  IvfTagInfoMap::iterator tagMapEnd = ivfTagInfo.end();
  mg_u_long tagNum = 0;
  while (tagMapHere != tagMapEnd) {
    cout << (*tagMapHere).first << " " << tagNum
	 << " " << (*tagMapHere).second.chunkFragCount << "\n";
    ++tagNum;
    ++tagMapHere;
  }
}
*/

static void OutputChunkInfo (mg_u_long mem, bool /*wordLevelIndex*/) {
  ++chunksWritten;

  // sanity check
  if (ivfWordInfo.size() != ivfWordInfoOccurOrder.size()) {
    Message ("ERROR: Word information size mismatch: %u vs %u\n",
	     (unsigned int)ivfWordInfo.size(),
	     (unsigned int)ivfWordInfoOccurOrder.size());
    exit (1);
  }
  if (ivfTagInfo.size() != ivfTagInfoOccurOrder.size()) {
    Message ("ERROR: Tag information size mismatch: %u vs %u\n",
	     (unsigned int)ivfTagInfo.size(),
	     (unsigned int)ivfTagInfoOccurOrder.size());
    exit (1);
  }

  icb.gamma_encode (numChunkDocs + 1, NULL);
  icb.gamma_encode (numChunkFrags + 1, NULL);
  icb.gamma_encode (mem + 1, NULL);
  icb.gamma_encode (ivfWordInfo.size() + 1, NULL);
  icb.gamma_encode (ivfTagInfo.size() + 1, NULL);

  /*  PrintChunkInfo (mem);*/

  // output word information in occurance order
  IvfWordInfoItArray::iterator wordHere = ivfWordInfoOccurOrder.begin();
  IvfWordInfoItArray::iterator wordEnd = ivfWordInfoOccurOrder.end();
  while (wordHere != wordEnd) {
    register IvfWordInfo &ivfWordInfo = (*(*wordHere)).second;
    
    icb.gamma_encode (ivfWordInfo.chunkWordCount + 1, NULL);
    if (ivfWordInfo.chunkWordCount >= 2) {
      icb.gamma_encode (ivfWordInfo.chunkFragCount, NULL);
    }

    ivfWordInfo.lastFragNum = 0;
    ivfWordInfo.chunkWordCount = 0;
    ivfWordInfo.chunkFragCount = 0;
    
    ++wordHere;
  }

  // output tag information in occurance order
  IvfTagInfoItArray::iterator tagHere = ivfTagInfoOccurOrder.begin();
  IvfTagInfoItArray::iterator tagEnd = ivfTagInfoOccurOrder.end();
  while (tagHere != tagEnd) {
    register IvfTagInfo &ivfTagInfo = (*(*tagHere)).second;

    icb.gamma_encode (ivfTagInfo.chunkFragCount + 1, NULL);

    ivfTagInfo.lastFragNum = 0;
    ivfTagInfo.chunkFragCount = 0;
    
    ++tagHere;
  }

  numChunkDocs = 0;
  numChunkFrags = 0;
  numChunkEntries = 0;
}


int process_ivf_1 (const TagInfo &tagInfo, const TextElArray &doc) {
  bool wordLevelIndex = tagInfo.indexLevel.empty();
  bool inFrag = false;
  if (wordLevelIndex) inFrag = true; // unconditional
  
  ++numDocs;
  ++numChunkDocs;

  // process each text element in this document
  TextElArray::const_iterator here = doc.begin();
  TextElArray::const_iterator end = doc.end();
  while (here != end) {
    if ((*here).elType == OpenTagE)
      ProcessOpenTag (tagInfo, *here, inFrag);
    else if ((*here).elType == CloseTagE)
      ProcessCloseTag (tagInfo, *here, inFrag);
    else if ((*here).elType == TextE)
      ProcessText (tagInfo, *here, inFrag);
    
    ++here;
  }

  // check the amount of memory needed for this chunk
  if (numChunkEntries >= entryCheckPoint) {
    mg_u_long mem = MemoryRequired (wordLevelIndex);
    if (mem >= invf_buffer_size * CHECK_CLOSE) {
      if (mem > maxMemNeeded) maxMemNeeded = mem;
      OutputChunkInfo (mem, wordLevelIndex);
      entryCheckPoint = (mg_u_long)
	((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);
      
    } else {
      entryCheckPoint = (mg_u_long)
	(entryCheckPoint * ((CHECK_FRAC * (invf_buffer_size - mem)) / mem) +
	 entryCheckPoint);
      if (entryCheckPoint <= numChunkEntries)
	entryCheckPoint = numChunkEntries + 1;
    }
  }
  
  return COMPALLOK;
}


static void CalcInvfDictSize (mg_u_long &totalBytes,
			      mg_u_long &indexStringBytes) {
  totalBytes = 0;	// The sum of the length of all words, including 
			// the length byte
  indexStringBytes = 0; // The amount of space required to store the 
			// words in the diction, this takes into account 
			// the prefixes
  const UCArray *lastWord = NULL;
  
  // calculate size of word information
  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
  while (wordHere != wordEnd) {
    mg_u_long wordSize = (*wordHere).first.size();
    totalBytes += wordSize + 1;
    indexStringBytes += wordSize + 2;
    if (lastWord != NULL)
      indexStringBytes -= PrefixLen (*lastWord, (*wordHere).first);
    lastWord = &((*wordHere).first);
    
    ++wordHere;
  }

  // calculate size of tag information
  lastWord = NULL;
  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
  while (tagHere != tagEnd) {
    mg_u_long tagSize = (*tagHere).first.size();
    totalBytes += tagSize + 1;
    indexStringBytes += tagSize + 2;
    if (lastWord != NULL)
      indexStringBytes -= PrefixLen (*lastWord, (*tagHere).first);
    lastWord = &((*tagHere).first);
    
    ++tagHere;
  }
}


//      OutputInvfDict ():
//                writes out the stemmed dictionary file
//                in the following format 
//                        lookback value (int)
//                        totalbytes value (int)
//                        indexstringbytes (int)
//                        for each word 
//                          wordlen (4 bits)
//                          prefix match (4 bits)
//                          word (wordlen bytes)
//                          word frequency (int)
//                          word count (int)
//
//        Accesses outside variables:     
//
//        Return value...:                
static void OutputInvfDict (char *filename) {
  // create the dictionary header
  invf_dict_header idh;
  idh.word_dict_size = ivfWordInfo.size();
  idh.tag_dict_size = ivfTagInfo.size();
  idh.num_docs = numDocs;
  idh.num_frags = numFrags;
  idh.num_words = numWords;
  idh.num_levels = ivfLevel.levelInfo.size();
  CalcInvfDictSize (idh.total_bytes, idh.index_string_bytes);

  // create the inverted dictionary file
  FILE *sp;
  if (!(sp = create_file (filename, INVF_DICT_SUFFIX, "wb", MAGIC_STEM_BUILD,
			  MG_MESSAGE)))
    return;

  // write out the dictionary header
  if (!idh.Write (sp)) { fclose (sp); return; }

  // remember where the word dictionary starts
  idh.word_dict_start = ftell (sp);
  
  // output the word dictionary
  const UCArray *lastWord = NULL;
  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
  while (wordHere != wordEnd) {
    // get the prefix and suffix lengths
    const UCArray &thisWord = (*wordHere).first;
    WritePreSufStr (sp, lastWord, thisWord);

    // output the number of fragments the word appeared in and the
    // number of times the word appeared
    WriteUL (sp, (*wordHere).second.fragCount);
    WriteUL (sp, (*wordHere).second.wordCount);

    // output frequency information for each level
    // note that we are expecting every word to have
    // level information
    LevelWorker *lwHere = (*wordHere).second.levels;
    LevelWorker *lwEnd = lwHere + idh.num_levels;
    while (lwHere != lwEnd) {
      WriteUL (sp, (*lwHere).count);
      ++lwHere;
    }
    
    lastWord = &thisWord;
    ++wordHere;
  }

  // remember where the tag dictionary starts
  idh.tag_dict_start = ftell (sp);
  
  // output the tag dictionary
  const UCArray *lastTag = NULL;
  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
  while (tagHere != tagEnd) {
    // get the prefix and suffix lengths
    const UCArray &thisTag = (*tagHere).first;
    WritePreSufStr (sp, lastTag, thisTag);

    // output the number of fragments the tag appeared in and the
    // number of times the tag appeared
    WriteUL (sp, (*tagHere).second.fragCount);
    WriteUL (sp, (*tagHere).second.tagCount);
    
    lastTag = &thisTag;
    ++tagHere;
  }

  // write out the updated header
  fseek (sp, sizeof (mg_u_long), SEEK_SET);
  if (!idh.Write (sp)) { fclose (sp); return; }
  
  fclose (sp);
}

static void OutputLevelFile (char *filename) {
  // create the level file
  FILE *f;
  if (!(f = create_file (filename, INVF_LEVEL_SUFFIX, "wb", MAGIC_INVF_LEVELS,
			 MG_MESSAGE)))
    return;

  // write out the information
  ivfLevel.Write (f);
  
  // close the file
  fclose (f);
}

static void OutputTransFile (char *filename) {
  // save the word number (in lastFragNum :-/)
  int i = 0;
  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
  while (wordHere != wordEnd) {
    (*wordHere).second.lastFragNum = i;
    ++i; ++wordHere;
  }

  // save the tag number (in lastFragNum :-/)
  i = 0;
  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
  while (tagHere != tagEnd) {
    (*tagHere).second.lastFragNum = i;
    ++i; ++tagHere;
  }
  
  // create the translation file
  FILE *f;
  if (!(f = create_file (filename, INVF_CHUNK_TRANS_SUFFIX, "wb",
			 MAGIC_CHUNK_TRANS, MG_MESSAGE)))
    return;

  stdio_bitio_buffer buffer(f);
  buffer.encodeStart();

  // write out the word translation table
  mg_u_long wordDictSize = ivfWordInfoOccurOrder.size();
  IvfWordInfoItArray::iterator wordItHere = ivfWordInfoOccurOrder.begin();
  IvfWordInfoItArray::iterator wordItEnd = ivfWordInfoOccurOrder.end();
  mg_u_long oN = 0;
  while (wordItHere != wordItEnd) {
    register IvfWordInfo &ivfWordInfo = (*(*wordItHere)).second;
    buffer.binary_encode (ivfWordInfo.lastFragNum + 1, wordDictSize + 1, NULL);
    ++oN;
    ++wordItHere;
  }

  // write out the tag translation table
  mg_u_long tagDictSize = ivfTagInfoOccurOrder.size();
  IvfTagInfoItArray::iterator tagItHere = ivfTagInfoOccurOrder.begin();
  IvfTagInfoItArray::iterator tagItEnd = ivfTagInfoOccurOrder.end();
  while (tagItHere != tagItEnd) {
    register IvfTagInfo &ivfTagInfo = (*(*tagItHere)).second;
    buffer.binary_encode (ivfTagInfo.lastFragNum + 1, tagDictSize + 1, NULL);
    ++oN;
    ++tagItHere;
  }

  // finish encoding and close the file
  buffer.encodeDone();
  fclose (f);
}

#ifndef SILENT
static void PrintStats () {
  Message ("Inverted buffer size:          %8u bytes\n", invf_buffer_size);
  Message ("Max memory needed for 1 chunk: %8u bytes\n", maxMemNeeded);

  Message ("Number of chunks written:      %8u\n", chunksWritten);
  Message ("Number of documents:           %8u\n", numDocs);
  Message ("Number of fragments:           %8u\n", numFrags);
  Message ("Number of words:               %8u\n", numWords);

  Message ("Size of word dictionary:       %8u\n", ivfWordInfo.size());
  Message ("Size of tag dictionary:        %8u\n", ivfTagInfo.size());
}
#endif

int done_ivf_1 (const TagInfo &tagInfo, char * filename) {
  bool wordLevelIndex = tagInfo.indexLevel.empty();

  char *temp_str = msg_prefix;
  msg_prefix = (char*)"ivf.pass1";

  // output the last chunk
  if (numChunkDocs > 0) {
    mg_u_long mem = MemoryRequired (wordLevelIndex);
    OutputChunkInfo (mem, wordLevelIndex);
    if (mem > maxMemNeeded) maxMemNeeded = mem;
  }
  
  // write out and encoded 1 to say there are no more chunks
  icb.gamma_encode (1, NULL);
  icb.encodeDone ();

  // write out the maximum memory required and close the file
  fseek (ic, sizeof (mg_u_long), 0);
  WriteUL (ic, maxMemNeeded);
  fclose (ic);

  // output the inverted dictionary
  OutputInvfDict (filename);

  // write out the translation file
  OutputTransFile (filename);

  // output the level information
  OutputLevelFile (filename);
  
  // output statistics
#ifndef SILENT
  PrintStats ();
#endif
  
  msg_prefix = temp_str;

  return COMPALLOK;
}