/**************************************************************************
 *
 * text.pass1.c -- Text compression (Pass 1)
 * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 * $Id$
 *
 **************************************************************************/

#include "sysfuncs.h"

#include "memlib.h"
#include "messages.h"
#include "huffman.h"
#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */


#include "mg_files.h"
#include "mg.h"
#include "build.h"
#include "locallib.h"
#include "words.h"
#include "text.h"
#include "hash.h"
#include "local_strings.h"
/* for more meaningful messages - jrm21 (gsdl) */
#ifdef HAVE_STRERROR
#include <string.h>
#include <errno.h>
#endif

#define POOL_SIZE 1024*1024
#define INITIAL_HASH_SIZE 7927


typedef struct hash_rec
  {
    mg_u_long wcnt;		/* word frequency */
    mg_u_long occurance_num;
    u_char *word;
  }
hash_rec;

typedef struct dict_data
  {
    hash_rec *HashTable;
    mg_u_long HashSize;
    mg_u_long HashUsed;
    mg_u_long wordnum;
    mg_u_long words_read;
    mg_u_long bytes_diff;
    huff_data hd;
  }
dict_data;



static mg_u_long LongestDoc = 0;
static mg_u_long occurance_num = 0;
static dict_data DictData[2];

static u_char *Pool;
static int PoolLeft;
static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
static mg_u_long MaxMemInUse = 0;
static mg_u_long MemInUse = 0;
static compression_stats_header csh =
{0, 0.0}; /* [RJM 07/97: 4G limit] */


static void 
ChangeMem (int Change)
{
  MemInUse += Change;
  if (MemInUse > MaxMemInUse)
    MaxMemInUse = MemInUse;
}




int 
init_text_1 (char *FileName)
{
  int which;
  if (!(Pool = Xmalloc (POOL_SIZE)))
    {
      Message ("Unable to allocate memory for pool");
      return (COMPERROR);
    }
  PoolLeft = POOL_SIZE;
  ChangeMem (POOL_SIZE);

  for (which = 1; which >= 0; which--)
    {
      u_char *word;
      hash_rec *ent;
      dict_data *dd = &DictData[which];

      dd->wordnum = 0;
      dd->words_read = 0;
      dd->bytes_diff = 0;
      dd->HashSize = INITIAL_HASH_SIZE;
      dd->HashUsed = 0;
      if (!(dd->HashTable = Xmalloc (sizeof (hash_rec) * dd->HashSize)))
	{
	  Message ("Unable to allocate memory for table");
	  return (COMPERROR);
	}
      ChangeMem (sizeof (hash_rec) * dd->HashSize);
      bzero ((char *) (dd->HashTable), sizeof (hash_rec) * dd->HashSize);

      word = Pool;
      *Pool++ = '\0';
      PoolLeft--;
      {
	register u_char *wptr;
	register int hsize = dd->HashSize;
	register mg_u_long hashval, step;

	HASH (hashval, step, word, hsize);
	wptr = (dd->HashTable + hashval)->word;
	while (wptr)
	  {
	    hashval += step;
	    if (hashval >= hsize)
	      hashval -= hsize;
	    wptr = (dd->HashTable + hashval)->word;
	  }
	ent = dd->HashTable + hashval;
      }
      ent->wcnt = 1;
      ent->word = word;
      dd->HashUsed = 1;
    }
  return (COMPALLOK);
}




int 
process_text_1 (u_char * s_in, int l_in)
{
  int which;
  u_char *end = s_in + l_in - 1;
  if (l_in > LongestDoc)
    LongestDoc = l_in;

  csh.num_docs++;
  csh.num_bytes += l_in;

  which = inaword (s_in, end);
  /*
   ** Alternately parse off words and non-words from the input
   ** stream beginning with a non-word. Each token is then
   ** inserted into the set if it does not exist or has it's
   ** frequency count incremented if it does.
   */
  for (; s_in <= end; which = !which)
    {
      u_char Word[MAXWORDLEN + 1];
      dict_data *dd = &DictData[which];

      /* First parse a word or non-word out of the string */
      if (which)
	PARSE_WORD (Word, s_in, end);
      else
	PARSE_NON_WORD (Word, s_in, end);

      dd->wordnum++;
      inputbytes += *Word;
      dd->words_read++;

      /* Search the hash table for Word */
      {
	register mg_u_long hashval, step;
	register int hsize = dd->HashSize;
	HASH (hashval, step, Word, hsize);
	for (;;)
	  {
	    register u_char *s1;
	    register u_char *s2;
	    register int len;
	    register hash_rec *ent;
	    ent = dd->HashTable + hashval;
	    if (!ent->word)
	      {
		int len = *Word + 1;
		if (len > PoolLeft)
		  {
		    if (!(Pool = Xmalloc (POOL_SIZE)))
		      {
			Message ("Unable to allocate memory for pool");
			return (COMPERROR);
		      }
		    PoolLeft = POOL_SIZE;
		    ChangeMem (POOL_SIZE);
		  }
		ent->occurance_num = occurance_num++;
		ent->wcnt = 1;
		ent->word = Pool;
		memcpy (Pool, Word, len);
		Pool += len;
		PoolLeft -= len;
		dd->HashUsed++;
		dd->bytes_diff += Word[0];
		break;
	      }
	    /* Compare the words */
	    s1 = Word;
	    s2 = ent->word;
	    len = *s1 + 1;
	    for (; len; len--)
	      if (*s1++ != *s2++)
		break;

	    if (len)
	      {
		hashval = (hashval + step);
		if (hashval >= hsize)
		  hashval -= hsize;
	      }
	    else
	      {
		ent->wcnt++;
		break;
	      }
	  }
      }

  
      if (dd->HashUsed >= dd->HashSize >> 1)
	{
	  hash_rec *ht;
	  mg_u_long size;
	  mg_u_long i;
	  size = prime (dd->HashSize * 2);
	  if (!(ht = Xmalloc (sizeof (hash_rec) * size)))
	    {
	      Message ("Unable to allocate memory for table");
	      return (COMPERROR);
	    }
	  ChangeMem (sizeof (hash_rec) * size);
	  bzero ((char *) ht, sizeof (hash_rec) * size);

	  for (i = 0; i < dd->HashSize; i++)
	    if (dd->HashTable[i].word)
	      {
		register u_char *wptr;
		register mg_u_long hashval, step;

		wptr = dd->HashTable[i].word;
		HASH (hashval, step, wptr, size);
		wptr = (ht + hashval)->word;
		while (wptr)
		  {
		    hashval += step;
		    if (hashval >= size)
		      hashval -= size;
		    wptr = (ht + hashval)->word;
		  }
		ht[hashval] = dd->HashTable[i];
	      }
	  Xfree (dd->HashTable);
	  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
	  dd->HashTable = ht;
	  dd->HashSize = size;


	}
    }
  return (COMPALLOK);
}				/* encode */



static int 
PackHashTable (dict_data * dd)
{
  int s, d;
  for (s = d = 0; s < dd->HashSize; s++)
    if (dd->HashTable[s].word)
      dd->HashTable[d++] = dd->HashTable[s];
  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
  ChangeMem (sizeof (hash_rec) * dd->HashUsed);
  if (!(dd->HashTable = Xrealloc (dd->HashTable,
				  sizeof (hash_rec) * dd->HashUsed)))
    {
      Message ("Out of memory");
      return COMPERROR;
    }
  dd->HashSize = dd->HashUsed;
  return COMPALLOK;
}





static int 
ent_comp (const void *s1, const void *s2)
{
  return casecompare (((hash_rec *) s1)->word, ((hash_rec *) s2)->word);
}



static void 
WriteHashTable (FILE * fp, dict_data * dd)
{
  frags_stats_header fsh;
  mg_u_long j = 0;
  u_char *curr;

  if (PackHashTable (dd) == COMPERROR)
    return;

  qsort (dd->HashTable, dd->HashUsed, sizeof (hash_rec), ent_comp);

  fsh.num_frags = dd->HashSize;
  fsh.mem_for_frags = dd->HashSize;
  for (j = 0; j < dd->HashSize; j++)
    fsh.mem_for_frags += dd->HashTable[j].word[0];

  /* [RPAP - Jan 97: Endian Ordering] */
  HTONUL(fsh.num_frags);
  HTONUL(fsh.mem_for_frags);

  fwrite (&fsh, sizeof (fsh), 1, fp);

  for (j = 0; j < dd->HashSize; j++)
    {
      curr = dd->HashTable[j].word;

      /* [RPAP - Jan 97: Endian Ordering] */
      HTONUL(dd->HashTable[j].wcnt);
      HTONUL(dd->HashTable[j].occurance_num);

      fwrite (&dd->HashTable[j].wcnt, sizeof (dd->HashTable[j].wcnt), 1, fp);
      fwrite (&dd->HashTable[j].occurance_num,
	      sizeof (dd->HashTable[j].occurance_num), 1, fp);

      /* [RPAP - Jan 97: Endian Ordering] */
      NTOHUL(dd->HashTable[j].wcnt);
      NTOHUL(dd->HashTable[j].occurance_num);

      fwrite (curr, sizeof (u_char), curr[0] + 1, fp);
    }
}


int 
done_text_1 (char *file_name)
{
  char *temp_str;
  FILE *fp;

  if (!(fp = create_file (file_name, TEXT_STATS_DICT_SUFFIX, "wb",
			  MAGIC_STATS_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
    {
      fprintf(stderr,"Couldn't create file %s%s:%s\n",
	      file_name, TEXT_STATS_DICT_SUFFIX,
#if defined(HAVE_STRERROR) || defined(__WIN32__)
	      strerror(errno)
#else
	      " "
#endif
	      );
      return COMPERROR;
    }

  temp_str = msg_prefix;
  msg_prefix = "text.pass1";

  /* [RPAP - Jan 97: Endian Ordering] */
  HTONUL(csh.num_docs);
  HTOND(csh.num_bytes); /* [RJM 07/97: 4G limit] */

  fwrite (&csh, sizeof (csh), 1, fp);

  /* [RPAP - Jan 97: Endian Ordering] */
  NTOHUL(csh.num_docs);
  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */

  WriteHashTable (fp, &DictData[0]);
  WriteHashTable (fp, &DictData[1]);
  msg_prefix = temp_str;
  fclose(fp);
  return COMPALLOK;
}				/* done_encode */