pcache.c 84.3 KB

Raw Blame History Permalink

/*
 * Copyright (C) 1996-1998 by the Board of Trustees
 *    of Leland Stanford Junior University.
 *
 * This file is part of the SimOS distribution.
 * See LICENSE file for terms of the license.
 *
 */

/*****************************************************************
 * pcache.c
 *
 * Primary cache implemented as up to 4-way associative
 * caches, compile time choice of associativity, size, & line size.
 *
 * $Author: blythe $
 * $Date: 2002/06/26 20:46:25 $
 *******************************************************************/

#include <stdio.h>
#include <time.h>
#include <assert.h>
#include <malloc.h>
#include <string.h>
#include "syslimits.h"
#include "sim_error.h"
#include "../memref.h"
#include "pcache.h"
#include "scache.h"
#include "simrecord.h"
#include "memsys.h"
#include "simutil.h"
#include "false_sharing.h"
#include "hw_events.h"
#include "registry.h"
#include "tcl_init.h"
#include "limits.h"
#include "arch_specifics.h"
#include "cpu_stats.h"
#if (defined(SIM_MIPS64) || defined(SIM_MIPS32))
#include "trace.h"
#else
#define TraceDataRef(a,b,c)
#endif
#ifdef MIPSY_MXS
#  include "ms.h"
#endif

/* We haven't been using these, so I'll make them compile time
   parameters for now. It will also speed up the caches. */
#define DCACHE_HIT_ALWAYS  0
#define DCACHE_MISS_ALWAYS 0
#define ICACHE_HIT_ALWAYS  0
#define ICACHE_MISS_ALWAYS 0

#define CURRENT_PC(_cpu) (CPUVec.CurrentPC(_cpu))

/*
 * Possible return status from the miss handling table.
 * MHTSUCCESS - Entry allocated everything looks good.
 * MHTMERGE   - This request merged with another. Returned other.
 * MHTFULL    - Entry not allocated because table was full.
 * MHTCONFLICT - Entry not allocated because it conflicted with one already
 *               allocated.
 */
typedef enum { MHTSUCCESS = 0, MHTFULL, MHTMERGE, MHTCONFLICT} MHTStatus;

Cache *CACHE;

bool skipCaches;

/* Variables needed for the lookup macros */
static int iTagShift;
static int iIndexMask;
static int dTagShift;
static int dIndexMask;
#ifndef SIM_ALPHA
static int iAddrDivisor;
static int dAddrDivisor;
#else
       int iAddrDivisor;
       int dAddrDivisor;
#endif

static bool useWriteBuffer;
static bool noUpgrades  = FALSE;   /* Send no upgrades to mem system */

/* Local Cache Functions */
static void   InitPCaches(void);
static void   ICachePUT(int, MHT *, int );
static void   DCacheFlush(int, int writeback, int retain, PA, int);
static void   DCachePUT(int, MHT *, int);
static Result ICacheMiss(int cpuNum, VA, PA, int *);
static Result DCacheReadMiss(int cpuNum, VA, PA, void *data, RefSize size,
                             RefFlavor flavor, int *indPtr);
static Result DCacheWriteMiss(int cpuNum, VA, PA, RefSize size, RefFlavor flavor,
                              struct DCacheSet* set, int way, bool upgrade,
                              int* mhtind);

static void      FreeMHT(int cpuNum, int entryNum);
static MHTStatus AllocMHT(int cpuNum, SCacheCmd cmd, VA vAddr, PA pAddr,
                          int size, int lru, int *mhtind);

static void RetryNAKedMHTEntry(int cpuNum,EventCallbackHdr *event, void *arg);

static bool AddToWriteBuffer(int cpuNum,PA pAddr, uint64 data, RefSize size,
                             int mhtind);

static bool AddrIsInWriteBuffer(int cpuNum, PA pAddr, RefSize size);

static void RetireWriteBuffer(int cpuNum, MHT *mht, byte *data);


/****************************************************************
 *
 * Cope with different associativities.
 *
 * The following code handles up to 4-way for now, but in the future
 * the interface is good enough to handle higher associativities.
 * Just extend the if tree and pass a pointer to the first word
 * of the set
 */

#define UPDATE_BIG_LRU(lruword,set)		\
{						\
   unsigned char* _lru = (unsigned char*) &(lruword);	\
   unsigned char _t1, _t2;			\
						\
   if (_lru[0] != (set)) {			\
      _t1 = _lru[1];				\
      _lru[1] = _lru[0];			\
      _lru[0] = (set);				\
      if (_t1 != (set)) {			\
	 _t2 = _lru[2];				\
	 _lru[2] = _t1;				\
	 if (_t2 != (set))  _lru[3] = _t2;	\
      }						\
   }						\
}

#define INIT_BIG_LRU(lruword)			\
{						\
   char* _lru = (char*) &(lruword);		\
   _lru[0] = 0; _lru[1] = 1; _lru[2] = 2; _lru[3] = 3; \
}

#define GET_BIG_LRU(lruword, assoc)	   (((char*) &(lruword))[(assoc)-1])

#define SET_BIG_LRU4(lruword, set)		\
{						\
   unsigned char* _lru = (unsigned char*) &(lruword); \
   unsigned char _t1,_t2;                    	\
						\
   if (_lru[3] != (set)) {			\
      _t1 = _lru[2];				\
      _lru[2] = _lru[3];			\
      _lru[3] = (set);				\
      if (_t1 != (set)) {			\
	 _t2 = _lru[1];				\
	 _lru[1] = _t1;				\
	 if (_t2 != (set))  _lru[0] = _t2;	\
      }						\
   }						\
}

#if ICACHE_ASSOC == 1
#define ICACHE_TOUCH(lruword, set)    /* nop */
#define ICACHE_INIT_LRU(lruword)      lruword = 0;
#define ICACHE_LRU(lruword)           0
#define ICACHE_MAKE_LRU(lruword, set) /* nop */
#endif

#if ICACHE_ASSOC == 2
#define ICACHE_TOUCH(lruword, set)    lruword = set;
#define ICACHE_INIT_LRU(lruword)      lruword = 0;
#define ICACHE_LRU(lruword)           (!lruword)
#define ICACHE_MAKE_LRU(lruword, set) lruword = !(set);
#endif

#if ICACHE_ASSOC == 4
#define ICACHE_TOUCH(lruword, set)    UPDATE_BIG_LRU(lruword, set)
#define ICACHE_INIT_LRU(lruword)      INIT_BIG_LRU(lruword)
#define ICACHE_LRU(lruword)           GET_BIG_LRU(lruword, ICACHE_ASSOC)
#define ICACHE_MAKE_LRU(lruword,set)  SET_BIG_LRU4(lruword, set)
#endif

#if ICACHE_ASSOC == 8
#define ICACHE_TOUCH(lruword, set)    UPDATE_BIG_LRU(lruword, set)
#define ICACHE_INIT_LRU(lruword)      INIT_BIG_LRU(lruword)
#define ICACHE_LRU(lruword)           GET_BIG_LRU(lruword, ICACHE_ASSOC)
#define ICACHE_MAKE_LRU(lruword,set)  SET_BIG_LRU8(lruword, set)
#endif

#if DCACHE_ASSOC == 1
#define DCACHE_TOUCH(lruword, set)    /* nop */
#define DCACHE_INIT_LRU(lruword)      lruword = 0;
#define DCACHE_LRU(lruword)           0
#define DCACHE_MAKE_LRU(lruword, set) /* nop */
#endif

#if DCACHE_ASSOC == 2
#define DCACHE_TOUCH(lruword, set)    lruword = set;
#define DCACHE_INIT_LRU(lruword)      lruword = 0;
#define DCACHE_LRU(lruword)           !lruword
#define DCACHE_MAKE_LRU(lruword, set) lruword = !(set);
#endif

#if DCACHE_ASSOC == 4
#define DCACHE_TOUCH(lruword, set)    UPDATE_BIG_LRU(lruword, set)
#define DCACHE_INIT_LRU(lruword)      INIT_BIG_LRU(lruword)
#define DCACHE_LRU(lruword)           GET_BIG_LRU(lruword, DCACHE_ASSOC)
#define DCACHE_MAKE_LRU(lruword, set) SET_BIG_LRU4(lruword, set)
#endif

#if DCACHE_ASSOC == 8
#define DCACHE_TOUCH(lruword, set)    UPDATE_BIG_LRU(lruword, set)
#define DCACHE_INIT_LRU(lruword)      INIT_BIG_LRU(lruword)
#define DCACHE_LRU(lruword)           GET_BIG_LRU(lruword, DCACHE_ASSOC)
#define DCACHE_MAKE_LRU(lruword, set) SET_BIG_LRU8(lruword, set)
#endif

/**** just for laughs, here's another UPDATE_BIG_LRU routine, with a few
 * more instructions but no branches or if tests after the initial
 * check.  Probably runs faster on a dynamically-scheduled processor like T5.
 * Keep x axis of array as set number instead of lru position.
 *
 * {
 *    unsigned char* lru = (unsigned char*) lruword;
 *    register unsigned int l = lru[h];
 *    if (l != 0) {
 *      lru[0] += (lru[0] < l);
 *      lru[1] += (lru[1] < l);
 *      lru[2] += (lru[2] < l);
 *      lru[3] += (lru[3] < l);
 *      lru[h] = 0;
 *    }
 * }
 */

/*****************************************************************
 * EVENT SUPPORT
 *****************************************************************/
#define DATA_READ_EVENT() \
  TraceDataRef(&PE[cpuNum], vAddr, pAddr); \
  STATS_INC(cpuNum, dReads, 1); \
  if (++(dcache->memSample) >=  MS_SAMPLE_MEMOP_INTERVAL) { \
     dcache->memSample = 0; \
     MEM_SAMPLE_EVENT(CPUVec.CycleCount(cpuNum), cpuNum, CURRENT_PC(cpuNum), \
                      vAddr, pAddr, 1); \
  }

#define DATA_WRITE_EVENT() \
  TraceDataRef(&PE[cpuNum], vAddr, pAddr); \
  STATS_INC(cpuNum, dWrites, 1); \
  if (++CACHE[cacheNum].DCache.memSample >=  MS_SAMPLE_MEMOP_INTERVAL) { \
      CACHE[cacheNum].DCache.memSample = 0; \
      MEM_SAMPLE_EVENT(CPUVec.CycleCount(cpuNum), cpuNum, CURRENT_PC(cpuNum), \
                       vAddr, pAddr, 0); \
   }

/****************************************************************
 * MemRefInit
 *
 *****************************************************************/
void
MemRefInit(void)
{
   if (!strcmp(CACHE_MODEL, "None")) {
      CACHE = (Cache *)calloc(TOTAL_CPUS, sizeof(Cache));
      skipCaches = TRUE;
   } else {
      ASSERT(!strcmp(CACHE_MODEL, "2Level"));
      InitPCaches();
      InitSCaches();
      skipCaches = FALSE;
   }

   MemsysInit();
}

/*****************************************************************
 * MemRefResetStats
 *
 *****************************************************************/
void
MemRefResetStats(int cpuNum)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   int scacheNum = GET_SCACHE_NUM(cpuNum);

   bzero((char*)&(CACHE[cacheNum].stats), sizeof(CACHE[cacheNum].stats));
   bzero((char*)&(SCACHE[scacheNum].stats), sizeof(SCACHE[scacheNum].stats));
}


/*****************************************************************
 * MemRefPrintPeriodicStats
 *
 * This is called every STAT_INTERVAL.
 *****************************************************************/
static struct CacheStats {
   unsigned int ICacheCnt, ICacheMisses;
   unsigned int DCacheCnt, DCacheMisses;
   unsigned int SCacheCnt, SCacheMisses;
} oldstats[SIM_MAXCPUS];

static int printcnt;

void
MemRefPeriodicStats(int cpuNum)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   int scacheNum = GET_SCACHE_NUM(cpuNum);
   uint newICacheCnt, newICacheMisses;
   uint newDCacheCnt, newDCacheMisses;
   uint newSCacheCnt, newSCacheMisses;
   uint totalICacheCnt, totalICacheMisses;
   uint totalDCacheCnt, totalDCacheMisses;
   uint totalSCacheCnt, totalSCacheMisses;
   struct CacheStats *s;
   Cache  *C  = &CACHE[cacheNum];
   SCache *SC = &SCACHE[scacheNum];

   totalICacheCnt = totalICacheMisses = 0;
   totalDCacheCnt = totalDCacheMisses = 0;
   totalSCacheCnt = totalSCacheMisses = 0;

   s = &(oldstats[cacheNum]);

   newICacheCnt = STATS_VALUE(cpuNum, iReads);
   totalICacheCnt += newICacheCnt;

   totalICacheMisses += newICacheMisses =
      (uint)C->stats.ICache.ReadMisses;

   totalDCacheCnt += newDCacheCnt = (uint)
      (STATS_VALUE(cpuNum, dReads) + STATS_VALUE(cpuNum, dWrites));

   totalDCacheMisses += newDCacheMisses = (uint)
      (C->stats.DCache.ReadMisses +
       C->stats.DCache.WriteMisses +
       C->stats.DCache.UpgradeMisses);

   totalSCacheCnt += newSCacheCnt = (uint)
      (SC->stats.Igets +
       SC->stats.Dgets +
       SC->stats.DgetXs +
       SC->stats.Dupgrades);

   totalSCacheMisses += newSCacheMisses = (uint)
      (SC->stats.IgetMisses +
       SC->stats.DgetMisses +
       SC->stats.DgetXMisses +
       SC->stats.DupgradeMisses);

#ifdef DELETE
   CPUPrint("C%d I %5.4f%% D %5.4f%% S %3.2f%% %3.2f%%\n",
            cacheNum,
            ((newICacheMisses - s->ICacheMisses == 0) ? 0 :
             ((100.0 * (newICacheMisses  - (uint)s->ICacheMisses)) /
              (newICacheCnt - (uint)s->ICacheCnt))),
            ((newDCacheMisses - (uint)s->DCacheMisses == 0) ? 0 :
             ((100.0 * (newDCacheMisses - (uint)s->DCacheMisses)) /
              (newDCacheCnt - (uint)s->DCacheCnt))),
            ((newSCacheMisses - (uint)s->SCacheMisses == 0) ? 0 :
             ((100.0 * (newSCacheMisses - (uint)s->SCacheMisses)) /
              (newSCacheCnt - (uint)s->SCacheCnt))));
#endif


   s->ICacheMisses = newICacheMisses;
   s->DCacheCnt    = newDCacheCnt;
   s->DCacheMisses = newDCacheMisses;
   s->SCacheMisses = newSCacheMisses;
   s->SCacheCnt    = newSCacheCnt;

   printcnt++;

   CPUPrint("C%d Total I Misses: %lld Total I Refs: %lld Rate: %5.3f%%\n",
              cacheNum,
              (uint64)totalICacheMisses,
              (uint64)totalICacheCnt,
              totalICacheCnt ? (100.0 *  totalICacheMisses)/totalICacheCnt : 0.0);
   CPUPrint("C%d Total D Misses: %lld Total D Refs: %lld Rate: %5.3f%%\n",
              cacheNum,
              (uint64)totalDCacheMisses,
              (uint64)totalDCacheCnt,
              totalDCacheCnt ? (100.0 *  totalDCacheMisses)/totalDCacheCnt : 0.0);
   CPUPrint("C%d Total S Misses: %lld Total S Refs: %lld Rate: %5.3f%%\n",
              cacheNum,
              (uint64)totalSCacheMisses,
              (uint64)totalSCacheCnt,
              totalSCacheCnt ? (100.0 *  totalSCacheMisses)/totalSCacheCnt : 0.0);

#if (MHT_SIZE >= 4) && (SMHT_SIZE >= 4)
   {
      /*
       * These stats are currently MHT occupancy histograms over all
       * cpu's. They can be changed to per cpu if desireable.
       */
      int i, c;
      SimCounter occ[MHT_SIZE+1], socc[SMHT_SIZE+1];

      for (i = 0; i < MHT_SIZE+1; i++) {
         occ[i] = 0;
         for (c = 0; c < TOTAL_CPUS; c++) {
            occ[i] += CACHE[c].stats.mhtOccupancyHist[i];
         }
      }

      for (i = 0; i < SMHT_SIZE+1; i++) {
         socc[i] = 0;
         for (c = 0; c < TOTAL_CPUS; c++) {
            socc[i] += SCACHE[c].stats.smhtOccupancyHist[i];
         }
      }

      CPUPrint("C%d MHT Occupancy: %lld %lld %lld %lld %lld\n",
                 cacheNum,
                 (uint64) occ[0], (uint64) occ[1],
                 (uint64) occ[2], (uint64) occ[3], (uint64) occ[4]);
      CPUPrint("C%d SMHT Occupancy: %lld %lld %lld %lld %lld\n",
                 cacheNum,
                 (uint64) socc[0], (uint64) socc[1],
                 (uint64) socc[2], (uint64) socc[3], (uint64) socc[4]);
   }
#endif
   /* You could print periodic memsys stats here if they existed */
}

/*****************************************************************
 * MemRefDumpStats
 *****************************************************************/
void
MemRefDumpStats(int cpuNum)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   int scacheNum = GET_SCACHE_NUM(cpuNum);
   int j;

   Cache  *C = &(CACHE[cacheNum]);
   SCache *SC = &(SCACHE[scacheNum]);
#ifdef SOLO
   CPUPrint("C%d libc Wait Time: %lld\n", cpuNum, STATS_VALUE(cpuNum, libcWaitTime));
#endif

   CPUPrint("C%d I Misses: %lld I Refs: %lld\n",
            cacheNum,
            (uint64)C->stats.ICache.ReadMisses,
            (uint64)STATS_VALUE(cpuNum, iReads));
   CPUPrint("C%d I Inval %lld LinesInval %lld\n",
            cacheNum,
            (uint64)C->stats.ICache.Inval,
            (uint64)C->stats.ICache.LinesInval);
   CPUPrint("C%d D Read Misses: %lld Reads: %lld\n",
            cacheNum,
            (uint64)C->stats.DCache.ReadMisses,
            (uint64)STATS_VALUE(cpuNum, dReads));
   CPUPrint("C%d D Write Misses: %lld Writes: %lld\n",
            cacheNum,
            (uint64)C->stats.DCache.WriteMisses,
            (uint64)STATS_VALUE(cpuNum, dWrites));
   CPUPrint("C%d D Upgrades %lld Writebacks %lld \n",
            cacheNum,
            (uint64)C->stats.DCache.UpgradeMisses,
            (uint64)C->stats.DCache.Writebacks);
   CPUPrint("C%d D Inval %lld LinesInval %lld Dwngrd %lld LinesWback %lld\n",
            cacheNum,
            (uint64)C->stats.DCache.Inval,
            (uint64)C->stats.DCache.LinesInval,
            (uint64)C->stats.DCache.Downgrade,
            (uint64)C->stats.DCache.LinesWriteback);

   CPUPrint("C%d IGet Retries %lld\n",
            cacheNum,
            (uint64)C->stats.ICache.RetriedIGets);

   CPUPrint("C%d DGet Retries %lld\n",
            cacheNum,
            (uint64)C->stats.DCache.RetriedDGets);

   CPUPrint("C%d DGetX Retries %lld\n",
            cacheNum,
            (uint64)C->stats.DCache.RetriedDGetXs);

   for (j = 0; j < MHT_SIZE+1; j++) {
      CPUPrint("C%d MHT%d %lld\n",  cacheNum,  j,
               (uint64)C->stats.mhtOccupancyHist[j]);
   }

   CPUPrint("C%d S IMisses %lld IRefs: %lld\n",
            cacheNum,
            (uint64)SC->stats.IgetMisses,
            (uint64)SC->stats.Igets);

   CPUPrint("C%d S DMisses: GetMisses %lld GetRefs %lld\n",
            cacheNum,
            (uint64)SC->stats.DgetMisses,
            (uint64)SC->stats.Dgets);

   CPUPrint("C%d S DMisses: GetXMisses %lld GetXRefs %lld\n",
            cacheNum,
            (uint64)SC->stats.DgetXMisses,
            (uint64)SC->stats.DgetXs);

   CPUPrint("C%d S NAKs: %lld Merges: %lld\n",
            cacheNum,
            (uint64)SC->stats.NAKs,
            (uint64)SC->stats.MergeMisses);

   CPUPrint("C%d S UpgMisses: %lld UpgRefs: %lld (%.2f%%) Wbacks %lld Replace %lld\n",
            cacheNum,
            (uint64)SC->stats.DupgradeMisses,
            (uint64)SC->stats.Dupgrades,
            (SC->stats.Dupgrades>0 ?
             (100.0 * ((double)SC->stats.DupgradeMisses /
                       (double)SC->stats.Dupgrades)): (double)-1.0),
            (uint64)SC->stats.Writebacks,
            (uint64)SC->stats.Replacements);

   CPUPrint("C%d S Inval %lld LinesInval %lld Dwngrd %lld LinesWback %lld\n",
            cacheNum,
            (uint64)SC->stats.Inval,
            (uint64)SC->stats.LinesInval,
            (uint64)SC->stats.Downgrade,
            (uint64)SC->stats.LinesWriteback);

   for (j = 0; j < SMHT_SIZE+1; j++) {
      CPUPrint("C%d SMHT%d %lld\n",  cacheNum,  j,
               (uint64)SC->stats.smhtOccupancyHist[j]);
   }

   CPUPrint("C%d S IGetMHTRetries %lld DGetMHTRetries %lld DGetXMHTRetries %lld DUGetXMHTRetries %lld\n",
            cacheNum,
            (uint64)SC->stats.IGetMHTRetries,
            (uint64)SC->stats.DGetMHTRetries,
            (uint64)SC->stats.DGetXMHTRetries,
            (uint64)SC->stats.DUGetXMHTRetries);

   CPUPrint("C%d S Prefetches %lld\n",
            cacheNum,
            (uint64)SC->stats.Prefetches);

#ifdef HWBCOPY
   if (SimConfigGetBool("Hwbcopy.Streaming")) {
      for (j = 1; j <= SimConfigGetInt("Hwbcopy.StreamDepth"); j++) {
         CPUPrint("C%d S StreamGets[%d] %lld\n",
                  cacheNum, j,
                  (uint64)SC->stats.StreamGets[j]);
         CPUPrint("C%d S StreamGetXs[%d] %lld\n",
                  cacheNum, j,
                  (uint64)SC->stats.StreamGetXs[j]);
      }
   }
#endif
   if (cpuNum == (TOTAL_CPUS-1)) {
      memsysVec.MemsysDumpStats();
   }
}

/*****************************************************************
 * InitPCaches
 * Initialize the primary caches.
 *****************************************************************/
void
InitPCaches(void)
{
   int i,j,k;
   int log2ICACHE_ASSOC = GetLog2(ICACHE_ASSOC);
   int log2DCACHE_ASSOC = GetLog2(DCACHE_ASSOC);

   if ( ICACHE_LINE_SIZE > SCACHE_LINE_SIZE ) {
      CPUError("ICache.LineSize must be smaller than SCacheLineSize\n");
   }
   if ( DCACHE_LINE_SIZE > SCACHE_LINE_SIZE ) {
      CPUError("DCache.LineSize must be smaller than SCacheLineSize\n");
   }
   if ( machines.DCacheAssoc != DCACHE_ASSOC ) {
      CPUError("DCache.Assoc !=%i (not implemented) \n",DCACHE_ASSOC);
   }
   if ( machines.ICacheAssoc != ICACHE_ASSOC ) {
      CPUError("ICache.Assoc !=%i (not implemented) \n",ICACHE_ASSOC);
   }

#ifdef DATA_HANDLING
   if ( DCACHE_HIT_ALWAYS || ICACHE_HIT_ALWAYS ) {
      CPUError("ICache.AHit and DCache.AMiss are not supported with data handling!\n");
   }
#endif
   if ( DCACHE_MISS_ALWAYS || ICACHE_MISS_ALWAYS ) {
      if ( SCACHE_HIT_TIME != 0 ) {
         CPUError("ICacheAMiss || DCacheAMiss set. SCacheHitTime must be set to 0!\n");
      }
   }
   if ( ICACHE_HIT_ALWAYS && ICACHE_MISS_ALWAYS) {
      CPUError("Make up your mind: ICache.HitAlways OR ICache.MissAlways\n");
   }
   if ( DCACHE_HIT_ALWAYS && DCACHE_MISS_ALWAYS) {
      CPUError("Make up your mind: DCache.HitAlways OR DCache.MissAlways\n");
   }

   /* Set up variables for the cache macros... it's better to compute
      these once here than every time we do a lookup. */

   iTagShift    = log2ICACHE_SIZE - log2ICACHE_ASSOC;
   iAddrDivisor = ICACHE_SIZE/ICACHE_ASSOC;
   iIndexMask   = ICACHE_INDEX - 1;

   dTagShift    = log2DCACHE_SIZE - log2DCACHE_ASSOC;
   dAddrDivisor = DCACHE_SIZE/DCACHE_ASSOC;
   dIndexMask   = DCACHE_INDEX - 1;

   CACHE = (Cache *)calloc(TOTAL_CPUS, sizeof(Cache));

   CPUPrint("ICACHE: Assoc = %d Size = %#x Line = %d\n",
            ICACHE_ASSOC, ICACHE_SIZE, ICACHE_LINE_SIZE);

   CPUPrint("DCACHE: Assoc = %d Size = %#x Line = %d\n",
            DCACHE_ASSOC, DCACHE_SIZE, DCACHE_LINE_SIZE);

   CPUPrint("SIMOS running with a write buffer size %d!\n",
            WRITE_BUFFER_SIZE);

   for (j=0; j<TOTAL_CPUS; j++) {
      /* Allocate and initialize the ICaches */
      CACHE[j].ICache.set =
         (struct ICacheSet *)calloc(ICACHE_INDEX,sizeof(struct ICacheSet));

      CACHE[j].DCache.set =
         (struct DCacheSet *)calloc(DCACHE_INDEX,sizeof(struct DCacheSet));

      for (i=0; i<ICACHE_INDEX; i++) {
         for (k=0; k<ICACHE_ASSOC; k++) {
            CACHE[j].ICache.set[i].tags[k] = INVALID_TAG;
         }
         ICACHE_INIT_LRU(CACHE[j].ICache.set[i].LRU);
      }

      /* Initialize the DCaches */
      for (i=0; i<DCACHE_INDEX; i++) {
         for (k=0; k<DCACHE_ASSOC; k++) {
            CACHE[j].DCache.set[i].tags[k] = INVALID_TAG;
         }
         DCACHE_INIT_LRU(CACHE[j].DCache.set[i].LRU);
      }

      /* Initialize the Miss Handling Tables */
      for (i=0; i<MHT_SIZE; i++) {
         CACHE[j].MHT[i].inuse = FALSE;
         CACHE[j].MHT[i].writeBuffer =
            MemAlign(sizeof(uint64), sizeof(*CACHE[j].MHT[i].writeBuffer));
         for (k = 0; k < 128; k++) {
            CACHE[j].MHT[i].writeBuffer->mask[k] = 0;
         }
      }
      CACHE[j].MHTnumInuse = 0;
      CACHE[j].activeWriteBuffers = 0;
   }

   useWriteBuffer = (WRITE_BUFFER_SIZE > 0);

   if ((TOTAL_CPUS == 1) && !UPGRADES_ON_UP) {
      /* *** Upgrades are not needed on a UP.  But some protocols (such as
       * flash) require them so they are configurable.
       */
      noUpgrades = TRUE;
   }
}

/*****************************************************************
 * MemRefPrefetch
 *
 *****************************************************************/
Result
MemRefPrefetch(int cpuNum, VA vAddr, PA pAddr, int hint)
{
   PFResult pfRet = PF_SUCCESS;

   switch (hint) {
   case 0: case 4: case 6:
      if (!STATS_VALUE(cpuNum, prefMHTStallStart)) {
         STATS_INC(cpuNum, prefStats.prefs, 1);
      } else {
         STATS_ADD_INTERVAL(cpuNum, prefStats.prefMHTStallTime, prefMHTStallStart);
      }
      pfRet = SCachePrefetch(cpuNum, vAddr, pAddr, MEMSYS_GET);
      {
         switch (pfRet) {
         case PF_SUCCESS:
            /* prefetch was immediately available from the memory system */
            STATS_INC(cpuNum, prefStats.prefL1Hits, 1);
            break;
         case PF_RESIDENT:
            STATS_INC(cpuNum, prefStats.prefL2Hits, 1);
            break;
         case PF_MERGE:
            STATS_INC(cpuNum, prefStats.prefMerges, 1);
            break;
         case PF_STALL:
            STATS_INC(cpuNum, prefStats.prefStalls, 1);
            break;
         case PF_FAILURE:
            STATS_INC(cpuNum, prefStats.prefMHTStall, 1);
            STATS_SET(cpuNum, prefMHTStallStart, CPUVec.CycleCount(cpuNum));
            return FAILURE;

         case PF_UPGRADE:
            STATS_INC(cpuNum, prefStats.prefUpgrades, 1);
            break;
         default:
            CPUWarning("Bad PREF return value\n");
         }
      }
      break;
   case 1: case 5: case 7:
      if (!STATS_VALUE(cpuNum, prefMHTStallStart)) {
         STATS_INC(cpuNum, prefStats.prefXs, 1);
      } else {
         STATS_ADD_INTERVAL(cpuNum, prefStats.prefXMHTStallTime, prefMHTStallStart);
      }

      pfRet = SCachePrefetch(cpuNum, vAddr, pAddr, MEMSYS_GETX);
      {
         switch (pfRet) {
         case PF_SUCCESS:
            /* prefetch was immediately available from
               the memory system */
            STATS_INC(cpuNum, prefStats.prefXL1Hits, 1);
            break;
         case PF_RESIDENT:
            STATS_INC(cpuNum, prefStats.prefXL2Hits, 1);
            break;
         case PF_MERGE:
            STATS_INC(cpuNum, prefStats.prefXMerges, 1);
            break;
         case PF_STALL:
            /* normal case: prefetch out in memsys */
            STATS_INC(cpuNum, prefStats.prefXStalls, 1);
            break;
         case PF_FAILURE:
            STATS_INC(cpuNum, prefStats.prefXMHTStall, 1);
            STATS_SET(cpuNum, prefMHTStallStart,CPUVec.CycleCount(cpuNum));
            return FAILURE;

         case PF_UPGRADE:
            STATS_INC(cpuNum, prefStats.prefXUpgrades, 1);
            break;
         default:
            CPUWarning("Bad PREF return value\n");
         }
      }
      break;
   default:
      CPUWarning("Unknown hint in PREF at %#x\n", vAddr);
      break;
   }
   return SUCCESS;
}

/*****************************************************************
 * Implementation of supported cache ops
 *****************************************************************/
Result
MemRefIndexHitWBInval(int cpuNum, VA vAddr, PA pAddr)
{
    int    ret;
    int    wasDirty;
    PA     realPA;
    byte   data[128];
    Result res = SUCCESS;

    ret = CacheExtractIndex(cpuNum, pAddr, SCACHE_LINE_SIZE,
                            &wasDirty, &realPA, data);
    if (ret) {
        /* Issue bogus command (get won't occur because of the -1),
           but with an associated writeback */
        if (wasDirty) {
            res = memsysVec.MemsysCmd(cpuNum, MEMSYS_GET, 0LL, -1,
                                   realPA, TRUE, data);
        } else {
            res = memsysVec.MemsysCmd(cpuNum, MEMSYS_GET, 0LL, -1,
                                   realPA, FALSE, data);
        }
    }
    return res;
}

Result
MemRefHitWBInval(int cpuNum, VA vAddr, PA pAddr)
{
    int    ret;
    int    wasDirty;
    byte   data[128];
    Result res = SUCCESS;

    ret = CacheExtract(cpuNum, pAddr, SCACHE_LINE_SIZE, &wasDirty, data);

    if (ret) { /* If line found */
        /* Issue bogus command (get won't occur because of the -1),
           but with an associated writeback */
        if (wasDirty) {
            res = memsysVec.MemsysCmd(cpuNum, MEMSYS_GET, 0LL, -1,
                                   pAddr, TRUE, data);
        } else {
            res = memsysVec.MemsysCmd(cpuNum, MEMSYS_GET, 0LL, -1,
                                   pAddr, FALSE, data);
        }
    }
    return res;
}


/*****************************************************************
 * MemRefReadInst
 *
 * Main entry into instruction cache from CPU
 * Return values:
 *          SUCCESS - Request hit in the cache, instruction returned.
 *          STALL   - Request missed in the cache, sent to the 2nd-level or memsys.
 *          FAILURE - Request missed but couldn't be issued to 2nd-level or
 *                    memory system. Caller must retry.
 *          BUSERROR - Request suffered a bus error.
 *
 * Simulate a two-way set assoc physically index ICache.
 * XXX - Should make this virtually index, physically tagged.
 *****************************************************************/
Result
MemRefReadInst(int cpuNum, VA vAddr, PA pAddr, Inst *inst)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   int ind  = ICACHE_INDEXOF(pAddr);
   PA tag = ICACHE_TAG(pAddr);
   struct ICacheSet* set = & CACHE[cacheNum].ICache.set[ind];
   Result ret;
   int lineindex, setindex;

   if ( ICACHE_HIT_ALWAYS ) {
      setindex = 0;
      set->data[setindex] = (byte*)CPUVec.PAToHostAddr(cpuNum,
                                                pAddr&~(PA)(ICACHE_LINE_SIZE-1),
                                                vAddr);
      lineindex = pAddr & (ICACHE_LINE_SIZE-1);
      *inst =  *(Inst *) (set->data[setindex]+ lineindex);
      QUICK_ICACHE_SET(cpuNum, pAddr, set, setindex);
      return SUCCESS;
   }

   if ( !ICACHE_MISS_ALWAYS ) {
      if (set->tags[0] == tag) {
         ICACHE_TOUCH(set->LRU, 0);
         setindex=0;
         MS_INSTRUCTION_SET(cpuNum,0);
         QUICK_ICACHE_SET(cpuNum, pAddr, set, setindex);
         goto hit;
      }

#if ICACHE_ASSOC > 1
      if (set->tags[1] == tag) {
         ICACHE_TOUCH(set->LRU, 1);
         setindex=1;
         MS_INSTRUCTION_SET(cpuNum,1);
         QUICK_ICACHE_SET(cpuNum, pAddr, set, setindex);
         goto hit;
      }
#endif

#if ICACHE_ASSOC > 2
      if (set->tags[2] == tag) {
         ICACHE_TOUCH(set->LRU, 2);
         setindex=2;
         MS_INSTRUCTION_SET(cpuNum,2);
         QUICK_ICACHE_SET(cpuNum, pAddr, set, setindex);
         goto hit;
      }
#endif

#if ICACHE_ASSOC > 3
      if (set->tags[3] == tag) {
         ICACHE_TOUCH(set->LRU, 3);
         setindex=3;
         MS_INSTRUCTION_SET(cpuNum,3);
         QUICK_ICACHE_SET(cpuNum, pAddr, set, setindex);
         goto hit;
      }
#endif
   }

   ret = ICacheMiss(cpuNum, vAddr, pAddr, &setindex);
   if (ret != SUCCESS) {
      return ret;
   } else {
      goto hit;
   }

hit:
   MS_INSTRUCTION(cpuNum, pAddr);
   lineindex = pAddr & (ICACHE_LINE_SIZE-1);
   *inst =  *(Inst *) (set->data[setindex] + lineindex);

   return SUCCESS;
}

/*****************************************************************
 * ICacheMiss
 *
 * XXX - Should make this virtually index, physically tagged.
 *****************************************************************/
Result
ICacheMiss(int cpuNum, VA vAddr, PA pAddr, int *setindex)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   int       mhtind;
   int       ind = ICACHE_INDEXOF(pAddr);
   int       lru;
   MHTStatus mhtret;
   SCResult  sret;

   if (VERBOSE_DEBUG) {
      CPUPrint("%d: ICacheMiss: VA: %8.8lx  PA: %8.8lx\n",cpuNum, vAddr, pAddr);
   }

#ifdef MIPSY_MXS
   if (!MxsApproveImiss(PE[cpuNum].st, pAddr)) {
       /* This means that the current Imiss could cause a previous load or
        * store to be aborted due to a coherency exception.  We stall the
        * miss until the condition clears.  This is a performance optimization
        * that becomes necessary if we speculate through SC instructions.
        */
       return FAILURE;
   }
#endif /* MIPSY_MXS */
   /*
    * Check the second level cache for the line. Before we
    * record the miss in the ICache MHT entry make sure
    * this isn't a re-issued ICache miss from a CONFLICT case.
    */
   lru = ICACHE_LRU(CACHE[cacheNum].ICache.set[ind].LRU);
   mhtret = AllocMHT(cpuNum, SC_IGET, vAddr,
                     pAddr,
                     ICACHE_LINE_SIZE, lru, &mhtind);
   switch (mhtret) {
   case MHTFULL:
      return FAILURE;
   case MHTCONFLICT:
      return FAILURE;
   case MHTMERGE:
      /*
       * This can occur if someone does a data acces to a line
       * and then jumps to it.
       */
      return STALL;
   default:
      ASSERT(mhtret == MHTSUCCESS);
      break;
   }

   if (CACHE[cacheNum].ICache.set[ind].tags[lru] != INVALID_TAG) {
      uint type = E_L1 | E_I | E_READ | E_FLUSH_CLEAN;
      PA replPA = ITAG_TO_PA(CACHE[cacheNum].ICache.set[ind].tags[lru], ind);

      L1_LINE_TRANS_EVENT(cpuNum, replPA, type, lru,
                          IS_KUSEG(CURRENT_PC(cpuNum)));
   }

   CACHE[cacheNum].ICache.set[ind].tags[lru] = INVALID_TAG;

   /*
    * Forward the request on to the secondary cache.
    */
   sret = SCacheFetch(cpuNum, vAddr, pAddr, SC_IGET, mhtind);

   if (sret == SCRETRY) {
      /*
       * We must of conflicted with a data miss. Deallocate
       * MHT entry and stall waiting for the SMHT to clear up.
       * XXX - Should me reissue this ourselves?
       */
      FreeMHT(cpuNum, mhtind);
      return FAILURE;
   } else {
      if (sret == SCBUSERROR) {
         FreeMHT(cpuNum, mhtind);
         return BUSERROR;
      }
   }

   /* Due to the MHT and the fact that the scache will call ICachePUT
      and will fill in the pcache, this routine will only be called on
      once on the miss to the icache line. */
   CACHE[cacheNum].stats.ICache.ReadMisses++;
   if (sret == SCSTALL) {
      return STALL;
   }
   /* SCacheFetch may return SCSUCCESS if zero latency misses
    * are configured. Record this as a miss but don't stall.
    */
   (*setindex) = lru;
   ASSERT(sret == SCSUCCESS);
   return SUCCESS;
}

/*****************************************************************
 * ICacheFlush
 *****************************************************************/
void
ICacheFlush(int cpuNum, PA paddr, int size)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   PA a;
   int ind;
   int  set;

   QUICK_ICACHE_CLEAR(cpuNum);

   CACHE[cacheNum].stats.ICache.Inval++;
   for (a = paddr; a < paddr + size; a += ICACHE_LINE_SIZE) {
      ind = ICACHE_INDEXOF(a);
      if (CACHE[cacheNum].ICache.set[ind].tags[0] == ICACHE_TAG(a)) {
         set = 0;
         goto foundit;
      }

#if ICACHE_ASSOC > 1
      if (CACHE[cacheNum].ICache.set[ind].tags[1] == ICACHE_TAG(a)) {
         set = 1;
         goto foundit;
      }
#endif
#if ICACHE_ASSOC > 2
      if (CACHE[cacheNum].ICache.set[ind].tags[2] == ICACHE_TAG(a)) {
         set = 2;
         goto foundit;
      }
#endif
#if ICACHE_ASSOC > 3
      if (CACHE[cacheNum].ICache.set[ind].tags[3] == ICACHE_TAG(a)) {
         set = 3;
         goto foundit;
      }
#endif
      continue;  /* not found, go around loop */

 foundit:
      CACHE[cacheNum].stats.ICache.LinesInval++;
      CACHE[cacheNum].ICache.set[ind].tags[set] = INVALID_TAG;
      ICACHE_MAKE_LRU(CACHE[cacheNum].ICache.set[ind].LRU, set);

      L1_LINE_TRANS_EVENT(cpuNum, a,
                          (E_L1 | E_I | E_READ | E_EXTERNAL | E_FLUSH_CLEAN), 0,
                          IS_KUSEG(CURRENT_PC(cpuNum)));

   }
}


/*****************************************************************
 * AllocMHT
 *
 * Allocate an entry in the miss handling table for the
 * specified first level miss. This routine is responsible for
 * checking for conflict and merge miss as well.
 *****************************************************************/
static MHTStatus
AllocMHT(int cpuNum, SCacheCmd cmd, VA vAddr, PA pAddr, int size,
         int lru, int *mhtind)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   int entry;
   int ind,cacheLineSize;
   if (cmd == SC_IGET) {
      ind = ICACHE_INDEXOF(pAddr);
      cacheLineSize = ICACHE_LINE_SIZE;
   } else {
      ind = DCACHE_INDEXOF(pAddr);
      cacheLineSize = DCACHE_LINE_SIZE;
   }
   ASSERT( size == cacheLineSize );

   if (CACHE[cacheNum].MHTnumInuse == 0) {
      entry = 0;  /* Special case for speed: allocate when MHT is empty */
   } else {
      int numChecked = CACHE[cacheNum].MHTnumInuse;
      int i;
      entry = -1;

      for (i = 0; (i < MHT_SIZE) && numChecked; i++) {
         if (!CACHE[cacheNum].MHT[i].inuse) {
            entry = i;
         } else {
            numChecked--;
            if (CACHE[cacheNum].MHT[i].ind == ind) {
               if ( (cmd==SC_IGET && CACHE[cacheNum].MHT[i].cmd==SC_IGET) ||
                   (cmd!=SC_IGET && CACHE[cacheNum].MHT[i].cmd!=SC_IGET) ) {
                  *mhtind = i;
                  if (!SameCacheLine(CACHE[cacheNum].MHT[i].pAddr, pAddr,
                                     cacheLineSize)) {
                     /* Currently we only support one miss per cache index */
                     return MHTCONFLICT;
                  }
                  /* Except we call them merges if two misses happen to the same
                   * line */
                  if (CACHE[cacheNum].MHT[i].cmd != cmd) {
                     /* Call it a conflict if we need a GETX and only a
                      * GET is outstanding.
                      */
                     return MHTCONFLICT;
                  }
                  return MHTMERGE;
               }
            }
         }
      }
      if (entry == -1) {
         if (i == MHT_SIZE) {
            /* MHT is full */
            return MHTFULL;
         } else {
            entry = i;
            }
      }
   }

   CACHE[cacheNum].MHTnumInuse++;
   CACHE[cacheNum].MHT[entry].inuse = TRUE;
   CACHE[cacheNum].MHT[entry].startTime = CPUVec.CycleCount(cpuNum);
   CACHE[cacheNum].MHT[entry].cmd = cmd;
   CACHE[cacheNum].MHT[entry].vAddr = vAddr;
   CACHE[cacheNum].MHT[entry].pAddr = pAddr;
   CACHE[cacheNum].MHT[entry].PC = CURRENT_PC(cpuNum);
   CACHE[cacheNum].MHT[entry].ind = ind;
   CACHE[cacheNum].MHT[entry].lru = lru;
   CACHE[cacheNum].MHT[entry].writeBuffer->active = FALSE;
   *mhtind = entry;
#ifdef MIPSY_MXS
   CACHE[cacheNum].MHT[entry].cpu_action = GetMxsAction (PE[cpuNum].st);
#endif

   return MHTSUCCESS;
}

/*****************************************************************
 * FreeMHT - Delete a MHT entry
 *
 *****************************************************************/
static void
FreeMHT(int cpuNum, int entryNum)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);

   ASSERT(CACHE[cacheNum].MHT[entryNum].inuse);
   CACHE[cacheNum].MHT[entryNum].inuse = 0;
   CACHE[cacheNum].MHTnumInuse--;
   ASSERT(CACHE[cacheNum].MHTnumInuse >= 0);
   ASSERT(CACHE[cacheNum].MHT[entryNum].writeBuffer->active == FALSE);
}

/*****************************************************************
 * ICachePUT
 * Since this is the ICache, no real writing.
 *****************************************************************/
static void
ICachePUT(int cpuNum, MHT *mht, int mode)
{
   int result;
   int cacheNum = GET_CACHE_NUM(cpuNum);
   PA paddr = mht->pAddr;
   int ind  = ICACHE_INDEXOF(paddr);
   PA tag = ICACHE_TAG(paddr);
   int lru  = mht->lru;
   char *data;
   int way =0;

   ASSERT(mht->cmd & SC_IGET);
   ASSERT(CACHE[cacheNum].ICache.set[ind].tags[lru] == INVALID_TAG);

   /* NOTE: This call is no longer optional! We use it to find
      the scache line that will be used to satisfy the data part of
      the miss */
   result = IsInSCache(cpuNum, paddr, MEMSYS_SHARED, &data, &way);

   if (!result) {
      CPUError("ICachePUT didn't find line in SCache (with IsInSCache())");
   }

   CACHE[cacheNum].ICache.set[ind].tags[lru] = tag;

#ifdef DATA_HANDLING
   CACHE[cacheNum].ICache.set[ind].data[lru] =
      data+((paddr&(SCACHE_LINE_SIZE-1)&~(PA)(ICACHE_LINE_SIZE-1)));
   /* Point L1 to data location in L2 */
#else
   CACHE[cacheNum].ICache.set[ind].data[lru] =
      (byte*)CPUVec.PAToHostAddr(cpuNum, (paddr&~(PA)(ICACHE_LINE_SIZE-1)), 0);
#endif

}

/*****************************************************************
 * MemRefReadData
 * Return values:
 *          SUCCESS - Request hit in the cache, data returned.
 *          STALL   - Request missed in the cache, sent to the 2nd-level or memsys.
 *          FAILURE - Request missed but couldn't be issued to 2nd-level or
 *                    memory system. Caller must retry.
 *          BUSERROR - Request suffered a bus error.
 *
 * Main entry into 1st level data cache from CPU.
 *****************************************************************/
Result
MemRefReadData(int cpuNum, VA vAddr, PA pAddr, void *data,
               RefSize size, RefFlavor flavor)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   PA  tag = DCACHE_TAG(pAddr);
   int ind = DCACHE_INDEXOF(pAddr);
   struct DCache *dcache = &CACHE[cacheNum].DCache;
   struct DCacheSet *set = &(dcache->set[ind]);
   Result ret;
   int lineindex;
   int setindex;


   if ( DCACHE_HIT_ALWAYS ) {
      setindex = 0;
      set->data[setindex] =
         (byte*)CPUVec.PAToHostAddr(cpuNum, pAddr&~(PA)(DCACHE_LINE_SIZE-1), vAddr);
      goto hit;
   }

   if ( !DCACHE_MISS_ALWAYS ) {
      if (((~EXCLUSIVE_TAG) & (set->tags[0])) == tag) {
         DCACHE_TOUCH(set->LRU, 0);
         setindex=0;
         goto hit;
      }

#if DCACHE_ASSOC > 1
      if (((~EXCLUSIVE_TAG) & (set->tags[1])) == tag) {
         DCACHE_TOUCH(set->LRU, 1);
         setindex=1;
         goto hit;
      }
#endif

#if DCACHE_ASSOC > 2
      if (((~EXCLUSIVE_TAG) & (set->tags[2])) == tag) {
         DCACHE_TOUCH(set->LRU, 2);
         setindex=2;
         goto hit;
      }
#endif

#if DCACHE_ASSOC > 3
      if (((~EXCLUSIVE_TAG) & (set->tags[3])) == tag) {
         DCACHE_TOUCH(set->LRU, 3);
         setindex=3;
         goto hit;
      }
#endif
   }

   /* Cache misses go here */

   if (VERBOSE_DEBUG) {
      CPUPrint("%d: Read  D$ MISS VA: 0x%8.8lx  PA: 0x%8.8lx\n",
               cpuNum, vAddr, pAddr);
   }

   ret = DCacheReadMiss(cpuNum, vAddr, pAddr, data, size, flavor, &setindex);

   if (ret != SUCCESS) {
      return ret;
   } else {
      ASSERT(setindex >=0 && setindex < DCACHE_ASSOC);
      goto hit;
   }

hit:
   if ( !DCACHE_HIT_ALWAYS ) {
      if (useWriteBuffer && AddrIsInWriteBuffer(cpuNum, pAddr, size)) {
         /*
          * We currently don't implement any kind of load bypassing so
          * we stall when a load hits a buffered write word.
          */
         return STALL;
      }
   }

   DATA_READ_EVENT();

   FALSE_SHARING_ACCESS(cpuNum,set->fSharing[setindex],pAddr);

   lineindex = pAddr & (DCACHE_LINE_SIZE-1);

   switch (size) {

   case WORD_SZ:
      *(int *) data =  *(int *) (set->data[setindex] + lineindex);
      if (VERBOSE_DEBUG) {
         CPUPrint("%d: Read  D$ hit  VA: 0x%8.8lx  PA: 0x%8.8lx  WORD data: 0x%8.8lx lineindex=%d\n",
                  cpuNum, vAddr, pAddr, *(int *) (set->data[setindex] + lineindex), lineindex);
      }
      break;

   case BYTE_SZ:
      *(byte *) data =  *(byte *) (set->data[setindex] + lineindex);
      if (VERBOSE_DEBUG) {
         CPUPrint("%d: Read  D$ hit  VA: 0x%8.8lx  PA: 0x%8.8lx  BYTE data: 0x%2.2x lineindex=%d\n",
                  cpuNum, vAddr, pAddr, *(char *) (set->data[setindex] + lineindex), lineindex);
      }
      break;

   case HALF_SZ:
      *(short *) data =  *(short *) (set->data[setindex] + lineindex);
      if (VERBOSE_DEBUG) {
         CPUPrint("%d: Read  D$ hit  VA: 0x%8.8lx  PA: 0x%8.8lx  HALF data: 0x%8.8lx lineindex=%d\n",
                  cpuNum, vAddr, pAddr, *(short *) (set->data[setindex] + lineindex), lineindex);
      }
      break;

   case DOUBLE_SZ:
      *(uint64 *) data =  *(uint64 *) (set->data[setindex] + lineindex);
      if (VERBOSE_DEBUG) {
#ifdef __alpha
          CPUPrint("%d: Read D$ hit VA: 0x%8.8lx PA: 0x%8.8lx DOUBLE data: 0x%16.16lx lineindex=%d\n",
                   cpuNum, vAddr, pAddr, *(uint64 *) (set->data[setindex] + lineindex), lineindex);
#else
          CPUPrint("%d: Read D$ hit VA: 0x%8.8lx PA: 0x%8.8lx DOUBLE data: 0x%16.16llx\n",
                   cpuNum, vAddr, pAddr, *(uint64 *) (set->data[setindex] + lineindex));
#endif
      }
      break;

   default:
      CPUError("Bad size %d passed to ReadDCache\n", size);
      break;
   }
   return SUCCESS;
}

/*****************************************************************
 * DCacheReadMiss
 *
 *****************************************************************/
static Result
DCacheReadMiss(int cpuNum, VA vAddr, PA pAddr, void *data,
               RefSize size, RefFlavor flavor, int *indPtr)
{
   PA writebackAddr;
   int cacheNum = GET_CACHE_NUM(cpuNum);
   int  ind = (pAddr/DCACHE_LINE_SIZE) % DCACHE_INDEX;
   int mhtind;
   int lru;
   SCacheCmd cmd;
   MHTStatus mhtret;
   SCResult sret;

   if (noUpgrades) {
      cmd = (flavor == LL_FLAVOR ? SC_DLLGETX : SC_DGETX);
   } else {
      cmd = (flavor == LL_FLAVOR ? SC_DLLGET : SC_DGET);
   }

   lru  = DCACHE_LRU(CACHE[cacheNum].DCache.set[ind].LRU);

   mhtret = AllocMHT(cpuNum, cmd, vAddr,
                     pAddr, DCACHE_LINE_SIZE, lru, &mhtind);

   switch (mhtret) {
   case MHTFULL:
      return FAILURE;
   case MHTCONFLICT:
      return FAILURE;
   case MHTMERGE:
      (*indPtr) = mhtind;
      return STALL;
   default:
      ASSERT(mhtret == MHTSUCCESS);
      break;
   }

   if (VERBOSE_DEBUG) {
      CPUPrint("%d: DCacheReadMiss: VA: %8.8lx  PA: %8.8lx\n",cpuNum, vAddr,
               pAddr);
   }

   if (CACHE[cacheNum].DCache.set[ind].tags[lru] & EXCLUSIVE_TAG) {
      CACHE[cacheNum].stats.DCache.Writebacks++;

#ifdef DATA_HANDLING
      {
         int way;
         char *scacheData;

         /* Generate writeback address */
         writebackAddr = DTAG_TO_PA(CACHE[cacheNum].DCache.set[ind].tags[lru],
                                    ind);

         if (!(IsInSCache(cacheNum, writebackAddr, MEMSYS_EXCLUSIVE,
                          &scacheData, &way))) {
            CPUError("DCacheReadMiss didn't find excl. line in SCache for WB");
         }
      }
#endif

      {
         uint type = E_L1 | E_D | E_READ | E_WRITEBACK;
         PA replPA = DTAG_TO_PA(CACHE[cacheNum].DCache.set[ind].tags[lru], ind);

         L1_LINE_TRANS_EVENT(cpuNum, replPA, type, lru,
                             IS_KUSEG(CURRENT_PC(cpuNum)));
      }

   } else {

      if (!(CACHE[cacheNum].DCache.set[ind].tags[lru] & INVALID_TAG)) {
         /* Generate "replacement" address */
         writebackAddr = DTAG_TO_PA(CACHE[cacheNum].DCache.set[ind].tags[lru],
                                    ind);
         if (VERBOSE_DEBUG) {
            CPUPrint("%d: dcache discarding clean line %8.8lx\n",
                     cpuNum,writebackAddr);
         }
      }

      if (!(CACHE[cacheNum].DCache.set[ind].tags[lru] & INVALID_TAG)) {
         uint type = E_L1 | E_D | E_READ | E_FLUSH_CLEAN;
         PA replPA = DTAG_TO_PA(CACHE[cacheNum].DCache.set[ind].tags[lru], ind);

         L1_LINE_TRANS_EVENT(cpuNum, replPA, type, lru,
                             IS_KUSEG(CURRENT_PC(cpuNum)));
      }
   }

   CACHE[cacheNum].DCache.set[ind].tags[lru] = INVALID_TAG;

   sret = SCacheFetch(cpuNum, vAddr, pAddr, cmd, mhtind);

   if (sret == SCRETRY) {
      /*
       * We must of conflicted with a data miss. Clean MHT and
       * stall.
       * XXX should we reissue this ourselves?
       */
      FreeMHT(cpuNum, mhtind);
      return FAILURE;
   } else if (sret == SCBUSERROR) {
      FreeMHT(cpuNum, mhtind);
      return BUSERROR;
   }

   CACHE[cacheNum].stats.DCache.ReadMisses++;

   if (sret == SCSTALL) {
      (*indPtr) = mhtind;
      return STALL;
   }
   /*
    * SCacheFetch may return SUCCESS if zero latency misses
    * are configured. Record this as a miss but don't stall.
    */
   (*indPtr) = lru;
   return SUCCESS;
}


/*****************************************************************
 * DCachePUT
 *****************************************************************/
static void
DCachePUT(int cpuNum, MHT *mht, int mode)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   PA pAddr = mht->pAddr;
   int ind  = DCACHE_INDEXOF(pAddr);
   int lru  = mht->lru;
   int way  = 0;
   PA tag   = DCACHE_TAG(pAddr);
   char *data;

   /* NOTE: This call is no longer optional! We use it to find
      the SCache line that will be used to satisfy the data part of
      the miss */
   if (!(IsInSCache(cpuNum, pAddr, mode, &data, &way))) {
      CPUError("DCachePUT didn't find line in SCache (with IsInSCache())");
   }

   if (mode & MEMSYS_EXCLUSIVE)
      tag |= EXCLUSIVE_TAG;

   if (mht->cmd == SC_DSCUGETX
       && (CACHE[cacheNum].DCache.set[ind].tags[lru] & INVALID_TAG)) {
      CPUVec.ClearLockFlag(cpuNum);   /* SC fails due to a race */
   }

   /* We don't want to copy data if it is already in the DCache
    * exclusive.
    */

   if (!((CACHE[cacheNum].DCache.set[ind].tags[lru] & EXCLUSIVE_TAG) &&
         ((CACHE[cacheNum].DCache.set[ind].tags[lru] & ~EXCLUSIVE_TAG) ==
          (tag & ~EXCLUSIVE_TAG)))) {

      CACHE[cacheNum].DCache.set[ind].tags[lru] = tag;

#ifdef DATA_HANDLING
      CACHE[cacheNum].DCache.set[ind].data[lru] =
         data+((pAddr&(SCACHE_LINE_SIZE-1)&~(PA)(DCACHE_LINE_SIZE-1)));
      /* Point L1 to data location in L2 */
#else
      CACHE[cacheNum].DCache.set[ind].data[lru] =
         (byte*)CPUVec.PAToHostAddr(cpuNum, pAddr&~(PA)(DCACHE_LINE_SIZE-1), mht->vAddr);
#endif
      CACHE[cacheNum].DCache.set[ind].fSharing[lru] =
         SCacheLineToFalseSharing(cacheNum,pAddr,way);

   }
   if (mht->writeBuffer->active) {
       ASSERT(CACHE[cacheNum].DCache.set[ind].tags[lru] & EXCLUSIVE_TAG);
       RetireWriteBuffer(cpuNum, mht, CACHE[cacheNum].DCache.set[ind].data[lru]);
   }
   DCACHE_TOUCH(CACHE[cacheNum].DCache.set[ind].LRU, lru);

}

/***************************************************************
 * DCacheFlush
 *
 * This routine now takes two parameters: writeback and discard.
 * Writeback: Should I write the line back to the scache if
 *            it is dirty?
 * Retain:   After this call, should the line be retained in  my
 *            cache?  (If the line was exclusive and writeback
 *            is set, the line is downgraded to Shared).
 *
 * NOTE: writeback==0 && retain==1 is effectively a NOP.  We'll
 * support that, but I don't see why it should ever be used.
 ****************************************************************/

/* FIXTHIS: These statistics are mimimally correct, but not much more */
static void
DCacheFlush(int cpuNum, int writeback, int retain, PA paddr, int size)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   PA a;
   int ind, set;

   ASSERT (writeback || !retain);
   /* One of these two should be set */

   /* ASSERT for now */
   ASSERT((paddr & (DCACHE_LINE_SIZE-1)) == 0);
   if (writeback && retain) {
      CACHE[cacheNum].stats.DCache.Downgrade++;
   } else if (!writeback && !retain) {
      CACHE[cacheNum].stats.DCache.Inval++;
   }


   for (a = paddr; a < paddr + size; a += DCACHE_LINE_SIZE) {
      ind = DCACHE_INDEXOF(a);

      if ((CACHE[cacheNum].DCache.set[ind].tags[0] & (~EXCLUSIVE_TAG))
          == DCACHE_TAG(a)) {
         set = 0;
         goto foundit;
      }

#if DCACHE_ASSOC > 1
      if ((CACHE[cacheNum].DCache.set[ind].tags[1] & (~EXCLUSIVE_TAG)) == DCACHE_TAG(a)) {
         set = 1;
         goto foundit;
      }
#endif

#if DCACHE_ASSOC > 2
      if ((CACHE[cacheNum].DCache.set[ind].tags[2] & (~EXCLUSIVE_TAG)) == DCACHE_TAG(a)) {
         set = 2;
         goto foundit;
      }
#endif

#if DCACHE_ASSOC > 3
      if ((CACHE[cacheNum].DCache.set[ind].tags[3] & (~EXCLUSIVE_TAG)) == DCACHE_TAG(a)) {
         set = 3;
         goto foundit;
      }
#endif

      continue;   /* not present, go around the loop */

 foundit:
      if ((CACHE[cacheNum].DCache.set[ind].tags[set] & EXCLUSIVE_TAG) != 0) {
         /* Line is exclusive */
         if (writeback) {
            CACHE[cacheNum].stats.DCache.LinesWriteback++;
            CACHE[cacheNum].DCache.set[ind].tags[set] = DCACHE_TAG(a);

#ifdef DATA_HANDLING
            {
               char *data;
               int way;

               /* NOTE: This call is not optional for
                  DATA_HANDLING..we use it to find the scache line
                  that will be used to receive the writeback */
               if (!(IsInSCache(cpuNum, a, 0, &data, &way))) {
                  CPUError("DCacheFlush didn't find line in SCache");
               }
               /* Write the line back to the scache */
            }
#endif

            L1_LINE_TRANS_EVENT(cpuNum, a,
                                (E_L1 | E_D | E_EXTERNAL | E_DOWNGRADE), 0,
                                IS_KUSEG(CURRENT_PC(cpuNum)));
         }
      }

      if (!writeback && !retain) {
         uint type;

         CACHE[cacheNum].stats.DCache.LinesInval++;

         if (CACHE[cacheNum].DCache.set[ind].tags[set] & EXCLUSIVE_TAG) {
            type = E_L1 | E_D | E_EXTERNAL | E_WRITEBACK;
         } else {
            type = E_L1 | E_D | E_EXTERNAL | E_FLUSH_CLEAN;
         }
         L1_LINE_TRANS_EVENT(cpuNum, a, type, 0,
                             IS_KUSEG(CURRENT_PC(cpuNum)));
      }

      if (retain) {
         CACHE[cacheNum].DCache.set[ind].tags[set] &= ~EXCLUSIVE_TAG;
         /* Take away exclusive ownership */
      } else {
         CACHE[cacheNum].DCache.set[ind].tags[set] = INVALID_TAG;
      }

      DCACHE_MAKE_LRU(CACHE[cacheNum].DCache.set[ind].LRU, set);
   }
}


/****************************************************************
 * MemRefWriteData
 * Return values:
 *          SUCCESS - Request hit in the cache, data written.
 *          STALL   - Request missed in the cache, sent to the 2nd-level or memsys.
 *          FAILURE - Request missed but couldn't be issued to 2nd-level or
 *                    memory system. Caller must retry.
 *          BUSERROR - Request suffered a bus error.
 *          SCFAILURE - Was an SC and it failed.
 *
 * Main entry into 1st level data cache writing from the CPU
 *****************************************************************/
Result
MemRefWriteData(int cpuNum, VA vAddr, PA pAddr, uint64 data,
                RefSize size, RefFlavor flavor)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   PA     tag =  DCACHE_TAG_EX(pAddr);
   struct DCacheSet* set;
   bool   upgrade;
   int    way;
   int    mhtind = -1;
   int    lineindex;
   int    ind = DCACHE_INDEXOF(pAddr);

   set = & CACHE[cacheNum].DCache.set[ind];

   /* See if weve been stalled on the MHT, update stats if so */
   if (useWriteBuffer) {
      STATS_ADD_INTERVAL(cpuNum, writeBufferStats.writeMHTStallTime,
                         writeMHTStallStart);
   }

   if ( DCACHE_HIT_ALWAYS ) {
      way = 0;
      set->data[way] =
         (byte*)CPUVec.PAToHostAddr(cpuNum, pAddr&~(PA)(DCACHE_LINE_SIZE-1), vAddr);
      goto foundit;
   }
   if (!DCACHE_MISS_ALWAYS ) {
      if (set->tags[0] == tag) {
         way = 0;
         goto foundit;
      }
#if DCACHE_ASSOC > 1
      if (set->tags[1] == tag) {
         way = 1;
         goto foundit;
      }
#endif
#if DCACHE_ASSOC > 2
      if (set->tags[2] == tag) {
         way = 2;
         goto foundit;
      }
#endif
#if DCACHE_ASSOC > 3
      if (set->tags[3] == tag) {
         way = 3;
         goto foundit;
      }
#endif
   }

   if (set->tags[0] == DCACHE_TAG(pAddr)) {
      way = 0;
      upgrade = TRUE;
      goto miss_or_upgrade;
   }
#if DCACHE_ASSOC > 1
   if (set->tags[1] == DCACHE_TAG(pAddr)) {
      way = 1;
      upgrade = TRUE;
      goto miss_or_upgrade;
   }
#endif
#if DCACHE_ASSOC > 2
   if (set->tags[2] == DCACHE_TAG(pAddr)) {
      way = 2;
      upgrade = TRUE;
      goto miss_or_upgrade;
   }
#endif
#if DCACHE_ASSOC > 3
   if (set->tags[3] == DCACHE_TAG(pAddr)) {
      way = 3;
      upgrade = TRUE;
      goto miss_or_upgrade;
   }
#endif

   /****** a true miss (not an upgrade) ********/
   upgrade = FALSE;
   way = DCACHE_LRU(set->LRU);

   if (set->tags[0] == DCACHE_TAG(pAddr)) {
      way = 0;
      upgrade = TRUE;
      goto miss_or_upgrade;
   }
#if DCACHE_ASSOC > 1
   if (set->tags[1] == DCACHE_TAG(pAddr)) {
      way = 1;
      upgrade = TRUE;
      goto miss_or_upgrade;
   }
#endif
#if DCACHE_ASSOC > 2
   if (set->tags[2] == DCACHE_TAG(pAddr)) {
      way = 2;
      upgrade = TRUE;
      goto miss_or_upgrade;
   }
#endif

miss_or_upgrade:
   {
      Result ret;

#ifdef DEBUG_BUFFER
      CPUPrint("%d: Write D$ MISS VA: 0x%8.8lx  PA: 0x%8.8lx\n",
               cpuNum, vAddr, pAddr);

#endif

      ret = DCacheWriteMiss(cpuNum, vAddr, pAddr, size, flavor, set, way,
                            upgrade, &mhtind);

      if (ret == FAILURE) {
         if (flavor == SC_FLAVOR) {
            /* SAH: Satisfies problem with write buffer where an SC
               was being merged with a regular store in the MHT. */
            return SCFAILURE;
         } else {
            STATS_INC(cpuNum, writeBufferStats.writeMHTStall, 1);
            STATS_SET(cpuNum, writeMHTStallStart, CPUVec.CycleCount(cpuNum));
            return STALL;
         }
      }
      if (ret == SCFAILURE || ret == BUSERROR) {
         return ret;
      }

      if (ret == STALL) {
         /* Check to see if a write buffer can avoid stalling otherwise
          * stall. SC don't every get bufferred. */
         if (useWriteBuffer && !(flavor & SC_FLAVOR)) {
            if(AddToWriteBuffer(cpuNum, pAddr, data, size, mhtind))
               return SUCCESS;
            else {
               STATS_INC(cpuNum, writeBufferStats.writeMHTStall, 1);
               STATS_SET(cpuNum, writeMHTStallStart, CPUVec.CycleCount(cpuNum));
               return STALL;
            }
         } else {
            return STALL;
         }
      }
      ASSERT(ret == SUCCESS);
   }
   /* after satisfying the miss (either if it merged with another write in the
    * write buffer, or if running on a magic memory system), rejoin
    * the hit case and finish processing the access.
    */

foundit:
   DCACHE_TOUCH(set->LRU, way);
   DATA_WRITE_EVENT();

   lineindex = vAddr & (DCACHE_LINE_SIZE-1);

   FALSE_SHARING_ACCESS(cpuNum,set->fSharing[way],pAddr);
   FALSE_SHARING_MODIFY(cpuNum,pAddr);

   switch (size) {
   case WORD_SZ:
      if (VERBOSE_DEBUG) {
         CPUPrint("%d: WRITE D$ hit  VA: 0x%8.8lx  PA: 0x%8.8lx  writing WORD: 0x%8.8lx \n",
                  cpuNum, vAddr, pAddr, (uint) data);
      }

      *(uint *) (set->data[way] + lineindex) = (uint)data;
      break;

   case BYTE_SZ:
      if (VERBOSE_DEBUG) {
         CPUPrint("%d: WRITE D$ hit  VA: 0x%8.8lx  PA: 0x%8.8lx  writing BYTE: 0x%2.2x \n",
                  cpuNum, vAddr, pAddr, (char) data);
      }
      *(char *) (set->data[way] + lineindex) = (char)data;
      break;

   case HALF_SZ:
      if (VERBOSE_DEBUG) {
         CPUPrint("%d: WRITE D$ hit  VA: 0x%8.8lx  PA: 0x%8.8lx  writing HALF: 0x%8.8lx \n",
                  cpuNum, vAddr, pAddr, (short) data);
      }
      *(short *) (set->data[way] + lineindex) = (short)data;
      break;

   case DOUBLE_SZ:
      if (VERBOSE_DEBUG) {
#ifdef __alpha
          CPUPrint("%d: WRITE D$ hit VA: 0x%8.8lx PA: 0x%8.8lx write DBLE: 0x%16.16lx\n",
                   cpuNum, vAddr, pAddr, (uint64) data);
#else
          CPUPrint("%d: WRITE D$ hit VA: 0x%8.8lx PA: 0x%8.8lx write DBLE: 0x%16.16llx\n",
                   cpuNum, vAddr, pAddr, (uint64) data);
#endif
      }
      *(uint64 *) (set->data[way] + lineindex) = (uint64)data;
      break;

   default:
       CPUError("Bad size request to ReadDCache");
       break;
   }

   return SUCCESS;
}


/*********************************************************************
 * DCacheWriteMiss
 *********************************************************************/
static Result
DCacheWriteMiss(int cpuNum, VA vAddr, PA pAddr, RefSize size, RefFlavor flavor,
                struct DCacheSet* set, int way, bool upgrade, int* mergemhtp)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   SCResult sret;
   MHTStatus mhtret;
   int mhtind;
   SCacheCmd cmd = SC_NO_CMD;

   if (flavor != SC_FLAVOR) {
      if (upgrade) cmd = SC_DUGETX;
      else         cmd = SC_DGETX;
   } else {
      /* Handling of the SC instruction */
      if (upgrade) {
         cmd = SC_DSCUGETX;
      } else {
         /* actual write miss on an SC */
         if ((DCACHE_MISS_ALWAYS || SCACHE_ASSOC == 1)
             && CPUVec.GetLockFlag(cpuNum)) {
            /* Direct-maped L2-cache.
             * We do not rely on the presence in the cache, but rather
             * on the LLbit. Issue a GETX in this case.
             * Same thing if we bypass the L1-cache.
             */
            PA instrLine, dataLine;
            cmd = SC_DGETX;
            STATS_INC(cpuNum, syncStats.llscConflictSuccess, 1);

            instrLine = (CURRENT_PC(cpuNum) & ~(SCACHE_LINE_SIZE-1))
               & (SCACHE_SIZE-1);

            dataLine = (pAddr & ~(SCACHE_LINE_SIZE-1))
               & (SCACHE_SIZE-1);

            if ((!DCACHE_MISS_ALWAYS) && (instrLine != dataLine)
                && (IS_KSEG0(instrLine))) {
               CPUError("Direct-map LL/SC conflict, cpu=%i PC=0x%x pAddr=0x%x\n",
                        cpuNum, CURRENT_PC(cpuNum), pAddr);
            }
         }  else {
            /* SC instructions fail if the line is not in the cache. */
            return SCFAILURE;
         }
      }
   }
   if (cmd == SC_NO_CMD) CPUError("cmd unset");

   /* --- pAddr gets cache-line aligned in call to AllocMHT --- */
   mhtret = AllocMHT(cpuNum, cmd, vAddr, pAddr,
                     DCACHE_LINE_SIZE, way, &mhtind);
   *mergemhtp = mhtind;
   if (mhtret == MHTFULL || mhtret == MHTCONFLICT)  return FAILURE;
   if (mhtret == MHTMERGE) {
      return STALL;
   }
   ASSERT(mhtret == MHTSUCCESS);

   if (VERBOSE_DEBUG) {
      CPUPrint("%d: DCacheWriteMiss:  VA: %8.8lx  PA: %8.8lx\n",cpuNum,
               vAddr, pAddr);
   }

   /* Write back the current dcache line to make space for the fill!! */

   if (set->tags[way] & EXCLUSIVE_TAG) {
      CACHE[cacheNum].stats.DCache.Writebacks++;

#ifdef DATA_HANDLING
      {
         char *scacheData;
         int cacheindex;
         PA writebackAddr;
         int myWay=0;

         cacheindex = CACHE[cacheNum].MHT[mhtind].ind;
         writebackAddr = DTAG_TO_PA(set->tags[way],cacheindex);
         if (VERBOSE_DEBUG) {
            CPUPrint("%d: Spilling %8.8lx to make way\n",cpuNum,writebackAddr);
         }
         if (!(IsInSCache(cpuNum, writebackAddr, MEMSYS_EXCLUSIVE,
                          &scacheData, &myWay))) {
            CPUError("DCacheWriteMiss didn't find exc. line in SCache for WB");
         }
      }
#endif
   }

   if (!upgrade) {
      /* replacing the previous line in this way */
#ifdef MIPSY_MXS
       if (!(set->tags[way] & INVALID_TAG)) {
          PA paddr = DTAG_TO_PA(set->tags[way],
                                 (set - CACHE[cacheNum].DCache.set));
          DoMxsIntervention(PE[cpuNum].st, paddr, DCACHE_LINE_SIZE, 0);
       }
#endif

#ifdef DATA_HANDLING
       if (!(set->tags[way] & INVALID_TAG)) {
          PA flushAddr = DTAG_TO_PA(set->tags[way],
                                    (set - CACHE[cacheNum].DCache.set));

          if (VERBOSE_DEBUG) {
             CPUPrint("%d: dcache discarding line %8.8lx\n",
                      cpuNum,flushAddr);
          }
       }
#endif
      set->tags[way] = INVALID_TAG;
   }

   sret = SCacheFetch(cpuNum, vAddr, pAddr, cmd, mhtind);

   if (sret == SCRETRY) {
      /*
       * We must of conflicted with another miss. Clean MHT and stall.
       * XXX Is this right?
       */
      FreeMHT(cpuNum, mhtind);
      return FAILURE;
   } else if (sret == SCBUSERROR) {
      FreeMHT(cpuNum, mhtind);
      return BUSERROR;
   }

   if (!upgrade) {
      CACHE[cacheNum].stats.DCache.WriteMisses++;
   } else {
      CACHE[cacheNum].stats.DCache.UpgradeMisses++;
   }

   if (sret == SCSUCCESS) {
      /*
       *SCacheFetch may return SUCCESS if zero latency misses are configured.
       */
      return SUCCESS;
   }

   ASSERT(sret == SCSTALL);

   if (flavor == SC_FLAVOR) {
      /* Store conditionals are always blocking. This should be just
       * return STALL except somebody added the following... why?
       */
       *mergemhtp = mhtind;
      if (CACHE[cacheNum].MHT[mhtind].inuse) return STALL;
      else                                   return SCFAILURE;
   }
   *mergemhtp = mhtind;
   return STALL;
}

void
MemRefRemoveReq(int cpuNum, PA pAddr, int size)
{
    MHTRemoveReq(cpuNum, pAddr, SCACHE_LINE_SIZE);
}

#ifdef MIPSY_MXS
/*
 * MxsClassifyMiss - Return the memstat classification for the specified
 * miss.
 */
int
MxsClassifyMiss(int cpuNum,  VA vaddr, PA pAddr, bool isICache)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   int entry;
   int paddrMask = ~((isICache ? ICACHE_LINE_SIZE : DCACHE_LINE_SIZE)-1);
   int stallType;

   for (entry = 0; entry < MHT_SIZE; entry++) {
      if (!CACHE[cacheNum].MHT[entry].inuse)
         continue;
      if ((CACHE[cacheNum].MHT[entry].pAddr & paddrMask) != (pAddr & paddrMask))
         continue;

      stallType = isICache ? E_I : E_D;

      if (CACHE[cacheNum].MHT[entry].smhtEntry == SECOND_LEVEL_HIT_ENTRY) {
         stallType |= E_L1;
      } else {
         if ((CACHE[cacheNum].MHT[entry].cmd == SC_DUGETX) ||
             (CACHE[cacheNum].MHT[entry].cmd == SC_DSCUGETX)) {
            stallType |= E_L2 | E_UPGRADE;
         } else {
            stallType |= E_L2;
         }
      }
      return stallType;
   }

   CPUError("MxsClassifyMiss failed!\n");
   return 0;
}
#endif /* MIPSY_MXS */


/*****************************************************************
 * Routines for dealing with the miss handling table MHT that
 * supports lockup free caches.
 *****************************************************************/

/*****************************************************************
 * SyncCaches
 *
 *****************************************************************/
extern Result
MemRefSync(int cpuNum)
{
   /*
    * First make sure the MHT in the caches are empty then send a sync
    * to the memory system.
    */

   if (CACHE[cpuNum].MHTnumInuse != 0) {
      return STALL;
   }

   if (memsysVec.MemsysCmd != NULL) {
      return memsysVec.MemsysCmd(cpuNum, MEMSYS_SYNC, 0LL, 0, 0LL, 0, 0);
   } else {
      return SUCCESS;
   }
}


/*****************************************************************
 * MHTRemoveReq
 * Blindly remove any entries corresponding to this address and size
 * that are in the MHT.
 *****************************************************************/
#define SMALLEST_LINE_SIZE   ((ICACHE_LINE_SIZE < DCACHE_LINE_SIZE) \
                              ? ICACHE_LINE_SIZE : DCACHE_LINE_SIZE)

void
MHTRemoveReq(int cpuNum, PA  pAddr, int size)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   PA startPaddr, endPaddr;
   MHT *mht;
   int i;

   startPaddr = (pAddr/SMALLEST_LINE_SIZE)*SMALLEST_LINE_SIZE;
   endPaddr = (pAddr + size)+SMALLEST_LINE_SIZE-1;

   for (i = 0; i < MHT_SIZE; i++) {
      if (!CACHE[cacheNum].MHT[i].inuse) continue;
      mht = &CACHE[cacheNum].MHT[i];
      if ((mht->pAddr >= startPaddr) && (mht->pAddr < endPaddr)) {
         mht->smhtEntry = SECOND_LEVEL_INVALID_ENTRY;
         /* jmc - in the cases I've tested this mht is never in
          * the event queue.  Mendel says he thinks that should
          * only happen if the event is naked, and then only
          * for one cycle (it's the retry-next-cycle mechanism).
          * Remove for now.
          */
         /* EventCallbackRemove((EventCallbackHdr *) mht); */
         FreeMHT(cpuNum, mht - CACHE[cacheNum].MHT);
	 CPUVec.Unstall(cpuNum);
      }
   }
}

/*****************************************************************
 * MHTReqDoneCallback - callback for bus arb / utilization
 *
 *****************************************************************/
void
MHTReqDoneCallback(int cpuNum,EventCallbackHdr *hdr, void *arg)
{
  MHT *mht = (MHT *)hdr;

  MHTReqDone(cpuNum, mht, mht->mode, mht->status);
}

/*****************************************************************
 * MHTReqDone
 *
 *****************************************************************/
void
MHTReqDone(int cpuNum, MHT *mht, int mode, int status)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);

   if (status == MEMSYS_STATUS_SUCCESS) {
      if (mht->cmd & SC_IGET) {

         L1_IMISS_EVENT(CPUVec.CycleCount(cpuNum), cpuNum, mht->vAddr, mht->pAddr,
                        (CPUVec.CycleCount(cpuNum) - mht->startTime),
                        (E_L1 | E_I | E_READ), mht->lru);

         ICachePUT(cpuNum, mht, mode);

      } else {
         uint type;

         switch (mht->cmd & (SC_DGET|SC_DGETX|SC_DUGETX|SC_DSCUGETX|SC_DLLGET|SC_DLLGETX)) {
         case SC_DGET:
         case SC_DLLGET:
            type = E_L1 | E_D | E_READ;
            break;
         case SC_DGETX:
         case SC_DLLGETX:
            type = E_L1 | E_D | E_WRITE;
            break;
         case SC_DUGETX:
            type = E_L1 | E_D | E_WRITE | E_UPGRADE;
            break;
         case SC_DSCUGETX:
            type = E_L1 | E_D | E_WRITE | E_SC_UPGRADE;
            break;
         default:
            type = 0;
            ASSERT(0);
         }

         L1_DMISS_EVENT(CPUVec.CycleCount(cpuNum), cpuNum, mht->PC, mht->vAddr,
                        mht->pAddr, CPUVec.CycleCount(cpuNum) - mht->startTime, type);

         DCachePUT(cpuNum, mht, mode);
#ifdef DEBUG_BUFFER
         CPUPrint("%d: Calling DCachePUT from MHTReqDone\n", cpuNum);
#endif
      }
      FreeMHT(cpuNum, mht - CACHE[cacheNum].MHT);
#ifdef MIPSY_MXS
      DoMxsAction (PE[cpuNum].st, (void *) (mht - CACHE[cacheNum].MHT),
                   mht->cpu_action);
#endif
   } else if (status == MEMSYS_STATUS_NAK) {

      /*
       * For NAKs - don't fill the cache line. For
       * most commands we need to retry the operation. The
       * exception is SC upgrades which should not be retried
       * since they will fail anyway.
       */
      if (mht->cmd == SC_DSCUGETX) {

         /* count the stalled time so far */
         SC_NAK_EVENT(CPUVec.CycleCount(cpuNum), cpuNum, mht->PC, mht->vAddr,
                      mht->pAddr, CPUVec.CycleCount(cpuNum) - mht->startTime,
                      E_L1 | E_D | E_WRITE | E_UPGRADE | E_SC_NAK);

         /* Invalidate the line from the first level cache (per the T5) */
         DCacheFlush(cpuNum, 1, 0, mht->pAddr & ~(DCACHE_LINE_SIZE-1),
                     DCACHE_LINE_SIZE);
         FreeMHT(cpuNum, mht - CACHE[cacheNum].MHT);
         CPUVec.Unstall(cpuNum);
#ifdef MIPSY_MXS
         DoMxsAction (PE[cpuNum].st, (void *) (mht - CACHE[cacheNum].MHT),
                      mht->cpu_action);
#endif
         return;
      }

      /* HOHA (Horrible Hack)
       * For transactions which were aborted due to a Cache Error, we
       * don't want to retry.
       * To be fixed.
       */

      /* RPB: disable this when using MXS or write buffers --
         the assumptions made by the HOHA no longer hold.
         Also to be fixed. */

#ifndef MIPSY_MXS
      if (WRITE_BUFFER_SIZE == 0) {
         if (CURRENT_PC(cpuNum) != mht->PC) {
            FreeMHT(cpuNum, mht - CACHE[cacheNum].MHT);
            CPUVec.Unstall(cpuNum);
            return;
         }
      }
#endif

      EventDoCallback(cpuNum,RetryNAKedMHTEntry, (EventCallbackHdr *) mht,
                      (void *)NULL, NAK_RETRY_TIME);
   } else {
      CPUError("Can't handle status in MHTReqDone\n");
   }
   /* --- WARNING: This was put in because SYNCs mean that a request ---
    * --- arrival does not necessarily wake up the processor --- */
   CPUVec.Unstall(cpuNum);
}

/*****************************************************************
 * RetryNAKedMHTEntry
 * Event callback for a NAKed first level miss. Retry the operation
 * in hopes it will succeed this time.
 *****************************************************************/
static void
RetryNAKedMHTEntry(int cpuNum,EventCallbackHdr *event, void *arg)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   MHT *mht = (MHT *) event;
   int mhtind = mht - CACHE[cacheNum].MHT;
   SCResult sret;

   if (mht->cmd == SC_DUGETX) {
         mht->cmd = SC_DGETX;
   }
   switch (mht->cmd) {
   case SC_IGET:
      CACHE[cacheNum].stats.ICache.RetriedIGets++;
      break;
   case SC_DGET:
   case SC_DLLGET:
      CACHE[cacheNum].stats.DCache.RetriedDGets++;
      break;
   case SC_DGETX:
   case SC_DLLGETX:
      CACHE[cacheNum].stats.DCache.RetriedDGetXs++;
      break;
   case SC_DUGETX:
      /* --- I had to add this because it happenned (MAH), not sure ---
       * --- what stat I need to inc here or if I do need to inc one ---
       */
      break;
   default:
      CPUError("RetryNAKedMHTEntry %d: invalid mht->cmd %d\n", cpuNum,
         mht->cmd);
   }
   sret = SCacheFetch(cpuNum, mht->vAddr, mht->pAddr,
                      mht->cmd, mhtind);
   if ((sret == SCSUCCESS) || (sret == SCSTALL)) {
      return;
   }
   ASSERT(sret == SCRETRY);

   /*
    * Try again a little later.
    */
   EventDoCallback(cpuNum,RetryNAKedMHTEntry, (EventCallbackHdr *) mht,
                   (void *)NULL , 1);
}


void
MemRefDebugWriteData(int cpuNum, VA vAddr, PA pAddr, char *data, int size)
{
   if (size == 0)
      return;

#ifndef SOLO
#ifdef SIM_ALPHA
   ASSERT(0);
#else
   if (IS_KSEG1(vAddr)) {
      uint flag;
      void *dat;
#if 0
      if (!RegistryIsInRange(vAddr, &dat, &flag) ||
          !(flag & REG_DATA)) {
         CPUWarning("CPU %d MemRefDebugWrite bad uncached addr at %#x\n",
                    cpuNum, vAddr);
      } else {
         bcopy(data, (char *)dat, size);
      }
#else
      if (!RegistryIsInRange(vAddr, &dat, &flag)) {
         CPUWarning("CPU %d MemRefDebugWrite bad uncached addr at %#x\n",
                    cpuNum, vAddr);
      } else if (!(flag & REG_DATA)) {
	/*XXXblythe should check result*/
      	(*CPUVec.UncachedPIO)(cpuNum, vAddr, 0, size, data);
      } else {
         bcopy(data, (char *)dat, size);
      }
#endif
      return;
   }
   if (data != (char *) PHYS_TO_MEMADDR(M_FROM_CPU(cpuNum),pAddr)) {
      bcopy(data, (char *) PHYS_TO_MEMADDR(M_FROM_CPU(cpuNum), pAddr), size);
   }
#endif /* SIM_ALPHA */
#endif /* SOLO */

#ifdef SOLO
   bcopy(data, (char *) vAddr, size);
#endif
#ifdef DATA_HANDLING
   {
      PA p, pEnd;
      char *cdata;
      int i;
      int way=0;
      int copySize = 0;

      /** Need to check write buffer */
      /* Yeah, need to do that indeed... */
      pEnd = pAddr+size;

      for (p = pAddr; p < pEnd; p += copySize) {
         int offset = ((uint)p & (SCACHE_LINE_SIZE-1));
         int lsize = SCACHE_LINE_SIZE - offset;
         int firstCPU = 1;
         copySize = (lsize > size) ? size : lsize;
         for (i = 0; i < TOTAL_CPUS; i++) {
            if (IsInSCache(i, p, MEMSYS_SHARED, &cdata, &way)) {
               bcopy(data+(p-pAddr), cdata+offset, copySize);
               if (firstCPU) {
                  size -= copySize;
                  firstCPU = 0;
               }
            }
         }
         /* not found in any of the caches... */
      }
   }
#endif
}

void
MemRefDebugReadData(int cpuNum, VA vAddr, PA pAddr, char *data, int size)
{
   uint flag;
   void *dat;

   if (size == 0)
      return;

#ifndef SOLO
   if (IS_KSEG1(vAddr)) {
#if 0
      if (!RegistryIsInRange(vAddr, &dat, &flag) ||
          !(flag & REG_DATA)) {
         CPUWarning("CPU %d MemRefRead bad uncached addr at %#x\n",
                    cpuNum, vAddr);
      } else {
         bcopy((char *)dat, data, size);
      }
#else
      if (!RegistryIsInRange(vAddr, &dat, &flag)) {
         CPUWarning("CPU %d MemRefRead bad uncached addr at %#x\n",
                    cpuNum, vAddr);
      } else if (!(flag & REG_DATA)) {
	/*XXXblythe should check result*/
      	(*CPUVec.UncachedPIO)(cpuNum, vAddr, 1, size, data);
      } else {
         bcopy((char *)dat, data, size);
      }
#endif
      return;
   }
   bcopy((char *) PHYS_TO_MEMADDR(M_FROM_CPU(cpuNum), pAddr), data, size);
#endif
#ifdef SOLO
   bcopy((char *) vAddr, data, size);
#endif
#ifdef DATA_HANDLING
   {
      int i;
      char *cdata;
      int copySize = 0;
      int way = 0;
      PA p, pEnd = pAddr+size;

      for (p = pAddr; p < pEnd; p += copySize) {
         int offset = ((uint)p & (SCACHE_LINE_SIZE-1));
         int lsize  = SCACHE_LINE_SIZE - offset;
         copySize = (lsize > size) ? size : lsize;
	 if (p + copySize > pEnd) {
	    copySize = pEnd - p;
         }
         for (i = 0; i < TOTAL_CPUS; i++) {
            if (IsInSCache(i, p, MEMSYS_SHARED, &cdata, &way)) {
               bcopy(cdata+offset, data+(p-pAddr), copySize);
               size -= copySize;
               break;
            }
         }
         /* not found in any of the caches...
          * CAVEAT: the data might be in transit deep in the guts of Magic.
          * For now, we're going to be very light-headed and just take
          * whatever is in memory, hoping for the best.
          */
      }
   }
#endif
}

/*****************************************************************
 * Uncached operations
 *****************************************************************/
Result
MemRefWriteUncached(int cpuNum, VA vAddr, PA pAddr, void *data,
                    RefSize refSize, bool accelerated)
{
   int size;

   switch(refSize) {
   case BYTE_SZ:
      size = 1;
      break;
   case HALF_SZ:
      size = 2;
      break;
   case WORD_SZ:
      size = 4;
      break;
   case DOUBLE_SZ:
      size = 8;
      break;
    default:
      size=8;
      CPUError("MemRefWriteUncached: invalid refSize %d\n", refSize);
   }

   /* I send the information into the main request path, but I overload */
   /* the use of the wbdata field as the pointer to my data to write */
   if (!accelerated) {
       return memsysVec.MemsysCmd(cpuNum, MEMSYS_UNCWRITE, pAddr, 0,
                               MEMSYS_NOADDR, size, (void *) data);
   } else {
       return memsysVec.MemsysCmd(cpuNum, MEMSYS_UNCWRITE_ACCELERATED, pAddr, 0,
                               MEMSYS_NOADDR, size, (void *) data);
   }
}


Result
MemRefReadUncached(int cpuNum, VA vAddr, PA pAddr, void *data, RefSize refSize)
{
   int size;

   if (CACHE[cpuNum].MHTnumInuse != 0) {
      return STALL;
   }

   switch(refSize) {
   case BYTE_SZ:
      size = 1;
      break;
   case HALF_SZ:
      size = 2;
      break;
   case WORD_SZ:
      size = 4;
      break;
   case DOUBLE_SZ:
      size = 8;
      break;
   }

   /* NOTE (DT)
    * The MHT index passed here is 0. The memsystem better make sure
    * that the MHT is empty when the uncached read is issued. The MHT
    * has to be empty, since all previous transactions MUST have
    * completed for this to issu.
    */
   return memsysVec.MemsysCmd(cpuNum, MEMSYS_UNCREAD, pAddr, 0,
                           MEMSYS_NOADDR, size, (void *) data);
}

Result
MemRefDMAWrite(int cpuNum, PA pAddr, int transId, int length, byte *data)
{
  Result ret;

  ret = memsysVec.MemsysCmd(cpuNum, MEMSYS_GETX|MEMSYS_DMAFLAVOR, pAddr,
                         transId, MEMSYS_NOADDR, length, data);
  return ret;
}

Result
MemRefDMARead(int cpuNum, PA pAddr, int transId, int length, byte *data)
{
  Result ret;

  ret = memsysVec.MemsysCmd(cpuNum, MEMSYS_GET|MEMSYS_DMAFLAVOR, pAddr,
                         transId, MEMSYS_NOADDR, length, data);
  return ret;
}

void
MemRefExit(void)
{
   /*
    * Drain memory system of any dirty data (e.g. writebacks)
    */
   memsysVec.MemsysDrain();
#ifndef SOLO
   memsysVec.MemsysDone();
#endif
}

/*****************************************************************
 * PrimaryFlush - Flush  a range of paddrs from the
 * primary caches. This is used when a line is replaced or downgraded
 * from the secondary scache.
 *****************************************************************/
void
PrimaryFlush(int cpuNum, int writeback, int retain, PA paddr, int size)
{
 if (!retain &&ICACHE_INCLUSION) {
      /* Only invalidates (not writebacks) concern the ICache. */
      ICacheFlush(cpuNum, paddr, size);
   }

   DCacheFlush(cpuNum, writeback, retain, paddr, size);
#ifdef MIPSY_MXS
   DoMxsIntervention(PE[cpuNum].st, paddr, size, retain);
#endif
}


/*****************************************************************
 * AddToWriteBuffer
 * Add a write request to an MHT entyr's write buffer
 *****************************************************************/
static bool
AddToWriteBuffer(int cpuNum, PA pAddr, uint64 data, RefSize size, int mhtind)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);

   if (!CACHE[cacheNum].MHT[mhtind].writeBuffer->active &&
       (CACHE[cacheNum].activeWriteBuffers >= WRITE_BUFFER_SIZE)) {
      return FALSE;
   } else {
      int offset = pAddr & (DCACHE_LINE_SIZE - 1);
      char *wbdata = CACHE[cacheNum].MHT[mhtind].writeBuffer->data + offset;
      char *wbmask = CACHE[cacheNum].MHT[mhtind].writeBuffer->mask + offset;

      switch (size) {
      case WORD_SZ:
         *(int *) wbdata = (int) data;
         *(int *) wbmask = (int) -1;
         if (VERBOSE_DEBUG) {
            CPUPrint("%d: Add to WB  PA: %8.8lx WORD: %8.8lx\n",
                     cpuNum, pAddr, data);
         }
         break;
      case BYTE_SZ:
         *(char *) wbdata = (char) data;
         *(char *) wbmask = (char) -1;
         if (VERBOSE_DEBUG) {
            CPUPrint("%d: Add to WB  PA: %8.8lx BYTE: %2.2lx\n",
                     cpuNum, pAddr, data);
         }
         break;
      case HALF_SZ:
         *(short *) wbdata = (short) data;
         *(short *) wbmask = (short) -1;
         if (VERBOSE_DEBUG) {
            CPUPrint("%d: Add to WB  PA: %8.8lx HALF: %4.4lx\n",
                     cpuNum, pAddr, data);
         }
         break;
      case DOUBLE_SZ:
         *(uint64 *) wbdata = (uint64) data;
         *(uint64 *) wbmask = (uint64) -1LL;
         if (VERBOSE_DEBUG) {
#ifdef __alpha
            CPUPrint("%d: Add to WB  PA: %8.8lx DOUBLE: %16.16lx\n",
                     cpuNum, pAddr, data);
#else
            CPUPrint("%d: Add to WB  PA: %8.8lx DOUBLE: %16.16llx\n",
                     cpuNum, pAddr, data);
#endif
         }
         break;
      default:
         CPUError("Bad size (%d) in AddToWriteBuffer\n", size);
      }

      if (!CACHE[cacheNum].MHT[mhtind].writeBuffer->active) {
         CACHE[cacheNum].MHT[mhtind].writeBuffer->active = TRUE;
         CACHE[cacheNum].activeWriteBuffers++;
      }

      return TRUE;
   }
}

/*****************************************************************
 * AddrIsInWriteBuffer
 *****************************************************************/
static bool
AddrIsInWriteBuffer(int cpuNum, PA pAddr, RefSize size)
{
   int cacheNum = GET_CACHE_NUM(cpuNum);
   int offset = pAddr & (DCACHE_LINE_SIZE - 1);
   bool match;
   PA testPA = pAddr - offset;
   MHT *mht;

   if (CACHE[cacheNum].activeWriteBuffers == 0) return FALSE;

   for (mht = CACHE[cacheNum].MHT;
        mht < (CACHE[cacheNum].MHT + MHT_SIZE);
        mht++) {
      if (mht->inuse && mht->writeBuffer->active &&
          (testPA == (mht->pAddr & ~(DCACHE_LINE_SIZE - 1)))) {
         char *wbmask = mht->writeBuffer->mask + offset;
         switch (size) {
         case WORD_SZ:
            match = ((*(int *) wbmask) != 0);
            if (VERBOSE_DEBUG) {
               CPUPrint("%d: AddrIsInWriteBuffer  PA: %8.8lx size: WORD  %s\n",
                        cpuNum, pAddr, match ? "TRUE" : "FALSE");
            }
            return match;
         case BYTE_SZ:
            match = ((*(char *) wbmask) != 0);
            if (VERBOSE_DEBUG) {
               CPUPrint("%d: AddrIsInWriteBuffer  PA: %8.8lx size: BYTE  %s\n",
                        cpuNum, pAddr, match ? "TRUE" : "FALSE");
            }
            return match;
         case HALF_SZ:
            match = ((*(short *) wbmask) != 0);
            if (VERBOSE_DEBUG) {
               CPUPrint("%d: AddrIsInWriteBuffer  PA: %8.8lx size: HALF  %s\n",
                        cpuNum, pAddr, match ? "TRUE" : "FALSE");
            }
            return match;
         case DOUBLE_SZ:
            match = ((*(uint64 *) wbmask) != 0LL);
            if (VERBOSE_DEBUG) {
               CPUPrint("%d: AddrIsInWriteBuffer  PA: %8.8lx size: DOUBLE  %s\n",
                        cpuNum, pAddr, match ? "TRUE" : "FALSE");
            }
            return match;
         default:
            CPUError("Bad size (%d) in AddrIsInWriteBuffer\n", size);
         }
      }
   }
   if (VERBOSE_DEBUG) {
      CPUPrint("%d: AddrIsInWriteBuffer  PA: %8.8lx  FALSE\n", cpuNum, pAddr);
   }
   return FALSE;

}
/*****************************************************************
 * RetiredWriteBuffer
 *****************************************************************/
static void
RetireWriteBuffer(int cpuNum, MHT *mht, byte *data)
{
   register int i;
   char *wbdata = mht->writeBuffer->data;
   char *wbmask = mht->writeBuffer->mask;

#ifdef DEBUG_DATA_VERBOSE
   char pbuf[384];
   char *pbufptr = pbuf;
   if (VERBOSE_DEBUG) {
      CPUPrint("%d: Retire WB: Addr = %8.8lx, Data =\n", cpuNum,
               mht->pAddr & ~(DCACHE_LINE_SIZE - 1));
   }
#endif

   for (i = 0; i < DCACHE_LINE_SIZE; i++) {
      if (*(wbmask+i) != 0) {
         *(data+i) = *(wbdata+i);
         *(wbmask+i) = 0;
#ifdef DEBUG_DATA_VERBOSE
         sprintf(pbufptr, "%02x", *(data+i));
      } else {
         sprintf(pbufptr, "XX");
      }
      pbufptr += 2;
      if (((i+1) % 32) == 0) sprintf(pbufptr++, "\n");
      else if (((i+1) % 4) == 0) sprintf(pbufptr++, " ");
#else
      }
#endif
   }
#ifdef DEBUG_DATA_VERBOSE
   if (VERBOSE_DEBUG) CPUPrint("%s\n", pbuf);
#endif

   mht->writeBuffer->active = FALSE;
   CACHE[cpuNum].activeWriteBuffers--;
}


#ifdef CC_CHECKER
extern void MustBeInDirectory(int cpu, uint;
int
CacheChecker(PA paddr, int inCpu, int isExclusive)
{
   int cpu;
   int ind;

   if (paddr != (PA) -1) {
      PA tag = isExclusive ? SCACHE_TAG(paddr):SCACHE_TAG_EX(paddr);
      ind = (paddr/SCACHE_LINE_SIZE) % SCACHE_INDEX;
#ifdef DIR_CHECKER
      MustBeInDirectory(inCpu, paddr);
#endif
      for (cpu = 0; cpu < TOTAL_CPUS; cpu++) {
         if (cpu == inCpu)
            continue;
         if (isExclusive) {
            if ((((~EXCLUSIVE_TAG)&CACHE[cacheNum].SCache.tags[ind][0]) == tag) ||
                (((~EXCLUSIVE_TAG)&CACHE[cacheNum].SCache.tags[ind][1]) == tag)) {
               CPUError("Cache check failed at paddr 0x%x\n", paddr);
            }
         } else {
            if ((CACHE[cacheNum].SCache.tags[ind][0] == tag) ||
                (CACHE[cacheNum].SCache.tags[ind][1] == tag)) {
               CPUError("Cache check failed at paddr 0x%x\n", paddr);
            }
         }
      }
   } else {
      for (cpu = 0; cpu < TOTAL_CPUS; cpu++) {
         for (ind = 0; ind < SCACHE_INDEX; ind++) {
            if (CACHE[cacheNum].SCache.tags[ind][0] != INVALID_TAG) {
               CacheChecker(SCACHE_TAG2PADDR(CACHE[cacheNum].SCache.tags[ind][0], ind), cpu,
                            (CACHE[cacheNum].SCache.tags[ind][0]&EXCLUSIVE_TAG)!=0);
            }
            if (CACHE[cacheNum].SCache.tags[ind][1] != INVALID_TAG) {
               CacheChecker(SCACHE_TAG2PADDR(CACHE[cacheNum].SCache.tags[ind][1], ind), cpu,
                            (CACHE[cacheNum].SCache.tags[ind][0]&EXCLUSIVE_TAG)!=0);
            }
         }
      }
   }
   return 0;
}
#endif /* CC_CHECKER */