/*
 * Copyright (C) 1996-1998 by the Board of Trustees
 *    of Leland Stanford Junior University.
 * 
 * This file is part of the SimOS distribution. 
 * See LICENSE file for terms of the license. 
 *
 */

/****************************************************************
 * qc.c
 *
 * This file handles the quickcheck and Physarray structures.
 * pa/phys_mem_ref gets called in case of callout from the 
 * Translation Cache
 * 
 * $Author: blythe $
 * $Date: 2002/05/29 01:09:10 $
 *****************************************************************/
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
/* #include <sys/immu.h> */
#include <fcntl.h>
#include <bstring.h>
#include <string.h>
#include <assert.h>
#include "simmisc.h"
#include "embra.h"
#include "cache.h"
#include "qc.h"
#include "stats.h"
#include "driver.h"
#include "clock.h"
#include "mem_control.h"
#include "main_run.h"
#include "cp0.h"
#include "annotations.h"
#include "simutil.h"
#include "machine_defs.h"
#include "firewall.h"
#include "simmagic.h"
#include "tc.h"
#include "tc_coherence.h"
#include "simtypes.h"

#ifndef EMBRA_USE_QC64

/* There are good reasons not to let backdoor references hit in the */
/* quick check */
/* 1. Some of them need to be relocated MPinUP (like CPUID) */
/* 2. They all need to be relocated when self-hosting */
/* This is not bad because the number of backdoor references should be */
/* small enough to make it non performance critical */
/* XXX -- The above argument no longer holds. */
/* The new interrupt subsystem makes many backdoor references, thus we
   will try to make backdoor references succeed in the quick check
   whenever we can */


/* #define DISABLE_QC */
#undef LOG_QC_UPDATE  /* log every change in qc_p or PhysArray */

#define TLB_ENT2VPN(_ent) (PAGE_NUMBER(TLBHI2ADDR(_ent)))

void qc_CheckForDuplicates(CPUState *P);


/* **********************************************************
 * SetMMUEntry: update the (potentially two different 
 * array when applicable
 * note that user_mode and supervisor_mode have the same
 * privileges.
 * **********************************************************/


static void SetMMUEntry( CPUState *P, VA vAddr, MA value)
{
  
   if (value && 
       !EMBRA_IS_MEMADDR(M_FROM_CPU(P->myNum),MMU_PROT_READ(value))) {
      CPUError("SetMMUEntry ASSERT: vAddr=%llx value=%llx not a MA\n",
               (uint64)vAddr,(uint64)value);
   }
   P->kernelMMU[PAGE_NUMBER(vAddr)] = value;
   if (embra.separateMMUs && 
       (IS_KUSEG(vAddr) || IS_SUPERV_SEG(vAddr)) ) {
      P->userMMU[PAGE_NUMBER(vAddr)] = value;
   }
}


void qc_cache_init(int cpuNum)
{
    if( embra.sequential) {
        int cpu;
        for( cpu = 0; cpu < TOTAL_CPUS; cpu++ ) {
            /* Record MMU reloc base */
            if (EMP[cpu].mmu) continue;
            EMP[cpu].kernelMMU = (MA*)ZALLOC_PERM(MMU_RELOC_SIZE,"EmbraMMU");
            EMP[cpu].mmu =  EMP[cpu].kernelMMU;
            CPUPrint("0x%x D MMU_RELOC_BASE_%d 0x%x\n", 
                     EMP[cpu].mmu, cpu,
                     (uint)EMP[cpu].mmu + MMU_RELOC_SIZE );
            if (embra.separateMMUs) { 
                EMP[cpu].userMMU = 
                    (MA*)ZALLOC_PERM(MMU_RELOC_SIZE,"EmbraMMU");
                CPUPrint("0x%x D MMU_RELOC_BASE_%d 0x%x (kernel) \n", 
                         EMP[cpu].userMMU, cpu, 
                         (uint)EMP[cpu].userMMU + MMU_RELOC_SIZE ); 
            } else { 
                EMP[cpu].userMMU = EMP[cpu].kernelMMU;
            }
        }
    } else {
        ASSERT (0);
    }
    
    /* Page mode doesn't use quick checks */
    if( embra.emode == EMBRA_PAGE )
        return;
    if (embra.useVQC){
        if( embra.sequential) {
            int cpu;
            for( cpu = 0; cpu < TOTAL_CPUS; cpu++ ) {
                /* Record QC_V reloc base */
                if (EMP[cpu].qc_v) continue;
                EMP[cpu].qc_v = (char*) ZALLOC_PERM(QC_VIRT_SIZE,"EmbraVQC");
                /* memory access entry holds value for QC_REG */
                EMP[cpu].cache_ax = EMP[cpu].qc_v;
                /* Nm output */
                CPUPrint("0x%x D QC_VBASE_%d 0x%x\n",
                         EMP[cpu].qc_v, cpu,
                         (uint)EMP[cpu].qc_v + QC_VIRT_SIZE );
                /* Record QC_P reloc base */
                EMP[cpu].qc_p =
                    (phys_info_t*)ZALLOC_PERM(QC_PHYS_SIZE(M_FROM_CPU(cpu)),
                                              "EmbraPQC");
                CPUPrint("0x%x D QC_PBASE_%d 0x%x\n",
                         EMP[cpu].qc_p, cpu,
                         (uint)EMP[cpu].qc_p + QC_PHYS_SIZE(M_FROM_CPU(cpu)));
            }
        } else {
            ASSERT (0);
        }
    } else { /* !embra.useVQC */
        if( embra.sequential) {
            int cpu;
            for( cpu = 0; cpu < TOTAL_CPUS; cpu++ ) {
                /* Record QC_V reloc base */
                if (EMP[cpu].pa_p) continue;
                EMP[cpu].pa_p = (pa_info_t*)
                    ZALLOC_PERM(PA_SIZE(M_FROM_CPU(cpu)),"EmbraPA");
                /* Nm output */
                CPUPrint("0x%x D PA_BASE_%d 0x%x\n",
                         EMP[cpu].pa_p, cpu,
                         (uint)EMP[cpu].pa_p + PA_SIZE(M_FROM_CPU(cpu)));
                /* memory access entry holds value for PA_REG */
                EMP[cpu].cache_ax=(pa_info_t *)(EMP[cpu].pa_p-
                           (MA_TO_UINT(SIM_MEM_ADDR(M_FROM_CPU(cpu))) >>
                            log2SCACHE_LINE_SIZE));
            }
        } else {
            ASSERT (0);
        }
    }
}

/* Invalidate the quick check entries specified in cpu_bits */
/* Invalidate physical first to avoid race condition */
void 
qc_clobber( PA pAddr, int cpuNum, EmVQCMemState state )
{
  phys_info_t* phys;
  pa_info_t* pa;
  char* virt = 0;
#ifdef DISABLE_QC
  return; /* EXP */
#endif

  if (embra.useVQC){
    /* Does this work?  We read vline, it changes, we invalidate a vline
       which was invalid and the pqc, but process uses vqc.  Could ll/sc
       the write of phys */
    phys = &EMP[cpuNum].qc_p[ADDR2SLINE(pAddr)];
    if( PQC_VLINE(*phys) ) {
      virt = &EMP[cpuNum].qc_v[PQC_VLINE(*phys)];
    }
    /*
     * remove the qc entry even if we only have a downgrade
     */
    
    *phys = PQC_SET_INV;
    if( virt )
       *virt =  MEM_INVALID;
  } else { /* !embra.useVQC */
     EMP[cpuNum].pa_p[ADDR2SLINE(pAddr)]=PA_SET_INV; 
  }
}


/* Transition from exclusive to read shared -- used when newly written */
/* code is executed.  This allows us to detect further writes */
void qc_downgrade(int cpuNum, VA vAddr, int new_state)
{

  ASSERT( VQC_SHARED(new_state) );
   switch(embra.emode) {
   case EMBRA_CACHE:
      if (embra.useVQC){
         ASSERT (EMP[cpuNum].mmu[PAGE_NUMBER(vAddr)]);
         EMP[cpuNum].qc_v[ADDR2SLINE(vAddr)] = new_state;     
      } else {
         EMP[cpuNum].mmu[PAGE_NUMBER(vAddr)]  = 
            MMU_PROT_READ(EMP[cpuNum].mmu[PAGE_NUMBER(vAddr)] );
      }  
      break;
   case EMBRA_PAGE:
      SetMMUEntry(&EMP[cpuNum],vAddr,MMU_PROT_READ(EMP[cpuNum].mmu[PAGE_NUMBER(vAddr)]));
      break;
  }
}

/* *************************************************************
 * for all cpus, downgrade the protection. called by
 * for TC coherence
 * *************************************************************/

void qc_downgrade_ifpresent(VA vAddr)
{
   int cpuNum;
   if (embra.emode == EMBRA_CACHE && embra.useVQC) {
      for (cpuNum=0;cpuNum<TOTAL_CPUS;cpuNum++) {
         if (VQC_EXCL(EMP[cpuNum].qc_v[ADDR2SLINE(vAddr)])) {
            EMP[cpuNum].qc_v[ADDR2SLINE(vAddr)] &=~MEM_D_EXCLUSIVE;
         }
      }
   } else {
      for (cpuNum=0;cpuNum<TOTAL_CPUS;cpuNum++) {
         SetMMUEntry(&EMP[cpuNum],vAddr, MMU_PROT_READ(EMP[cpuNum].mmu[PAGE_NUMBER(vAddr)]));
      }
   }
}

static long clearVec=0;  /* bitvector, each CPU calls qc_clear only ones */
/* Clear mmu relocation info (for the mapped areas) */
/* and clear quick cache state info (for all cached areas)*/
static void qc_clear(int cpuNum)
{

ASSERT(!(clearVec&(1<<cpuNum)));

clearVec=clearVec|(1<<cpuNum);

}

/* Page mode: note that driver.c:EmbraInstallMemAnnotation will be
 * called after qc_renew runs, so we are free to put in translations
 * here for all pages, even those with load/store annotations on them.  
 */
void 
qc_renew( int cpuNum )

{
   K0A i;
   MA mAddr;
   int machNo = M_FROM_CPU(cpuNum);

   if( !embra.MPinUP || cpuNum == 0 ) qc_cache_init(cpuNum);
   /* qc_clear(cpuNum); */

   /* If mmu has not been filled-in for the valid KSEG0 range,
    * do it now.
    */
   if( !EMP[cpuNum].kernelMMU[PAGE_NUMBER(__MAGIC_OSPC_END)] ) {
      /* Store the proper page translations for all other K0 addresses */
      if (!CPUVec.CheckFirewall) { 
         for(i = (Reg32)K0BASE ; i < (Reg32)(K0BASE + MEM_SIZE(machNo)) ; i += DEFAULT_PAGESZ ) {
            /* Own text segment exclusive */
            MA mAddr = PHYS_TO_MEMADDR(machNo, K0_TO_PHYS_REMAP(i, cpuNum));
            if(embra.emode == EMBRA_PAGE || !embra.useVQC)
               EMP[cpuNum].kernelMMU[PAGE_NUMBER(i)] = MMU_PROT_WRITE(mAddr);
            else /* Cache Mode with QC */
               EMP[cpuNum].kernelMMU[PAGE_NUMBER(i)] = mAddr;
         }
      } else {
         /* k0 pages without firewall permission should not be in
          * the mmu
          */
         for(i = (Reg32)K0BASE ; i < (Reg32)(K0BASE + MEM_SIZE(machNo)); i += DEFAULT_PAGESZ ) {
            /* Own text segment exclusive */
            PA pAddr = K0_TO_PHYS_REMAP(i, cpuNum);
            MA mAddr = PHYS_TO_MEMADDR(machNo, pAddr);
            if (SimMagic_IsIncoherent(pAddr)) {
               EMP[cpuNum].mmu[PAGE_NUMBER(i)] = 0;
            } else if (embra.emode == EMBRA_CACHE && embra.useVQC) {
               EMP[cpuNum].mmu[PAGE_NUMBER(i)] = mAddr;
            } else if (!CPUVec.CheckFirewall(cpuNum, pAddr)) {
               EMP[cpuNum].kernelMMU[PAGE_NUMBER(i)] = MMU_PROT_READ(mAddr);
            } else {
               EMP[cpuNum].kernelMMU[PAGE_NUMBER(i)] = MMU_PROT_WRITE(mAddr);
            }
         }
      }

      if (annWatchpoints) {
         uint64 addr;
         AnnPtr rec;
         for (addr = AnnFirst("load", &rec); rec; addr = AnnNext(&rec)) {
            VA vAddr = (VA) addr;
            if (IS_KSEG0(vAddr)) {
               PA pAddr = K0_TO_PHYS(vAddr);
               EMP[cpuNum].kernelMMU[PAGE_NUMBER(pAddr)] = 0;
            }
         }
         for (addr = AnnFirst("store", &rec); rec; addr = AnnNext(&rec)) {
            VA vAddr = (VA) addr;
            if (IS_KSEG0(vAddr)) {
               PA pAddr = K0_TO_PHYS(vAddr);
               EMP[cpuNum].kernelMMU[PAGE_NUMBER(pAddr)] = 0;
            }
         }
      }
   }

   /* remap region requires special handling. Currently, we can only
    * remap one page.
    */
   mAddr = PHYS_TO_MEMADDR(machNo, K0_TO_PHYS_REMAP(K0BASE, cpuNum));
   if(embra.emode == EMBRA_PAGE || !embra.useVQC)
     EMP[cpuNum].kernelMMU[PAGE_NUMBER(K0BASE)] = MMU_PROT_WRITE(mAddr);
   else
     EMP[cpuNum].kernelMMU[PAGE_NUMBER(K0BASE)] = mAddr;

   /* the OSPC range is mapped to the second page of KSEG0 in the
    * 32 bit address space version, since we need this to be in cached
    * address space (for Flite). We cannot place a translation for it in
    * the mmu, since all the references have to be fielded by
    * simmagic.c.
    *
    * Note that the OSPC range is only enabled if the remap range is.
    */
   ASSERT( __MAGIC_OSPC_BASE + DEFAULT_PAGESZ == __MAGIC_OSPC_END );
   if ( remapVec->RemapEnable[cpuNum] )
     EMP[cpuNum].kernelMMU[PAGE_NUMBER(__MAGIC_OSPC_BASE)] = 0;


  /* Ensure no code is hanging around in TC that depended on previous
   * state of these pages
   */
  Clear_Translation_State( TCFLUSH_ALL);

} 



void qc_tlb_inval_page( int cpuNum, int idx)
{
  EntryHi hi = EMP[cpuNum].tlbEntry[idx].Hi & (~TLBHI_G);
  EntryLo lo0 = EMP[cpuNum].tlbEntry[idx].Lo0;
  EntryLo lo1 = EMP[cpuNum].tlbEntry[idx].Lo1;

  
  /*
   * check that there is no mapping from qc_v 
   */

   if (embra.emode ==EMBRA_CACHE && embra.useVQC && !IS_KSEG0(hi)) { 
      int i;
      int base = ADDR2SLINE(TLBHI2ADDR(hi));
      int max = 2 * LINES_PER_PAGE;
      ASSERT( !(TLB_ENT2VPN(hi)&1));
      for (i=0;i<max;i++) {
         if (EMP[cpuNum].qc_v[base+i]) {
            CPUWarning("qc_tlb_inval_page hi=%#x lo=%#x %#x i=%d mapped \n",
                       hi,lo0,lo1,i);
         }
      }
   }
   if( !IS_UNMAPPED_TLBHI( hi ) ) {
      SetMMUEntry(&EMP[cpuNum],TLB_ENT2VPN(hi) * DEFAULT_PAGESZ,0);
      SetMMUEntry(&EMP[cpuNum],(TLB_ENT2VPN(hi)|1)*DEFAULT_PAGESZ,0);
   }
  

}

void qc_tlb_replace_page( int cpuNum, int idx)
{
   qc_tlb_inval_page( cpuNum, idx);

}


void qc_cache_inval_page( int cpuNum, int idx)
/* invalidates vQC entry and destroys backmap from pQC */
/* it does not change cache information in pQC */

{
  PLN pline;
  EntryHi hi = EMP[cpuNum].tlbEntry[idx].Hi & (~TLBHI_G);
  EntryLo lo0 = EMP[cpuNum].tlbEntry[idx].Lo0;
  EntryLo lo1 = EMP[cpuNum].tlbEntry[idx].Lo1;


  if (!embra.useVQC) return;   /* physarray does not contain a backmap */
#ifdef DISABLE_QC
  return; /* EXP */
#endif
  ASSERT( embra.emode == EMBRA_CACHE );

  if( ! IS_KSEG0( hi ) ) {
    bzero( &EMP[cpuNum].qc_v[TLBENT2SLINE(hi)], LINES_PER_PAGE );
    /* If we want, we can destroy the backmap from physical to */
    /* virtual address in phys_info.  This should aid performance, */
    /* because now some needless virtual QC clobberings won't */
    /* happen, but it should not impact correctness */ 
    if( IS_VALID(lo0) ) {
      for( pline = TLBENT2SLINE(lo0);
           pline < (TLBENT2SLINE(lo0) + LINES_PER_PAGE);
           pline++) {
        /* Note that some of the backpoints from this physical page */
        /* may still be valid because they may point to K0 addresses */
        /* There is no need to zero these*/
        /* They can also point to other mapped pages (this kind of */
        /* aliasing is especially common with K2) */
        /* Therefore only destroy the backpointers to the virtual */
        /* page which was just unmapped.  Leave the cache info */
        if( TLBHI2ADDR(PQC_VLINEADDR(EMP[cpuNum].qc_p[pline]))  == 
            TLBHI2ADDR(hi) ) {
          ASSERT(EMP[cpuNum].qc_v[PQC_VLINE(EMP[cpuNum].qc_p[pline])] ==
                 MEM_INVALID);
          /* This zeroes the vline, but leaves the cache information  */
          EMP[cpuNum].qc_p[pline] = PQC_STATUS(EMP[cpuNum].qc_p[pline]);
        }
      }
    }
    bzero( &EMP[cpuNum].qc_v[TLBENT2SLINE(hi)+LINES_PER_PAGE], LINES_PER_PAGE );
    /* If we want, we can destroy the backmap from physical to */
    /* virtual address in phys_info.  This should aid performance, */
    /* because now some needless virtual QC clobberings won't */
    /* happen, but it should not impact correctness */ 
    if( IS_VALID(lo1) ) {
      for( pline = TLBENT2SLINE(lo1);
           pline < (TLBENT2SLINE(lo1) + LINES_PER_PAGE);
           pline++) {
        /* Note that some of the backpoints from this physical page */
        /* may still be valid because they may point to K0 addresses */
        /* There is no need to zero these*/
        /* They can also point to other mapped pages (this kind of */
        /* aliasing is especially common with K2) */
        /* Therefore only destroy the backpointers to the virtual */
        /* page which was just unmapped.  Leave the cache info */
        if( TLBHI2ADDR(PQC_VLINEADDR(EMP[cpuNum].qc_p[pline]))  == 
            TLBHI2ADDR(hi) ) {
          ASSERT(EMP[cpuNum].qc_v[PQC_VLINE(EMP[cpuNum].qc_p[pline])] ==
                 MEM_INVALID);
          /* This zeroes the vline, but leaves the cache information  */
          EMP[cpuNum].qc_p[pline] = PQC_STATUS(EMP[cpuNum].qc_p[pline]);
        }
      }
    }
  }
}


/* Always invalidate the previous virtual line.  That way, physical */
/* entries are aliased only once */
void set_qc_state( int cpuNum, VLN vline, PLN pline, int new_state )
{
   PA pAddr =  SLINE2ADDR(pline);
#ifdef LOG_QC_UPDATE
  CPUPrint("LOG: %lld qc_p/pa update pAddr = %x newstate %x\n",EMP[cpuNum].cycleCount,pline,new_state);

#endif

  ASSERT(!CPUVec.CheckFirewall);  /* this function doesn't support the firewall yet*/

  /* WATCHPOINTS */
  if (annWatchpoints == TRUE) {
     if (AnnFMRangeCheck(vline << log2SCACHE_LINE_SIZE,
                         ANNFM_LD_TYPE | ANNFM_ST_TYPE)) {
        new_state = MEM_INVALID;
#if 0 
        LogEntry("DEBUG", cpuNum, "Leaving cache line 0x%x empty\n",vline);
#endif
     }
  }
  if (new_state==MEM_D_EXCLUSIVE &&
      TCcoherence_is_code(PHYS_TO_MEMADDR(M_FROM_CPU(cpuNum), pAddr))) {
     new_state = MEM_D_SHARED;
  }
  
  if (embra.useVQC) {
     
     VLN old_vline = PQC_VLINE(EMP[cpuNum].qc_p[pline]);
     if( old_vline ) {
        EMP[cpuNum].qc_v[old_vline] = MEM_INVALID;
     }
     
     EMP[cpuNum].qc_v[vline] = new_state;   
     switch( new_state ) {
     case MEM_INVALID: 
        EMP[cpuNum].qc_p[pline] = 0;
        break;
     case MEM_D_EXCLUSIVE:
        EMP[cpuNum].qc_p[pline] = PQC_SET_DIRTY( vline );
        break;
     case MEM_I_EXCLUSIVE:
        ASSERT (0);
        break;
     case MEM_I_SHARED:
     case MEM_D_SHARED:
        EMP[cpuNum].qc_p[pline] = PQC_SET_SHARED( vline );
        break;
     }
     if (new_state != MEM_INVALID) { 
        MA mmuEntry = EMP[cpuNum].mmu[PAGE_NUMBER(SLINE2ADDR(vline))];
        ASSERT (mmuEntry);
        if (ADDR2SLINE(MEMADDR_TO_PHYS(M_FROM_CPU(cpuNum),
                                       MMU_PROT_READ(mmuEntry)))!=
            (pline&~(LINES_PER_PAGE-1))) {
           CPUError("EMBRA: set_qc_state error vLine=%x pLine=%x mmuEntry=%x \n",
                    vline,pline,mmuEntry);
        }
     }
  } else { /*!embra.useVQC */
   switch( new_state ) {
    case MEM_INVALID: 
      EMP[cpuNum].pa_p[pline] = PA_SET_INV;
      break;
    case MEM_D_EXCLUSIVE:
    case MEM_I_EXCLUSIVE:
      EMP[cpuNum].pa_p[pline] = PA_SET_DIRTY;
      break;
    case MEM_I_SHARED:
    case MEM_D_SHARED:
      EMP[cpuNum].pa_p[pline] = PA_SET_SHARED;
      break;
    }

  }

}


/* XXX - Caller check ASID and/or global bit */
 void qc_cache_reload_page( int cpuNum, VA vLine, EntryLo lo0, EntryLo lo1)
{
   PA pLine;

   if( IS_VALID(lo0) ) {
     for( pLine = ADDR2SLINE(TLBLO2ADDR(lo0)); 
          pLine < ADDR2SLINE(TLBLO2ADDR(lo0)) + LINES_PER_PAGE; 
          pLine++, vLine++ ) {
       uint tag = EMP[cpuNum].cache_tag[SCACHE_INDEXOF(SLINE2ADDR(pLine))];
       if( CACHE_VALID( tag ) && CACHE_PLINE( tag ) == pLine ) {
         if( CACHE_EXCL( tag ) && IS_DIRTY(lo0) ) {
           set_qc_state(cpuNum, vLine, pLine, MEM_D_EXCLUSIVE);
         } else {
           /* Don't know I/D */
           set_qc_state(cpuNum, vLine, pLine, MEM_D_SHARED);
         }

       }
     }
   } else { 
      vLine += LINES_PER_PAGE;
   }
   if( IS_VALID(lo1) ) {
     for( pLine = ADDR2SLINE(TLBLO2ADDR(lo1)); 
          pLine < ADDR2SLINE(TLBLO2ADDR(lo1)) + LINES_PER_PAGE; 
          pLine++, vLine++ ) {
       uint tag = EMP[cpuNum].cache_tag[SCACHE_INDEXOF(SLINE2ADDR(pLine))];
       if( CACHE_VALID( tag ) && CACHE_PLINE( tag ) == pLine ) {
         if( CACHE_EXCL( tag ) && IS_DIRTY(lo1) ) {
           set_qc_state(cpuNum, vLine, pLine, MEM_D_EXCLUSIVE);
         } else {
           /* Don't know I/D */
           set_qc_state(cpuNum, vLine, pLine, MEM_D_SHARED);
         }

       }
     }
   }
}


static void qcMapOnePage(int cpuNum,VA vAddr, EntryLo lo)
{

   if (!EMBRA_IS_PADDR(M_FROM_CPU(cpuNum), TLBLO2ADDR(lo))) {
      /* BAD mapping -- setting mmu entry to 0 so mem_ref will be called and it will bus error */
      SetMMUEntry(&EMP[cpuNum],vAddr,0);
      CPUWarning("bad physical address (0x%x) being mapped into tlb\n", TLBLO2ADDR(lo));
   } else if (IS_VALID(lo)) {
      PA pAddr = REMAP_PHYS(TLBLO2ADDR(lo),cpuNum);
      MA mAddr = PHYS_TO_MEMADDR(M_FROM_CPU(cpuNum), pAddr);
      ASSERT ( EMBRA_IS_MEMADDR(M_FROM_CPU(cpuNum), mAddr));
      if ( embra.emode == EMBRA_PAGE
           && CPUVec.CheckFirewall && SimMagic_IsIncoherent(pAddr)) {
         SetMMUEntry(&EMP[cpuNum],vAddr,0);
      } else if (embra.emode==EMBRA_PAGE && annWatchpoints && 
                 AnnFMRangeCheck(vAddr,ANNFM_LD_TYPE|ANNFM_ST_TYPE)) {
         SetMMUEntry(&EMP[cpuNum],vAddr,0);
      } else if ( embra.emode == EMBRA_PAGE 
          && IS_DIRTY(lo) 
          && !TCcoherence_is_code(PHYS_TO_MEMADDR(M_FROM_CPU(cpuNum), pAddr))
          && !(CPUVec.CheckFirewall && !CPUVec.CheckFirewall(cpuNum, pAddr)) ) {
         /* OK to write this page directly in TC without calling out
          * to mem_ref
          */
         SetMMUEntry(&EMP[cpuNum],vAddr, MMU_PROT_WRITE(mAddr));
      } else {
         SetMMUEntry(&EMP[cpuNum],vAddr, MMU_PROT_READ(mAddr));
      }
   }
}
   
void qc_map_page( int cpuNum, int idx)
{
   EntryHi hi = EMP[cpuNum].tlbEntry[idx].Hi;
   EntryLo lo0 = EMP[cpuNum].tlbEntry[idx].Lo0;
   EntryLo lo1 = EMP[cpuNum].tlbEntry[idx].Lo1;
   VA vAddr0 = TLB_ENT2VPN(hi)*DEFAULT_PAGESZ;
   VA vAddr1 = (TLB_ENT2VPN(hi)|1)*DEFAULT_PAGESZ;


   ASSERT( vAddr0 != vAddr1);
   if( !IS_KSEG0( hi ) && (IS_VALID(lo0) || IS_VALID(lo1))) {
      if( EMP[cpuNum].kernelMMU[TLB_ENT2VPN(hi)] != 0 ) {
         qc_erase_etlb( cpuNum, hi );
      }
   }
   qcMapOnePage(cpuNum,vAddr0,lo0);
   qcMapOnePage(cpuNum,vAddr1,lo1);

   if (embra.emode == EMBRA_CACHE) { 
      qc_cache_reload_page( cpuNum, ADDR2SLINE(TLBHI2ADDR(hi)), lo0 ,lo1);
   }
}

void qc_flush_etlb(int cpuNum)
{
}

void qc_erase_etlb(int cpuNum, EntryHi hi)
{
}

K0A non_excepting_tv( int cpuNum, VA vAddr)
{
   MA mAddr;
   if (IS_BACKDOOR(vAddr)) {
     return vAddr;
   }
   ASSERT(cpuNum >= 0 && cpuNum < TOTAL_CPUS);
   if (!EMP[cpuNum].mmu) {
      CPUWarning("\n\nEMBRA: non_excepting_tc called before init\n\n");
      return 0;
   }

   mAddr = EMP[cpuNum].kernelMMU[PAGE_NUMBER(vAddr)];

   if( mAddr ) {
     mAddr += PAGE_OFFSET( vAddr );
     /* Only needed for page mode, but instead of checking its */
     /* cheaper to just do it */ 
     return MEMADDR_TO_K0(M_FROM_CPU(cpuNum), MMU2ADDR(mAddr));
   }
   return 0;
}


/* This function acesses true TLB state */
void qc_mmu_switch( int cpuNum, unsigned old_asid, unsigned new_asid,
                    uint forceSelfSwitch)
{
  static int saved_asid[SIM_MAXCPUS];
  int numTlbEntries;
  int i;
  
  /* Don't cxt switch to 0.  We leave the old asid in the */
  /* quick_ASID variable, fooling the rest of the code into */
  /* thinking that the previous ASID is running.  This is not */
  /* hardware consistent, but it should be sematically correct */

  if (forceSelfSwitch) {
     /* normally switching to self is a nop.  However we may need
      * to force it to bring the qc into consistency with a state
      * change in the outside world (firewall, incoherent line,
      * change in the amount of physical memory, etc).
      */
     ASSERT(old_asid == new_asid);
     ASSERT(old_asid == CURRENT_ASID(cpuNum));
     goto force_self;
  }

  /* Optimizations - Dont cxt switch to self (that is solid) */
  if( old_asid == new_asid )
	return;

#ifndef TORNADO
  /*
   * Unlike IRIX, Tornado uses ASID 0 for non-global entries
   */


  /* Switches to 0 are nops.  We leave the old asid in the quick_ASID */
  /* variable, fooling the rest of the code into thinking that the */
  /* previous ASID is running.  This is not hardware consistent, but */
  /* it should be semantically correct */
  /* Ex. 6,0,6 stays at 6 */
  /* Ex. 6,0,6,0,5 looks like 6,5 */
  if( new_asid == 0 )
	{
	  /* Switch to 0 is nop */
      ASSERT( CURRENT_ASID(cpuNum) != 0 );
      saved_asid[cpuNum] = CURRENT_ASID(cpuNum);/*old_asid;*/
	  return;
	}
  else
	{
	  /* Switch out of 0 */
	  if( old_asid == 0 )
		/* Switched back to ourselves, do nothing */
		if( new_asid == saved_asid[cpuNum] )
		  return;
		else
		  /* We've gone (for example) 5,0,8 */
		  old_asid = saved_asid[cpuNum];
	      /* and fallthrough */
	}


  ASSERT( new_asid != old_asid );
  ASSERT( new_asid != 0 );
#endif  /* TORNADO */

  quick_ASID[cpuNum] = new_asid;

force_self:
  qc_flush_etlb(cpuNum);
  numTlbEntries = EMP[cpuNum].numTlbEntries;
  for(i = 0; i < numTlbEntries; i++ ) {
     /* If the entry is (1)valid, (2)has the previous asid, and (3)does */
     /* not have the global bit set, then invalidate that page in the */
     /* quick check array */
     int global = IS_GLOBAL_HI( EMP[cpuNum].tlbEntry[i].Hi);
     if( GET_ASID( EMP[cpuNum].tlbEntry[i].Hi ) == old_asid && !global) {
        int hi =  EMP[cpuNum].tlbEntry[i].Hi;
        ASSERT (TLBENT2SLINE(hi) == TLBENT2SLINE(hi&~TLBHI_G));
        if( embra.emode == EMBRA_CACHE ) {
           qc_cache_inval_page( cpuNum,i);
        }
        qc_tlb_inval_page( cpuNum, i);
     }
  }
  for(i = 0; i < numTlbEntries; i++ ) {
     /* Now restore old mappings.  Global mappings are never erased */
     if( GET_ASID( EMP[cpuNum].tlbEntry[i].Hi ) == CURRENT_ASID(cpuNum) ) {
        qc_map_page( cpuNum,i);
     }
  }
}


void qc_CheckForDuplicate(CPUState *P,  int index)
{
   int j;
   Reg hi = P->tlbEntry[index].Hi & ~TLBHI_G;
   if (IS_UNMAPPED_TLBHI(hi)) return;
   for(j=0;j<P->numTlbEntries;j++) {
      Reg other = P->tlbEntry[j].Hi & ~TLBHI_G;
      if (j==index) continue;
      if (IS_UNMAPPED_TLBHI(other) || (other == 0)) continue;
      if (other==(hi&~TLBHI_G)) {
         CPUWarning("Duplicate tlb entry: new=0x%llx old=0x%llx indx=%d\n", 
                    (Reg64)hi, (Reg64)other, j);
      }
   }
}

void qc_CheckForDuplicates(CPUState *P)
{
   Reg hiEntries[MAX_NTLBENTRIES];
   int i,j;
   for(i = 0; i < P->numTlbEntries; i++ ) {
      Reg hi = P->tlbEntry[i].Hi;
      hiEntries[i] = hi & ~TLBHI_G;
      if (IS_UNMAPPED_TLBHI(hi)) continue;
      for(j=0;j<i;j++) {
         ASSERT( hiEntries[j] != hi);
      }
   }
}


void EmFirewallChange(PA pAddr, uint grant, uint64 cpumask)
{
   MA mAddr;
   VA vAddr = PHYS_TO_K0(pAddr);
   uint cpu;

   ASSERT(embra.emode == EMBRA_PAGE);   /* this function doesn't support
                                         * cache mode yet 
                                         */
   if (grant) {
      /* easy case: only need to update kseg0 that might be denied.
       * kseg2 and kuseg will get added on next qc miss 
       */
      for (cpu = 0; cpu < TOTAL_CPUS; cpu++) {
         if ( ! (cpumask & (1 << cpu)))
            continue;
         mAddr = PHYS_TO_MEMADDR(M_FROM_CPU(cpu), pAddr);
         if (!TCcoherence_is_code(PHYS_TO_MEMADDR(M_FROM_CPU(cpu), pAddr))) {
            SetMMUEntry(&EMP[cpu],vAddr,MMU_PROT_WRITE(mAddr));
         }
      }

   } else {
      for (cpu = 0; cpu < TOTAL_CPUS; cpu++) {
         if ( ! (cpumask & (1 << cpu)))
            continue;

         mAddr = PHYS_TO_MEMADDR(M_FROM_CPU(cpu), pAddr);
         /* Denying access that used to be granted.  Take it out of kseg0...
          */
         SetMMUEntry(&EMP[cpu],vAddr,MMU_PROT_READ(mAddr));

         /* ... and out of kuseg or kseg2.  for now do this inefficiently */
         qc_mmu_switch(cpu, CURRENT_ASID(cpu), CURRENT_ASID(cpu), 1);
      }
   }
}

/****************************************************************************/
/* phys_mem_ref wrapper */
/* This implements the "fast reload" from the physically indexed quick */
/* check array pQC */ 
/***********************************************************************/
/* Returning 0 means rewind QC, returning a value means use that value */

MA phys_mem_ref(VA vAddr, EmVQCMemState new_state, MA mAddr, int cpuNum)

{
   PLN pline;
   phys_info_t pLineInfo;
   PA pAddr;
   EmbraState* P = &EMP[cpuNum];
   MA retval;
   /* ASSERT( cpuNum < TOTAL_CPUS );*/

   if( VQC_INST( new_state ) ) {
      K0A k0Addr;
      vAddr = IN_BD( vAddr )?CLEAR_BD( vAddr ) + INST_SIZE: vAddr;
      k0Addr = non_excepting_tv(cpuNum, vAddr);    /* MMU lookup */
      if( k0Addr ) {
         pAddr = K0_TO_PHYS_REMAP(k0Addr, cpuNum);
      } else {
         /* TLB fault */
         retval =  mem_ref( vAddr, new_state, cpuNum );
         ASSERT( !retval || (uint)retval >0x1000);
         return retval;
      }
   } else {
      if( (uint)mAddr < 0x1000 ) {
         /* TLB fault */
         retval= mem_ref( vAddr, new_state, cpuNum );
         ASSERT( !retval || (uint)retval >0x1000);
         return retval;
      }
      pAddr = MEMADDR_TO_PHYS( M_FROM_CPU(cpuNum), mAddr );
   }

   /* VQC missed */
   /* STAT_VQC(new_state);*/


   pline = ADDR2SLINE(pAddr);

   pLineInfo = P->qc_p[pline];


   switch( new_state ) {
   case MEM_INVALID:
      ASSERT(0);
      break;
   case MEM_I_EXCLUSIVE:
      ASSERT(0);
      break;
   case MEM_D_EXCLUSIVE:
      /* Expensive Assert */
      /* ASSERT( P->qc_v[ADDR2SLINE(vAddr)] == MEM_INVALID || 
         VQC_SHARED( P->qc_v[ADDR2SLINE(vAddr)] ) );     */

      if( PQC_DIRTY( pLineInfo ) &&
          ( PQC_VLINE( pLineInfo ) == ADDR2SLINE( vAddr ) ||
            !PQC_VLINE( pLineInfo ) )
          /* The problem is that the kernel can write a location, then */
          /* NOT set the dirty bit for the TLB entry which allows user */
          /* to write to this location */
          /* Returns 1 for K0 addresses */
          && Em_Is_Tlb_Writable(cpuNum, vAddr, CURRENT_ASID(cpuNum) ) ) {
         /* VASSERT( cache_verify_excl(cpuNum, pline), 
            ("%d vAddr 0x%x pAddr 0x%x state %d", 
            cpuNum, vAddr, pAddr, new_state) );  EXP
            */
         set_qc_state(cpuNum, ADDR2SLINE(vAddr), pline, new_state );
         /* Use below line to support parallel cache mode */
         /* return (MPinUP || TOTAL_CPUS == 1) ?
                            PHYS_TO_MEMADDR( M_FROM_CPU(cpuNum),  pAddr ): 0;*/
         retval = PHYS_TO_MEMADDR( M_FROM_CPU(cpuNum), pAddr );
         ASSERT( !retval || (uint)retval >0x1000);
         return retval;

      }

    
      break;
   case MEM_I_SHARED:
   case MEM_D_SHARED:
      if (new_state == MEM_I_SHARED) { 
         VASSERT( (vAddr == CLEAR_BD(P->PC) || vAddr == CLEAR_BD(P->PC)+INST_SIZE),
                  ("vAddr 0x%x\nPC 0x%x\n",
                   vAddr, P->PC) );
      }
      /* Either the line is invalid and we are reading */
      /* or the line is exclusive and we are executing.  I need to */
      /* see that case to detect self-modified code */
      /* Expensive Assert */
      /* VASSERT( ( (P->qc_v[ADDR2SLINE(vAddr)] == MEM_INVALID) ||
         (VQC_INST(new_state) &&
         (VQC_EXCL(P->qc_v[ADDR2SLINE(vAddr)])) ) ),
         ("%d vAddr 0x%x QC_V 0x%x\n", 
         cpuNum, vAddr, P->qc_v[ADDR2SLINE(vAddr)]) ); */
     
      if( PQC_VALID( pLineInfo ) &&
          ( PQC_VLINE( pLineInfo ) == ADDR2SLINE( vAddr ) ||
            !PQC_VLINE( pLineInfo ) ) &&
          /* The problem is that the kernel can read/write a */
          /* location, then */ 
          /* NOT set the valid bit for the TLB entry which allows user */
          /* access to this location */
          /* Returns 1 for K0 addresses */
       
          Em_Is_Tlb_Readable(cpuNum, vAddr,CURRENT_ASID(cpuNum) ) ) {
  
         /* If we are detecting an execute after write hazard, this */
         /* downgrades the line to read/exec so that future writes */
         /* will be detected */
         /* Otherwise this condition is detected in mem_ref */
         /* VASSERT( cache_verify_shared(cpuNum, pline), 
            ("%d vAddr 0x%x pAddr 0x%x state %d", 
            cpuNum, vAddr, pAddr, new_state) ); EXP */

         set_qc_state(cpuNum, ADDR2SLINE(vAddr), pline, new_state );

         /* Use below line to support parallel cache mode */
         /* return (MPinUP || TOTAL_CPUS == 1) ?
                               PHYS_TO_MEMADDR( M_FROM_CPU(cpuNum), pAddr ) : 0; */

         retval= PHYS_TO_MEMADDR( M_FROM_CPU(cpuNum), pAddr );
         ASSERT( !retval || (uint)retval >0x1000);
         return retval;
      }
      break;
   }
   /* Can't just filter backdoor addresses because some of them are relocated */
   /* Returning a non-zero result causes the quick check to not rewind */
   /* No need to rewind when we are MPinUP */
   retval= mem_ref( vAddr, new_state, cpuNum );
   ASSERT( !retval || (uint)retval >0x1000);
   return retval;
}



/**********************************************************************
 * PhysArray Memory Reference :
 *   called on a miss in the MMU or (hit in MMU and miss in PhysArray)
 **********************************************************************/

unsigned pa_mem_ref(VA vAddr, EmVQCMemState new_state, MA mAddr, int cpuNum)
{


#if SIMBASEADDR != 0x78000000
   /* Dispatch backdoor references quickly */
   if( IS_BACKDOOR(vAddr) ) {
      STAT_INC( backdoor_ref );
      STAT_INC( backdoor_unaltered_ref );
      return vAddr;
   }
#endif


   return (unsigned) mem_ref(vAddr, new_state, cpuNum);
}



/****************************************************************************/
/* Debugging functions.  They verify invariants */
/****************************************************************************/
/* Insure qc maps what the TLB wants mapped */
void qc_consistency_check( int cpuNum )
{
  int i;
  for( i = 0; i < EMP[cpuNum].numTlbEntries; i++) {
     if( IS_VALID(EMP[cpuNum].tlbEntry[i].Lo0) ) {
        if( (uint)EMP[cpuNum].mmu[TLB_ENT2VPN(EMP[cpuNum].tlbEntry[i].Hi)] !=
            (uint)EMP[cpuNum].tlbEntry[i].Lo0 & 0xfffff000 ) {
           CPUPut("%d Help! MMUreloc inconsistent with TLB\n", cpuNum);
           while( 1 )
              ; /* spin */
        }
     }
     if( IS_VALID(EMP[cpuNum].tlbEntry[i].Lo1) ) {
        if( (uint)EMP[cpuNum].mmu[TLB_ENT2VPN(EMP[cpuNum].tlbEntry[i].Hi)|1] !=
            (uint)EMP[cpuNum].tlbEntry[i].Lo1 & 0xfffff000 ) {
           CPUPut("%d Help! MMUreloc inconsistent with TLB\n", cpuNum);
           while( 1 )
              ; /* spin */
        }

     }
  }
}

void 
qc_insure_other_qc_invalid(int cpuNum, PA pAddr)
{
  phys_info_t* phys;
  char* virt;
  int i;
  for( i = 0; i < TOTAL_CPUS; i++ )
    {
      if( i != EMP[cpuNum].myNum )
        {
          phys = &EMP[i].qc_p[ADDR2SLINE(pAddr)];
          ASSERT(!PQC_VALID(*phys));
          virt = &EMP[i].qc_v[PQC_VLINE(*phys)];
          ASSERT( *virt == 0 );
        }
    }
}

void 
qc_insure_other_qc_invalid_or_read(int cpuNum, PA pAddr)
{
  phys_info_t* phys;
  char* virt;
  int i;
  for( i = 0; i < TOTAL_CPUS; i++ )
    {
      if( i != EMP[cpuNum].myNum )
        {
          phys = &EMP[i].qc_p[ADDR2SLINE(pAddr)];
          ASSERT(!PQC_VALID(*phys) || PQC_SHARED(*phys) );
		  /* Can't say anything about virtual addresses because the */
          /* mapping could have changed, and the backmapped virt addr */
          /* points to a new phys addr (which could be owned excl) */
		  /* But now we remove backmap on unmap */
          virt = &EMP[i].qc_v[PQC_VLINE(*phys)];
          ASSERT((*virt == MEM_INVALID) || VQC_SHARED(*virt));
        }
    }
}




/* checks if all MMU entries are also in the TLB with the matching asid */
/* for now just with processor 0 */
void
MMUCheck(int procNum)

{
   int i;
   int myASID = CURRENT_ASID(procNum);
   int writing=0;
   unsigned ppn;
   IDX tlb_index;

   for (i=0;i<(MMU_RELOC_SIZE/4);i++){
      if (EMP[procNum].mmu[i]){
         if ((int)EMP[procNum].mmu[i]<0) writing=1;
         tlb_index = Tlb_Lookup( procNum, GET_REGION(i*DEFAULT_PAGESZ),
                                 CONVERT_TO_VPN2(i), myASID );
      
         if (tlb_index) {
            EntryLo lo;
            tlb_index--;
            /* We have a matching VPN and ASID */
            lo =  (i & 1) ? EMP[procNum].tlbEntry[tlb_index].Lo1 :
                            EMP[procNum].tlbEntry[tlb_index].Lo0;

            if (IS_VALID( lo )) {
               /* Valid */
               if( IS_DIRTY(lo) || !(writing) ) {
                  /* Page is dirty or we are reading - OK */
                  ppn = GET_PFN(lo);
                  if (ppn!=PAGE_NUMBER(((unsigned)EMP[procNum].mmu[i]&0x7fffffff)-MA_TO_UINT(SIM_MEM_ADDR(M_FROM_CPU(procNum)))))
                     CPUPrint("ERROR: MMU Entry VPN %x : %x different from TLB (%x)\n",i,EMP[procNum].mmu[i], ppn);
               } else {
                  /* Page is not dirty and MMU is writable write */
            
                  CPUPrint("ERROR: MMU Entry VPN %x : %x writable, TLB is not!\n",i,EMP[procNum].mmu[i]);
               }
            } else { 
               /* TLB Invalid entry */
               if (!IS_KSEG0(i*DEFAULT_PAGESZ))
                  CPUPrint("ERROR: MMU Entry VPN %x : %x not in TLB\n",i,EMP[procNum].mmu[i]);
            }
        
            if (((unsigned)EMP[procNum].mmu[i]&0x7fffffff)>(MA_TO_UINT(SIM_MEM_ADDR(M_FROM_CPU(procNum)))+MEM_SIZE(M_FROM_CPU(procNum))))
               CPUPrint("ERROR: MMU Entry VPN %x : %x not in MemFile\n",i,EMP[procNum].mmu[i]);
         } else {
            if (!IS_KSEG0(i*DEFAULT_PAGESZ))
               CPUPrint("ERROR: MMU Entry VPN %x : %x not in TLB\n",i,EMP[procNum].mmu[i]);
        
         }
      }
   }
}


void
PACheck(int procNum)

{
   int i,line_no;
   pa_info_t x;
   for (i=0;i<(PA_SIZE(M_FROM_CPU(procNum)));i++){
      x=EMP[procNum].pa_p[i];

      if (x!=PA_SET_INV){
         line_no = i & (SCACHE_INDEX -1);   /* direct mapped */
         if ((PA_SHARED(x)&& !CACHE_SHARED(EMP[procNum].cache_tag[line_no]))
             ||
             (PA_DIRTY(x) && !CACHE_EXCL(EMP[procNum].cache_tag[line_no]))){
            CPUPrint("ERROR : Cache State %x PA : %x , C.Tag : %x\n",i,x,EMP[procNum].cache_tag[line_no]);
         }
         if (CACHE_PLINE(EMP[procNum].cache_tag[line_no])!=i) {
            CPUPrint("ERROR : Wrong Tag pline %x C.Tag.line : %x\n",i,CACHE_PLINE(EMP[procNum].cache_tag[line_no]));
         }
      }

   }

}

#endif