/*
 * Copyright (C) 1996-1998 by the Board of Trustees
 *    of Leland Stanford Junior University.
 * 
 * This file is part of the SimOS distribution. 
 * See LICENSE file for terms of the license. 
 *
 */

/*****************************************************************
 * ethernet.c - Routines to simulate an ethernet device.
 *
 * Ethernet interface emulation.
 * Created by: Ed Bugnion
 * Revised by: Dan Teodosiu, 07/96
 *
 * NOTES:
 * This interface writes the data directly to memory (and doesn't
 * care about coherence). Should be changed to use DMA.
 *
 * Last modified by: $Author: blythe $
 *****************************************************************/

#include <stdio.h>
#include <sys/types.h>
#ifdef __alpha
#include <sys/ioctl.h>
#else
#ifndef i386
#include <sys/unistd.h>
#endif
#endif
#ifdef i386
#include <sys/ioctl.h>
#endif
#include <sys/mman.h>
#include <sys/file.h>
#include <sys/signal.h>
#ifndef __alpha
#ifndef linux
#include <sys/ioccom.h>
#include <sys/filio.h>
#endif
#endif
#include <sys/time.h>
#include <unistd.h>
#include <string.h>
#include <sys/socket.h>
#include <net/if.h>
#include <errno.h>
#include <netinet/in.h>
#include <stddef.h>
#include <stdlib.h>
#include <netdb.h>

#ifndef linux
#include <sys/uio.h>
#include <netinet/if_ether.h>
#include <poll.h>
#endif

#include "syslimits.h"
#include "simtypes.h"
#include "ethernet.h"
#include "sim.h"
#include "sim_error.h"
#include "simutil.h"
#include "cpu_interface.h"
#include "checkpoint.h"
#include "../../devices/network/cluster.h"
#include "machine_params.h"
#include "arch_specifics.h"
#include "eventcallback.h"
#include "list.h"

#ifdef VCS_FAKE
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/ip.h>
void VcsFakeInit(void);
#endif


static int SimetherCheckpointCB(CptDescriptor *cptd);

/*
 * Seems to be a bug in the SGI offsetof() at 6.2
 */
#if defined(sgi) && defined(_COMPILER_VERSION) && (_COMPILER_VERSION >= 400)
#undef offsetof
#define offsetof(s, m)  (size_t)(&(((s *)0)->m))
#endif

/**********************************************************************
 * offset computation
 **********************************************************************/

static int simetherIndex[sizeof(SimetherRegisters)];

typedef enum {
  NONE,ETHER_ADDR, 
  NUM_RCV,NUM_SND,NUM_CHUNKS,
  INTR_CPU,
  RCV_ADDR, RCV_MAXLEN,RCV_LEN,RCV_FLAG,
  SND_FIRST,SND_LAST,SND_FLAG,
  CHUNK_ADDR, CHUNK_LEN
} RegisterNames ;

char* simetherRegisterNamesStr[] = { 
   "none","ether_addr",
   "num_rcv","num_snd","num_chunk",
   "intr_cpu", 
   "rcv_addr", "rcv_maxLen","rcv_len", "rcv_flag",
   "snd_first", "snd_last", "snd_flag",
   "chunk_addr", "chunk_len" };

static RegisterNames simetherOffset[sizeof(SimetherRegisters)];

static void
SimetherAddEntry(int offset, RegisterNames n, int index)
{
   simetherOffset[offset] = n;
   simetherIndex[offset] = index;
}

#define ADD_ENTRY(_f,_n,_i) \
    SimetherAddEntry(offsetof(SimetherRegisters,_f),_n,_i)

static void 
SimetherOffsetsInit(void)
{
   int i;
   for(i=0;i<sizeof(SimetherRegisters);i++) {
      simetherOffset[i] = NONE;
      simetherIndex[i] = -1;
   }
   for(i=0;i<6;i++) { 
      ADD_ENTRY(etheraddr[i],ETHER_ADDR,i);
   }
   ADD_ENTRY(intrCPU,INTR_CPU,0);
   ADD_ENTRY(numRcvEntries,NUM_RCV,0);
   ADD_ENTRY(numSndEntries,NUM_SND,0);
   ADD_ENTRY(numSndChunks,NUM_CHUNKS,0);
   for(i=0;i<ETHER_MAX_RCV_ENTRIES; i++) {
      ADD_ENTRY(rcvEntries[i].pAddr,RCV_ADDR,i);
      ADD_ENTRY(rcvEntries[i].maxLen,RCV_MAXLEN,i);
      ADD_ENTRY(rcvEntries[i].len,RCV_LEN,i);
      ADD_ENTRY(rcvEntries[i].flag,RCV_FLAG,i);
   }
   for(i=0;i<ETHER_MAX_SND_ENTRIES;i++) { 
      ADD_ENTRY(sndEntries[i].firstChunk,SND_FIRST,i);
      ADD_ENTRY(sndEntries[i].lastChunk,SND_LAST,i);
      ADD_ENTRY(sndEntries[i].flag,SND_FLAG,i);
   }
   for( i=0;i<ETHER_MAX_SND_CHUNKS;i++) {
      ADD_ENTRY(sndChunks[i].pAddr,CHUNK_ADDR,i);
      ADD_ENTRY(sndChunks[i].len,CHUNK_LEN,i);
   }
}

/***************************************************************************
 * 
 * local cluster structures
 *
 ***************************************************************************/

#define USE_LOCAL_CLUSTER 1

#define LOCAL_CLUSTER_DMA_LATENCY 300 /* Nanoseconds per cache-line */
#define LOCAL_CLUSTER_MBIT 10 /* Megabit-per-second */

#define LOCAL_CLUSTER_MAX_MSGS 1024
/* If broadcast is 0, if_num is the interface to send to. */
/* If broadcast is 1, if_num is the only interface not to send to. */
typedef struct LocalClusterMessage {
    EventCallbackHdr hdr;
    List_Links link;
    int if_num;
    int broadcast;
    int len;
    char data[SIMETHER_MAX_TRANSFER_SIZE];
} LocalClusterMessage;

#define CMSG_TO_LIST(_m) (&((_m)->link))
#define LIST_TO_CMSG(_l) ((LocalClusterMessage *) (((char *)(_l)) - sizeof(EventCallbackHdr)))

static int LocalClusterActive = 0;
static LocalClusterMessage messageStorage[LOCAL_CLUSTER_MAX_MSGS];
/* List of available LocalClusterMessage structures */
static List_Links freeMessageList;

static int LocalClusterSendPacket(int cpuNum, int src_if_num,
                                  struct iovec *iov, int io_len);
static void LocalClusterMessageArrive(int cpuNum, EventCallbackHdr *hdr,
                                      void *arg);

/***************************************************************************
 * 
 * simether internals
 *
 ***************************************************************************/

static int disableEthernet = 0;
static int numEtherInterfaces = 0;

typedef struct SimetherState { 
   /*  ethersim/ UPD related  stuff */
   int fd;
   struct sockaddr_in  toaddr;  /* Address of ethernet simulator  process. */
   int  toaddrlen;     /* Length of the address. */
   int  ethernum;
   int  machine;       /* Machine that this controller is on */

   /* ethersim/cluster synchronization */
   char rcvLock[128];
   /* Interaction with the interrupt architecture */
   int intrPosted;
   EtherIntFunc int_f; /* function to call for raising/clearing an int */
    
   /* Ring buffer managment */
   int rcvPtr;
   int chunkOwner[ETHER_MAX_SND_CHUNKS];
   int sndAcks[ETHER_MAX_SND_ENTRIES];

   /* Kernel-visible state (through uncached accesses */
   SimetherRegisters copy;

} SimetherState;

static SimetherState* simetherState = 0;

/* parameters */

char *EthersimHostname;
char *EtherAddress;
int   EtherSendPort;
int   RestoreEthernet;

int
SimetherInit(int restore, EtherIntFunc int_f)
{
   SimetherState *state; 
   extern int errno;
   struct sockaddr_in sin;
   struct hostent *host;
   int if_num;
   int tmp[6];
   int on,i;

   /* Initialize each configured board */
   SimetherOffsetsInit();

   if (!RestoreEthernet) {
      disableEthernet = 1;
   }
   Simcpt_Register("ether", SimetherCheckpointCB, ALL_CPUS);
   if (restore) {
      Simcpt_Restore("ether");
   } else { 
      numEtherInterfaces = TOTAL_ETHER_CONTROLLERS;
      simetherState = (SimetherState *)
         ZMALLOC(sizeof(SimetherState)*numEtherInterfaces,"SimetherState");
   }

   if ((numEtherInterfaces > 1) && USE_LOCAL_CLUSTER) {
       ASSERT(!LocalClusterActive);
       LocalClusterActive = 1;
       List_Init(&freeMessageList);
       for (i = 0; i < LOCAL_CLUSTER_MAX_MSGS; i++) {
           LocalClusterMessage *cmsg = messageStorage + i;
           bzero((char *) (cmsg), sizeof(messageStorage[0]));
           List_InitElement(CMSG_TO_LIST(cmsg));
           List_Insert(CMSG_TO_LIST(cmsg), LIST_ATREAR(&freeMessageList));
       }
   }

   for (if_num=0; if_num<numEtherInterfaces; if_num++) {    
      int j, machine;
      state  = simetherState + if_num;
      state->ethernum = if_num;
      for (machine = 0; machine < NUM_MACHINES; machine++) {
          if ((FIRST_ETHER_CONTROLLER(machine) <= if_num) &&
              (LAST_ETHER_CONTROLLER(machine) >= if_num)) {
              state->machine = machine;
              break;
          }
      }

      /* 
       * hardcoded values for now 
       */
      if (!restore) { 
         state->copy.numRcvEntries = 16;
         state->copy.numSndEntries = ETHER_MAX_SND_ENTRIES;
         state->copy.numSndChunks  = ETHER_MAX_SND_CHUNKS;
         for(j=0;j<state->copy.numRcvEntries;j++)
            state->copy.rcvEntries[j].flag = OS_OWNED;
         for(j=0;j<state->copy.numSndEntries;j++) { 
            state->copy.sndEntries[j].flag = OS_OWNED;
            state->sndAcks[j] = 1;
         }
         for(j=0;j<state->copy.numSndChunks;j++) 
            state->chunkOwner[j] = OS_OWNED;
      }
      /*
       * Create the UDP socket that will be used to send and receive packets`
       * to and from the simulator.
       */
      if( disableEthernet ) {
         state->fd = -1;
         if( sscanf(EtherAddress,"%i:%i:%i:%i:%i:%i",
                    &tmp[0],&tmp[1],&tmp[2],&tmp[3],&tmp[4],&tmp[5])!=6) {
            CPUError("Simether: Parsing of '%'s failed \n", EtherAddress);
         }
         for(i=0;i<5;++i)
            state->copy.etheraddr[i] = tmp[i];
         state->copy.etheraddr[5] = if_num + tmp[5];
	 state->int_f = int_f;
      } else {
         state->fd = socket(AF_INET, SOCK_DGRAM, 0);
         if (state->fd < 0) {
            perror("Simether_init:socket");
            return -1;
         }
         bzero((char *)&sin, sizeof (sin));
         sin.sin_port = htons(0);
         sin.sin_family = AF_INET;
         if (bind(state->fd, (struct sockaddr *) &sin, sizeof (sin)) < 0) {
            perror("Simether_init:bind");
            return -1;
         } 
         on = 1;
         if (ioctl(state->fd, FIONBIO, &on) < 0) {
            perror("Simether_init: FIOBIO");
         }	
            
         host = gethostbyname(EthersimHostname);
         if (!host) {
            Sim_Warning("EthersimHostname : %s\n", EthersimHostname);
            perror("Simether_init:gethostbyname EthersimHostname");
            return -1;
         }
         state->toaddr.sin_family = host->h_addrtype;
         bcopy(host->h_addr, (char*)&state->toaddr.sin_addr, host->h_length);
         state->toaddr.sin_port = htons(EtherSendPort);
         state->toaddrlen = sizeof(state->toaddr);
         if( !restore )
	    /* note: this is currently unused */
            state->copy.intrCPU = SIM_MAXCPUS;
         if( sscanf(EtherAddress,"%i:%i:%i:%i:%i:%i",
                    &tmp[0],&tmp[1],&tmp[2],&tmp[3],&tmp[4],&tmp[5])!=6) {
            CPUError("Simether: Parsing of '%'s failed \n", EtherAddress);
         }
         for(i=0;i<5;++i)
            state->copy.etheraddr[i] = tmp[i];
         state->copy.etheraddr[5] = if_num + tmp[5];
	 state->int_f = int_f;
      }
   }
   
#ifdef VCS_FAKE
   VcsFakeInit();
#endif
    
   return 0;
}


void
SimetherPoll(void)
{
#ifndef linux
   static struct pollfd fds[ETHER_MAX_CONTROLLERS];
   SimetherState *state; 
   int i;

   for (i=0; i < numEtherInterfaces; i++) {
     state  = &simetherState[i];
     ASSERT(state->fd != 0);
     fds[i].fd = state->fd;
     fds[i].events = POLLRDNORM;
     fds[i].revents = 0;
   }
   poll(fds, numEtherInterfaces, 0);

   for (i = 0; i < numEtherInterfaces; i++) {
      SIM_DEBUG(('i', "SimetherPoll fd: 0x%x  events: 0x%x  revents: 0x%x\n",
                 fds[i].fd, fds[i].events, fds[i].revents));
   }
   
   for(i = 0; i < numEtherInterfaces; i++)
     if (fds[i].revents & POLLRDNORM) SimetherReceivePacket( i, 0 , 0 ) ;
#else
  Sim_Warning("Uh oh... SimetherPoll called\n");
#endif

}


void SimetherInitCluster(void)
{
   int i, j;
   char etheraddr[6];

   for (i = 0; i < numEtherInterfaces; i++) {
     for (j = 0; j < 6; j++) etheraddr[j] = simetherState[i].copy.etheraddr[j];
     ClusterSetEtherAddr(etheraddr, i);
   }
}

char* SimetherAddr(int iface_num) 
{
   static char etheraddr[6];
   int i;

   for (i = 0; i < 6; i++)
     etheraddr[i] = simetherState[iface_num].copy.etheraddr[i];
   return etheraddr;
}


static void 
SimetherClearSlot(SimetherState *state) 
{
  int i;
  SimetherRegisters *regs = &state->copy;
  for(i=0;i<state->copy.numRcvEntries;i++) { 
    if( regs->rcvEntries[i].flag == OS_OWNED ) 
      return;
  }
  for(i=0;i<state->copy.numSndEntries;i++) {
    if( regs->sndEntries[i].flag != OS_OWNED || 
	!state->sndAcks[i] ) 
      return;
  }
  /* clear the interrupt bit */
  if (state->intrPosted) {
    /* clear the interrupt line */
    SIM_DEBUG(('e', "ETHER-trace: %d ClearSlot\n", state->ethernum)); 
    if (state->int_f) state->int_f(state->ethernum, 0);
      state->intrPosted = 0; /* just like EWOULDBLOCK case */
  }
}

static void 
SimetherRaiseSlot(SimetherState *state, char* cause)
{
  if (!state->intrPosted) {
    SIM_DEBUG(('e', "ETHER-trace: %d RaiseSlot (%s)\n", state->ethernum, cause));
    if (state->int_f) state->int_f(state->ethernum, 1);
    state->intrPosted = 1;
  } else {
    SIM_DEBUG(('e', "ETHER-trace: %d RaiseSlot (%s) already posted\n",
	     state->ethernum, cause));
  }
}

static void 
SimetherSendOver(SimetherState *state, int index) 
{
   int i,lastChunk;
   /* release send chunks */
   i = state->copy.sndEntries[index].firstChunk;
   lastChunk = state->copy.sndEntries[index].lastChunk;
   while( 1 ) { 
      state->chunkOwner[i] = OS_OWNED;
      if( i==lastChunk ) break;
      i++;
      if( i >= state->copy.numSndChunks ) 
         i -= state->copy.numSndChunks ;
   }
   /* Packet fully sent */
   state->copy.sndEntries[index].flag = OS_OWNED;
 
   /* Set done flag immediately. should be base mode only */
   SimetherRaiseSlot( state, "send" );
}

#ifdef VCS_FAKE
/* Types of packets */
#define VCS_FAKE_REQUEST 0
#define VCS_FAKE_REPLY 1
#define VCS_FAKE_ERROR 2
/* Markers, in case we get lost in the packet */
#define VCS_FAKE_MARKER "01xy"
#define VCS_FAKE_SMARKER "01xy"
#define VCS_FAKE_RMARKER "02xy"
#define VCS_FAKE_EMARKER "03xy"
#define VCS_FAKE_MSIZE 4

#define VCS_FAKE_OFF 0 
#define VCS_FAKE_SAVE 1
#define VCS_FAKE_MATCH 2

/* These two defines are reaching into the structure
   below. So if that changes (Fat chance!) we must change
   the defines too. They take a ethernet packet and return
   the IP ID field or the IP Checksum field
*/
/* #define VCS_FAKE_IPID(x) (*((u_short *)(((u_char *)(x)) + 18))) */
#define VCS_FAKE_IPID(x) (((struct ip *)(((u_char *)(x)) + sizeof(struct ether_header)))->ip_id)
/* #define VCS_FAKE_IPCKSUM(x) (*((u_short *)(((u_char *)(x)) + 24))) */
#define VCS_FAKE_IPCKSUM(x) (((struct ip *)(((u_char *)(x)) + sizeof(struct ether_header)))->ip_sum)

/* structure for storing the previously traced packets for comparison */
#define VCS_FAKE_MAX_BUF 40	/* maximum number of captured packets */
struct ipbuf_t {
  int len;
  int replyLen;
  int used;
  char *request;
  char *reply;
} ipbuf[VCS_FAKE_MAX_BUF];

static int ipbufid = 0;
static int vfd;		/* file descriptor for storing or matching */
static int efd;
static char dumpbuf[1600];	/* need this for copying data at various points */

int VcsFakeType = 0;	 /* Type of operation to be done, nothing, save or match */
char *VcsFakeFilename = NULL; /* filename for writing or reading packets */

/* 
   If we are using VCSFAKESAVE, just open the file to save the packets.
   If VCSFAKEMATCH, then read in the packets from the file and store them away.
*/
void
VcsFakeInit(void)
{
  u_long marker, prevmarker = -1, dbuflen;
  struct ipbuf_t *ptr, *replyPtr;

  if(VcsFakeType == VCS_FAKE_OFF) {
    /* default behavior, just return */
    return;
  }

  /* if in Save mode, just open the file and return */
  if(VcsFakeType == VCS_FAKE_SAVE) {
    if((vfd = open(VcsFakeFilename, O_CREAT|O_RDWR|O_TRUNC, 0777)) == NULL) {
      CPUError("Error: Could not open dump file %s\n", VcsFakeFilename);
      vfd = -1;
    }
    return;
  }

  if(VcsFakeType == VCS_FAKE_MATCH) {
    if((vfd = open(VcsFakeFilename, O_RDONLY, 0777)) == NULL) {
      CPUError("Error: Could not open dump file %s\n", VcsFakeFilename);
      vfd = -1;
      return;
    }
    
    CPUWarning("Opened VcsFake dumpfile\n");

    while(read(vfd, &marker, VCS_FAKE_MSIZE) == VCS_FAKE_MSIZE) {
      if(!bcmp(&marker, VCS_FAKE_SMARKER, VCS_FAKE_MSIZE)) {
	marker = VCS_FAKE_REQUEST;
      } else if(!bcmp(&marker, VCS_FAKE_RMARKER, VCS_FAKE_MSIZE)) {
	marker = VCS_FAKE_REPLY;
      } else {
	CPUError("VcsFakeEther: Unknown marker");
	marker = -1;
	return;
      }

      if(marker == VCS_FAKE_REQUEST) {
	ptr = &(ipbuf[ipbufid++]);
	ptr->used = 0;
	read(vfd, &(ptr->len), sizeof(ptr->len));
	ptr->request = (char *)ZMALLOC(ptr->len,"etherReq");
	read(vfd, ptr->request, ptr->len);
	VCS_FAKE_IPID(ptr->request) = 0;
	VCS_FAKE_IPCKSUM(ptr->request) = 0;
	CPUWarning("REQ (%d): SRC = 0x%x DST = 0x%x\n", ptr->len, 
		    *((int *)&(ptr->request[26])), *((int *)&(ptr->request[30])));
	ptr->reply = 0;
	ptr->replyLen = 0;
      } else if(marker == VCS_FAKE_REPLY) {
	if(prevmarker == VCS_FAKE_REQUEST) {
	  read(vfd, &(ptr->replyLen), sizeof(ptr->replyLen));
	  ptr->reply = (char *)ZMALLOC(ptr->replyLen,"VCS");
	  read(vfd, ptr->reply, ptr->replyLen);
	  CPUWarning("REP (%d): SRC = 0x%x DST = 0x%x\n", ptr->replyLen, 
		    *((int *)&(ptr->reply[26])), *((int *)&(ptr->reply[30])));
	} else {
	  /* Reply without a matching request */
	  read(vfd, &(dbuflen), sizeof(u_long));
	  read(vfd, dumpbuf, dbuflen);
	  CPUWarning("REP (Unsolicited)(%d): SRC = 0x%x DST = 0x%x\n", ptr->replyLen, 
		    *((int *)&(dumpbuf[26])), *((int *)&(dumpbuf[30])));
	}
      }
      prevmarker = marker;
    }
    CPUWarning("VcsFake init done\n");
  }

}
#endif

/********************************************************************
 * SimetherSendPacket
 * Returns 0 if the interrupt was posted, 1 if the transmission is
 * still in flight
 ********************************************************************/


static int SimetherSendPacket(SimetherState *state, int index ) 
{
#ifndef i386
   int err, i, j, lastChunk;
   struct msghdr  msg;
   int length = 0;
   int cpuNum = FIRST_CPU(state->machine); 

   /* The following two cannot be on the stack since in base
    * mode we get to this point while running on the kernel stack.
    * They take waayyy to much memory and cause a TLB miss
    * while in the back door.
    *
    * Making them static appears to be safe because each cpu process
    * gets its own copy.  In non-base-mode runs we are safe because
    * this code is non-reentrant.
    */
   static struct iovec iovec[ETHER_MAX_SND_CHUNKS+1];
   static char extraBuf[ETHERMIN + sizeof(struct ether_header)];

   j = state->copy.sndEntries[index].firstChunk;
   lastChunk = state->copy.sndEntries[index].lastChunk;
   for(i=0;i<ETHER_MAX_SND_CHUNKS+1; i ++) { 
      ASSERT(IS_VALID_PA(M_FROM_CPU(cpuNum), state->copy.sndChunks[j].pAddr));
      iovec[i].iov_base = PHYS_TO_MEMADDR(M_FROM_CPU(cpuNum),
                                          state->copy.sndChunks[j].pAddr);
      iovec[i].iov_len  = state->copy.sndChunks[j].len;
      length += state->copy.sndChunks[j].len;
      if( j == lastChunk ) { 
         i++;
         break;
      }
      j++;
      if( j >= state->copy.numSndChunks  ) 
         j -= state->copy.numSndChunks ;
   } 
   ASSERT( i <=state->copy.numSndChunks );

   if (length < ETHERMIN + sizeof(struct ether_header)) {
      bzero(extraBuf,sizeof(extraBuf));
      iovec[i].iov_base = extraBuf;
      iovec[i].iov_len = ETHERMIN + sizeof(struct ether_header) - length;
      length =  ETHERMIN + sizeof(struct ether_header);
      i++;
   }

   {
      int j;
      SIM_DEBUG_DETAIL(('i', "ETHER-trans", cpuNum, "controller=%d ",
                        state->ethernum));
      for(j=0; j<i; j++) {
         CPUPrint(" %x/%i",  iovec[j].iov_base, iovec[j].iov_len);
      }
      CPUPrint("\n");
   }
   
   msg.msg_name = (caddr_t) &state->toaddr;
   msg.msg_namelen = state->toaddrlen;
   msg.msg_iov = iovec;
   msg.msg_iovlen = i;
   msg.msg_accrights = (caddr_t) 0;
   msg.msg_accrightslen = 0;
   if( ClusterSendPacket(cpuNum,iovec, msg.msg_iovlen) ||
       LocalClusterSendPacket(cpuNum, state->ethernum,
                              iovec, msg.msg_iovlen) ) {
      /* packet sent through the cluster */
      /* no timing model for now */
      SimetherSendOver(state,index);
      return 0; /* override the copy */

   } else { 
#ifdef VCS_FAKE
     if(VcsFakeType == VCS_FAKE_MATCH) {
       int i, j = 0;
       struct ipbuf_t *ptr;
       struct sockaddr_in myaddr; 
       int myaddrlen, matchNoReply = -1;

       /* Since we have to zero out the checksum and id, 
	  make a copy in a single buffer */
       for(i=0;i<msg.msg_iovlen;i++) { 
	 bcopy(iovec[i].iov_base, &(dumpbuf[j]), iovec[i].iov_len);
	 j += iovec[i].iov_len;
       }
       /* zero the IP id and check sum fields of the packet */
       VCS_FAKE_IPID(dumpbuf) = 0;
       VCS_FAKE_IPCKSUM(dumpbuf) = 0;
       for(i=0;i<ipbufid;i++) {
	 ptr = &(ipbuf[i]);
	 if(!(ptr->used) && (ptr->len == length) &&
	    (bcmp(ptr->request, dumpbuf, length) == 0)) {
	   /* found a match */
	   iovec[0].iov_base = ptr->reply;
	   iovec[0].iov_len = ptr->replyLen;
	   myaddrlen = sizeof(struct sockaddr_in);
	   getsockname(state->fd, &myaddr, &myaddrlen);
	   msg.msg_name = (caddr_t) &myaddr;
	   msg.msg_namelen = myaddrlen;
	   msg.msg_iov = iovec;
	   msg.msg_iovlen = 1;
	   ptr->used = 1;
	   break;
	 }
       }

       err = 0;

       if(i == ipbufid) {
	 /* did not find a match */
	 CPUError("VcsFakeEther: No match (%d) SRC = 0x%x DST = 0x%x\n", length, 
		    *((int *)&(dumpbuf[26])), *((int *)&(dumpbuf[30])));
	 /*
	   write(vfd, VCS_EMARKER, VCS_MSIZE);
	   write(vfd, &length, sizeof(length));
	   write(vfd, devPtr->req.buffer, length);
	   */
	 err = -1;
       } else if(iovec[0].iov_len) {
	   /* found a match with a reply, send the message. It is
	      has been switched previously to be the reply, addressed to
	      me. 
	      */
	   CPUWarning("VCS_FAKE: Match(%d) SRC = 0x%x DST = 0x%x\n", ptr->len, 
		       *((int *)&(ptr->request[26])), *((int *)&(ptr->request[30])));
	   /*
	   if (sendmsg(state->fd, &msg, 0) < 0) {
	     CPUError("simetherOutput: sendmsg");
	     err = -1;
	   }
	   */
	   SimetherSendOver(state,index);
	   if(SimetherReceivePacket(state->ethernum, dumpbuf, length) < 0) {
	     CPUError("simetherOutput: sendmsg");
	     err = -1;
	   }
	   return(0);
       } else {
	 /* Found a match, but there was no associated reply. Do nothing */
	 CPUWarning("VCS_FAKE: Match No reply (%d) SRC = 0x%x DST = 0x%x\n", 
		    ptr->len, *((int *)&(ptr->request[26])), *((int *)&(ptr->request[30])));
       }

       SimetherSendOver(state,index);
       return 0; 
     }
#endif

      if (sendmsg(state->fd, &msg, 0) < 0) {
         perror("simetherOutput: sendmsg");
         err = -1;
      } else {
#ifdef VCS_FAKE
	if(VcsFakeType == VCS_FAKE_SAVE) {
	  /* Send was successful, save the packet */
	  int i;
	  
	  write(vfd, VCS_FAKE_SMARKER, VCS_FAKE_MSIZE);
	  write(vfd, &length, sizeof(length));
	  for(i=0;i<msg.msg_iovlen;i++) { 
	    write(vfd, iovec[i].iov_base, iovec[i].iov_len);
	  }
	}
#endif
         err = 0;
      }
      SimetherSendOver(state,index);
      return err; /* override the copy */
   }
   /* return 1;- make compiler happy */  /* allow the copy */
#else 
  Sim_Warning("Uh oh... SimetherSendPacked called\n");
  return 1;
#endif

}

/***********************************************************************
 * SimetherReceivePacket.
 *
 * Called either in interrupt dispatcher (simcp0 poll) when using UDP
 * to carry or in the normal context of the CPU simulator if we bypass
 * UPD.
 *
 * !!! Never install a callback from when using UDP
 *
 ************************************************************************/

int
SimetherReceivePacket(int iface_num, char *packet, int packetSize)
{ 
#ifndef i386
   SimetherState *state = simetherState +iface_num;
   int size = 0;
   struct sockaddr_in fromaddr;
   int  fromaddrlen;
   char buffer[SIMETHER_MAX_TRANSFER_SIZE];
   int cpuNum = FIRST_CPU(state->machine);
   ASSERT( iface_num < ETHER_MAX_CONTROLLERS);
   /* 
    * Choose a free receiving buffer or drop the packet 
    * in this model, we do not drop packets on the floor, but
    * delay the processing of the UDP packets.
    */
    
   if( !packet ) {
      ASSERT( !disableEthernet );
      /* the packet comes from the UDP port */
        
      if( state->copy.rcvEntries[state->rcvPtr].flag == OS_OWNED ) {
         /* 
          * wait for the kernel to process
          */
         SIM_DEBUG(('e', "ETHER-receive: %d Recieve buffers full \n", state->ethernum));
         LogEntry("ETHER",cpuNum,"controller=%d receive buffers full\n",
                  state->ethernum);
#ifdef sgi
         sginap( 1 ); /* free up CPU */
#endif
#ifdef sun
	sleep(1);
#endif
         return -1;
      }
        
      fromaddrlen = sizeof(fromaddr);
      size = recvfrom(state->fd, buffer,SIMETHER_MAX_TRANSFER_SIZE,
                      0, (struct sockaddr *) &fromaddr, &fromaddrlen);
      if (size < 0) {
         if (errno != EWOULDBLOCK)
            perror("simetherRecvProcess:recvfrom");
      } else if (size < sizeof(struct ether_header)) {
         Sim_Warning("simetherRecvProcess: Ethernet packet too small (%d)\n",
                     size);
      }
      ASSERT( state->copy.rcvEntries[state->rcvPtr].flag == CONTROLLER_OWNED);

      packet = buffer;
      packetSize = size;
#ifdef VCS_FAKE
      if(VcsFakeType == VCS_FAKE_SAVE) {
	/* save the received packet to the file */
	write(vfd, VCS_FAKE_RMARKER, VCS_FAKE_MSIZE);
	write(vfd, &size, sizeof(size));
	write(vfd, buffer, size);
      }
#endif
      /*DMA the contents of the packet */
   } else { 
      if( state->copy.rcvEntries[state->rcvPtr].flag == OS_OWNED ) {
         CPUWarning("CLUSTER-packet: drop packet \n");
         return -1;
      }
      ASSERT( packetSize );
   } 
   bcopy(packet,
         (void *)PHYS_TO_MEMADDR(M_FROM_CPU(cpuNum),
                                 state->copy.rcvEntries[state->rcvPtr].pAddr),
         packetSize);
        
#if 0
    CPUPrint(" ------ incoming packet DMA-ed to : 0x%x len=%i  ----------\n",
             PHYS_TO_MEMADDR(M_FROM_CPU(cpuNum),
                             (char*)state->copy.rcvEntries[state->rcvPtr].pAddr),
             packetSize  );
    for(i=0;i<packetSize;i++) { 
      char  x =  packet[i];
      if( x && x < 127 ) { 
        CPUPrint("%c",x);
      } else { 
        CPUPrint("(%x)", x);
      }
      CPUPrint("\n");
    }
    CPUPrint("\n------------------------------------\n");
#endif
 
   /* Update ring buffer */
   state->copy.rcvEntries[state->rcvPtr].len = packetSize;
   state->copy.rcvEntries[state->rcvPtr].flag = OS_OWNED;
   state->rcvPtr++;
   if( state->rcvPtr >= state->copy.numRcvEntries  ) {
      state->rcvPtr -=  state->copy.numRcvEntries ;
   }
 
   /* timing model ? */
   if( 1 ) {
      /* uses UPD. No use for a timing model */
      SimetherRaiseSlot( state, "receive" );
      if( packetSize > 0 )
         LogEntry("ETHER-receive",
                  cpuNum,
                  "controller=%d len=%4i ring \n",
                  state->ethernum, packetSize);
   }
   return packetSize;
#else
   Sim_Warning("Uh oh.. in SimetherReceivePacket\n");
   return 0;
#endif
}






/**********************************************************************
 * OS interface
 **********************************************************************/

EthRegister
SimetherIO(int iface_num, int offset, int is_write, EthRegister data)
{
  SimetherRegisters *regs  = &simetherState[iface_num].copy;
  SimetherState     *state = simetherState + iface_num;
  int                index = simetherIndex[offset];
  int                copy  = 1;
  EthRegister  newVal, oldVal;
  int                cpuNum=FIRST_CPU(state->machine);

  if (iface_num >= numEtherInterfaces) {
     CPUPrint("ETHER: cpu=%i accessed out-of-range controller num %i ",
              cpuNum, iface_num);
     return 0;
  }

  ASSERT( offset < sizeof(SimetherRegisters) );
  ASSERT( iface_num < ETHER_MAX_CONTROLLERS );
  ASSERT( index >= 0 );

  newVal = data;
  oldVal = *(EthRegister *) ((char*)regs+offset);

#ifdef notdef
  if( SimConfigGetBool("Ether.Trace")) { 
    if( !is_write ) {
      CPUPrint("ETHER-trace: %lld cpu=%i intr=%i controller=%d "
	       "RD (%-10s %2i) val=%3d\n", 
	       (CPUVec.CycleCount ?
		(uint)CPUVec.CycleCount(cpuNum) : 0 ),
	       cpuNum,  state->intrPosted, state->ethernum,
	       simetherRegisterNamesStr[simetherOffset[offset]],
	       index, oldVal);
    } else { 
      CPUPrint("ETHER-trace: %lld cpu=%i intr=%i controller=%d "
	       "WR (%-10s %2i) val=%3d newVal=%3d\n", 
	       (CPUVec.CycleCount ?
		(uint)CPUVec.CycleCount(cpuNum) : 0 ), 
	       cpuNum,  state->intrPosted, state->ethernum,
	       simetherRegisterNamesStr[simetherOffset[offset]],
	       index, oldVal,newVal);
    }
  }
#endif

  SIM_DEBUG(('e', "ETHER-trace: %d IO %-6s (%-10s,%2d) val=%3d newVal=%3d \n", 
             state->ethernum,
             (is_write?"WRITE":"READ"),
              simetherRegisterNamesStr[simetherOffset[offset]],
              index, oldVal,newVal)); 
            
  
  switch( simetherOffset[offset] ) { 
  case NONE:          ASSERT(0); copy = 0; break;

  case ETHER_ADDR:    ASSERT( !is_write ); return regs->etheraddr[index];

  case NUM_RCV:
  case NUM_SND:
  case NUM_CHUNKS:    ASSERT( !is_write ); break;

  case INTR_CPU: 
    ASSERT( state->copy.intrCPU == SIM_MAXCPUS );
    ASSERT( !state->intrPosted );
    regs->intrCPU = newVal;  /* must be set before SimmpVec->intrCon..
			      * increment */   
    break; /* simply copy. */

  case RCV_ADDR: 
  case RCV_MAXLEN:
    ASSERT( regs->rcvEntries[index].flag == OS_OWNED );
    ASSERT( is_write );
    break;

  case RCV_LEN:
    ASSERT( regs->rcvEntries[index].flag == OS_OWNED );
    ASSERT( !is_write );

  case RCV_FLAG: 
    ASSERT( index >= 0 && index < regs->numRcvEntries );
    ASSERT( !is_write || regs->rcvEntries[index].flag == OS_OWNED );
    ASSERT( !is_write || newVal == CONTROLLER_OWNED );
    SimetherClearSlot(state); /* ClearSlot if everything is received. ** */
    break;

  case SND_FIRST:
  case SND_LAST: 
    ASSERT( regs->sndEntries[index].flag == OS_OWNED );
    ASSERT( is_write );
    break;

  case SND_FLAG: 
    ASSERT( index >= 0 && index < regs->numSndEntries );
    ASSERT( !is_write || regs->sndEntries[index].flag == OS_OWNED );
    ASSERT( !is_write || newVal == CONTROLLER_OWNED );
    if( !is_write ) {
      if( regs->sndEntries[index].flag == OS_OWNED ) { 
	ASSERT( !state->sndAcks[index] );
	state->sndAcks[index] = 1;
	/* ClearSlot if everything is received */
	SimetherClearSlot(state);         }
    } else {  
      int i;
      state->sndAcks[index] = 0;
      i = regs->sndEntries[index].firstChunk;
      while( 1 ) { 
	ASSERT( state->chunkOwner[i] == OS_OWNED );
	state->chunkOwner[i] = CONTROLLER_OWNED;
	if( i == regs->sndEntries[index].lastChunk )
	  break;
	i++;
	if( i>= state->copy.numSndChunks   ) 
	  i -= state->copy.numSndChunks ;
      }
      copy = SimetherSendPacket(state,index);
    }
    break;
  case CHUNK_ADDR: 
  case CHUNK_LEN: 
    ASSERT( index >= 0 && index < regs->numSndChunks );
    ASSERT( state->chunkOwner[index] == OS_OWNED );
    break;
  default: 
    ASSERT( 0 );

  }

  if( copy && is_write ) 
     *(EthRegister *)(((char *)regs) + offset) = newVal;
  return oldVal;
}


/* NOTE:
 * For checkpoint format compatibility with the old version, we checkpoint
 * the ether state as ints, instead of as uint's. At some point, this
 * should be changed.
 */
static int
SimetherCheckpointCB(CptDescriptor *cptd)
{
  int          if_num,i;
  unsigned int tmp;
  
  numEtherInterfaces = TOTAL_ETHER_CONTROLLERS;
  if (cptVersion.ver == 3) {
     Simcpt_CptInt(cptd,"NumControllers",NO_INDEX,NO_INDEX,&(numEtherInterfaces));
  }

  if (cptd->mode == CPT_RESTORE) {
    ASSERT( !simetherState );
    simetherState = (SimetherState *)
      ZMALLOC(sizeof(SimetherState)*numEtherInterfaces,"SimetherState");
  } 
  for(if_num = 0; if_num< numEtherInterfaces; if_num++) {

    SimetherState     *state = simetherState + if_num;
    SimetherRegisters *regs  = &state->copy;

    /* 
     * I/O accessible state 
     */
    for(i=0;i<6;i++) {
      char c = regs->etheraddr[i];
      Simcpt_CptChar(cptd,"etheraddr",if_num,i,&c);
      regs->etheraddr[i] = c;
    }

    tmp = regs->numRcvEntries;
    Simcpt_CptUint(cptd, "numRcvEntries",if_num, NO_INDEX,&tmp);
    regs->numRcvEntries = tmp;

    tmp = regs->numSndEntries;
    Simcpt_CptUint(cptd, "numSndEntries",if_num, NO_INDEX,&tmp);
    regs->numSndEntries = tmp;

    tmp = regs->numSndChunks;
    Simcpt_CptUint(cptd, "numChunks",if_num, NO_INDEX,&tmp);
    regs->numSndChunks = tmp;

    for(i=0;i<regs->numRcvEntries;i++) {
      tmp = regs->rcvEntries[i].pAddr;
      Simcpt_CptHex(cptd, "rcvpAddr",if_num, i,&tmp);
      regs->rcvEntries[i].pAddr = tmp;
      
      tmp = regs->rcvEntries[i].maxLen;
      Simcpt_CptHex(cptd, "rcvmaxLen",if_num, i,&tmp);
      regs->rcvEntries[i].maxLen = tmp;

      tmp = regs->rcvEntries[i].len;
      Simcpt_CptHex(cptd, "rcvlen",if_num, i,&tmp);
      regs->rcvEntries[i].len = tmp;

      tmp = regs->rcvEntries[i].flag;
      Simcpt_CptHex(cptd, "rcvflag",if_num, i,&tmp);
      regs->rcvEntries[i].flag = tmp;
    }

    for(i=0;i<regs->numSndEntries;i++) { 
      tmp = regs->sndEntries[i].firstChunk;
      Simcpt_CptHex(cptd, "sndfirst",if_num, i,&tmp);
      regs->sndEntries[i].firstChunk = tmp;

      tmp = regs->sndEntries[i].lastChunk;
      Simcpt_CptHex(cptd, "sndlast",if_num, i,&tmp);
      regs->sndEntries[i].lastChunk = tmp;

      tmp = regs->sndEntries[i].flag;
      Simcpt_CptHex(cptd, "sndflag",if_num, i,&tmp);
      regs->sndEntries[i].flag = tmp;
    }

    for(i=0;i<regs->numSndChunks;i++) {
      tmp = regs->sndChunks[i].pAddr;
      Simcpt_CptHex(cptd, "chunkspAddr",if_num, i,&tmp);
      regs->sndChunks[i].pAddr = tmp;

      tmp = regs->sndChunks[i].len;
      Simcpt_CptHex(cptd, "chunklen",if_num, i,&tmp);
      regs->sndChunks[i].len = tmp;
    }
                   
    Simcpt_CptInt(cptd, "ethernum",if_num, NO_INDEX, &(state->ethernum));
    ASSERT( state->ethernum == if_num );

    /* Formerly placed in the prom checkpoint file */
    tmp = regs->intrCPU;
    Simcpt_CptUint(cptd, "etherIntrCPU", if_num, NO_INDEX, &tmp);
    regs->intrCPU = tmp;
      
    Simcpt_CptInt(cptd, "intrPosted",if_num, NO_INDEX, &(state->intrPosted));
    Simcpt_CptInt(cptd,"rcvPtr",if_num, NO_INDEX, &(state->rcvPtr));
      
    for(i=0;i<regs->numSndChunks;i++) {
      Simcpt_CptInt(cptd, "chunkOwner",if_num, i,&state->chunkOwner[i]);
    }
    for(i=0;i<regs->numSndEntries;i++) {
      Simcpt_CptInt(cptd, "sndAcks",if_num, i,&state->sndAcks[i]);
    }
  }
  return 0;
}

static int
LocalClusterSendPacket(int cpuNum, int src_if_num,
                       struct iovec *iov, int io_len)
{
#ifndef linux
    LocalClusterMessage *cmsg;
    List_Links *itemPtr;
    struct ether_header* etherhdr =  (struct ether_header*)iov[0].iov_base;
    unsigned char *destaddr = (unsigned char *)&etherhdr->ether_dhost;
    int broadcast = 0;
    int if_num, j, len = 0;
    SimTime time;

    if (!LocalClusterActive)
        return 0;

/*
 * Find out if the controller we want is in this simulation. If not, use ethersim,
 * otherwise determine the interface number we want
 */

    
    CPUPrint("LOCAL_CLUSTER: cpu %d looking for dest etheraddr = %i:%i:%i:%i:%i:%i\n ", 
               cpuNum,destaddr[0],destaddr[1],destaddr[2], 
               destaddr[3],destaddr[4],destaddr[5]);
    
    ASSERT(iov[0].iov_len >= sizeof(struct ether_header));

    for (if_num = 0; if_num < numEtherInterfaces; if_num++) {
        for (j = 0; j < 6; j++) {
            if (destaddr[j] != simetherState[if_num].copy.etheraddr[j]) {
                break;
            }
        }
        if (j == 6) break; /* found */
    }
    if (if_num == numEtherInterfaces) {
        for (j = 0; j < 6; j++) {
            if (destaddr[j] != 0xff) {
                break;
            }
        }
        if (j == 6) {
           CPUPrint("LOCAL_CLUSTER: found broadcast address (from interface %d)\n", 
                    src_if_num); 
           if_num = src_if_num;   /* Don't send back to me */
            broadcast = 1;
        } else {
            /* Address is not within this simulation */
           CPUPrint("LOCAL_CLUSTER: Could not find etheraddr = %i:%i:%i:%i:%i:%i, going through ethersim \n", 
              destaddr[0],destaddr[1],destaddr[2], 
              destaddr[3],destaddr[4],destaddr[5]); 
        return 0; /* can't find the etheraddr */
        }      
    } else {
        /* destination is controller if_num */
         CPUPrint("LOCAL_CLUSTER: found at contr=%i  etheraddr = %i:%i:%i:%i:%i:%i \n", 
                  if_num, 
                  simetherState[if_num].copy.etheraddr[0], 
                  simetherState[if_num].copy.etheraddr[1], 
                  simetherState[if_num].copy.etheraddr[2], 
                  simetherState[if_num].copy.etheraddr[3], 
                  simetherState[if_num].copy.etheraddr[4], 
                  simetherState[if_num].copy.etheraddr[5]); 
    }        

/*
 * Create the event callback
 */
    if (List_IsEmpty(&freeMessageList)) {
        CPUWarning("LOCAL_CLUSTER: Ethernet buffer overflowed (increase size or reduce latency)\n");
        ASSERT(0);
    }
    itemPtr = List_First(&freeMessageList);
    ASSERT(itemPtr);
    List_Remove(itemPtr);
    cmsg = LIST_TO_CMSG(itemPtr);

    cmsg->if_num = if_num;
    cmsg->broadcast = broadcast;
    for(j=0;j<io_len;j++) {
        bcopy(iov[j].iov_base,cmsg->data+len,iov[j].iov_len);
        len += iov[j].iov_len;
    }
    cmsg->len = len;

    /* Simulate DMA time */
    time = (NanoSecsToCycles(LOCAL_CLUSTER_DMA_LATENCY) * len) /
        SCACHE_LINE_SIZE;
    /* Add transfer delay time */
    time += NanoSecsToCycles(((SimTime)len * 8000) / LOCAL_CLUSTER_MBIT);

    EventDoCallback(cpuNum, LocalClusterMessageArrive,
                    (EventCallbackHdr *)cmsg, NULL, time); 
#else
    CPUError("Not ported to linux\n");
#endif 
    return 1;
}

static void
LocalClusterMessageArrive(int cpuNum, EventCallbackHdr *hdr, void *arg)
{
    LocalClusterMessage *cmsg = (LocalClusterMessage *)hdr;
    int i;
    
    ASSERT(cmsg->if_num >= 0 && cmsg->if_num < numEtherInterfaces);
    
    if (cmsg->broadcast) {
       /*
       CPUPrint("LOCAL_CLUSTER: Broadcast message arrived from %d.\n", 
                cmsg->if_num); 
                */
        for (i = 0; i < numEtherInterfaces; i++) {
            if (i != cmsg->if_num) {
                SimetherReceivePacket(i, cmsg->data, cmsg->len);
            }
        }
    } else {
       /* 
       CPUWarning("LOCAL_CLUSTER: message arrived at %d from %d.\n", 
                  cpuNum, cmsg->if_num);
                  */
       SimetherReceivePacket(cmsg->if_num, cmsg->data, cmsg->len);
    }

    List_Insert(CMSG_TO_LIST(cmsg), LIST_ATREAR(&freeMessageList));
}