Currently when packets are received on an interface running CARP
and a ICMP error message is generated, the ICMP error source
address is the interface's IP address, not the virtual address.

This has the result that ICMP redirects, unreachables, etc, are
generated with an incorrect source address and are ignored by 
a sending host that is directly attached to a CARP interface.

This patch records when a mbuf was received on a CARP virtual
interface and uses this information to use the virtual IP
on the packet created when returning an ICMP error message.

(As I expect that such information may be required anyway when
 CARP supports having a virtual interface on a different subnet
 to the real interface, I have added this information into 
 struct pkthdr.  An alternative may be to rerun carp_forus in
 icmp_do_error and adjust rcvif there, but I haven't tested
 that.)

Chris Pascoe
2004/06/29

Index: sys/mbuf.h
===================================================================
RCS file: /cvs/src/sys/sys/mbuf.h,v
retrieving revision 1.76
diff -u -p -r1.76 mbuf.h
--- sys/mbuf.h	22 Jun 2004 22:46:30 -0000	1.76
+++ sys/mbuf.h	27 Jun 2004 07:57:46 -0000
@@ -82,6 +82,7 @@ struct	pkthdr {
 	SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */
 	int	len;			/* total packet length */
 	int	csum;			/* Hardware checksum info */
+	struct	ifnet *carpif;		/* carp interface, for ICMP errors */
 };
 
 /* description of external storage mapped into mbuf, valid if M_EXT set */
@@ -237,6 +238,7 @@ struct mbuf *_sk_mget(int, int);
 		(m)->m_data = (m)->m_pktdat; \
 		(m)->m_flags = M_PKTHDR; \
 		SLIST_INIT(&(m)->m_pkthdr.tags); \
 		(m)->m_pkthdr.csum = 0; \
+		(m)->m_pkthdr.carpif = (struct ifnet *)NULL; \
 	} \
 } while (/* CONSTCOND */ 0)
Index: netinet/ip_icmp.c
===================================================================
RCS file: /cvs/src/sys/netinet/ip_icmp.c,v
retrieving revision 1.65
diff -u -p -r1.65 ip_icmp.c
--- netinet/ip_icmp.c	22 Jun 2004 07:35:20 -0000	1.65
+++ netinet/ip_icmp.c	27 Jun 2004 07:57:47 -0000
@@ -86,6 +86,11 @@
 #include <netinet/ip_var.h>
 #include <netinet/icmp_var.h>
 
+#include "carp.h"
+#if NCARP > 0
+#include <netinet/ip_carp.h>
+#endif
+
 /*
  * ICMP routines: error generation, receive packet processing, and
  * routines to turnaround packets back to the originator, and
@@ -241,6 +246,7 @@ icmp_do_error(struct mbuf *n, int type, 
 	m->m_len += sizeof(struct ip);
 	m->m_pkthdr.len = m->m_len;
 	m->m_pkthdr.rcvif = n->m_pkthdr.rcvif;
+	m->m_pkthdr.carpif = n->m_pkthdr.carpif;
 	nip = mtod(m, struct ip *);
 	/* ip_v set in ip_output */
 	nip->ip_hl = sizeof(struct ip) >> 2;
@@ -639,6 +645,11 @@ icmp_reflect(struct mbuf *m)
 			break;
 	}
 	icmpdst.sin_addr = t;
+#if NCARP > 0
+	if ((ia == (struct in_ifaddr *)0) && (m->m_pkthdr.carpif != NULL))
+		ia = ifatoia(ifaof_ifpforaddr(sintosa(&icmpdst),
+		    m->m_pkthdr.carpif));
+#endif
 	if ((ia == (struct in_ifaddr *)0) && (m->m_pkthdr.rcvif != NULL))
 		ia = ifatoia(ifaof_ifpforaddr(sintosa(&icmpdst),
 		    m->m_pkthdr.rcvif));
Index: net/if_ethersubr.c
===================================================================
RCS file: /cvs/src/sys/net/if_ethersubr.c,v
retrieving revision 1.78
diff -u -p -r1.78 if_ethersubr.c
--- net/if_ethersubr.c	26 Jun 2004 06:01:14 -0000	1.78
+++ net/if_ethersubr.c	27 Jun 2004 07:57:47 -0000
@@ -695,8 +695,8 @@ ether_input(ifp, eh, m)
 #endif /* NVLAN > 0 */
 
 #if NCARP > 0
-	if (ifp->if_carp &&
-	    carp_forus(ifp->if_carp, eh->ether_dhost))
+	if (ifp->if_carp && (m->m_pkthdr.carpif = 
+	    carp_forus(ifp->if_carp, eh->ether_dhost)) != NULL)
 		goto decapsulate;
 #endif /* NCARP > 0 */