
	These were sent to me by Ahmon Dancy but haven't been integrated
	yet.
	
	-- Emil

diff -c -N -r src.old/conntrack.c src/conntrack.c
*** src.old/conntrack.c	Wed Dec 31 16:00:00 1969
--- src/conntrack.c	Sun Jan 19 14:41:20 2003
***************
*** 0 ****
--- 1,124 ----
+ /* (C) 2003, Ahmon Dancy */
+ 
+ #include <malloc.h>
+ #include "darkstat.h"
+ #include "bignum.h"
+ #include "hash.h"
+ #include "conntrack.h"
+ 
+ /* Since this is mainly for http spy tracking, the timeout
+    can be fairly low. */
+ #define CONNTRACK_TIMEOUT 15
+ 
+ #define CONNTRACK_HASHSIZE 50
+ 
+ hash_table *conntrack_ht;
+ pthread_mutex_t conntrack_mutex=PTHREAD_MUTEX_INITIALIZER;
+ 
+ void conntrack_init_ht() {
+   conntrack_ht=make_hash_table(CONNTRACK_HASHSIZE);
+ }
+ 
+ hash_key *conntrack_make_key() {
+   hash_key *key=malloc(sizeof(hash_key));
+ 
+   if (!key) {
+     perror("malloc");
+     exit(1);
+   }
+ 
+   key->data=malloc(sizeof(conntrack_key));
+   if (!key->data) {
+     perror("malloc");
+     exit(1);
+   }
+   
+   key->len=sizeof(conntrack_key);
+   
+   return key;
+ }
+ 
+ void conntrack_free_key(hash_key *key) {
+   free(key->data);
+   free(key);
+ }
+ 
+ hash_key *conntrack_fill_key(hash_key *key,
+ 			     dword host1, word port1,
+ 			     dword host2, word port2) {
+   conntrack_key *keydata=(conntrack_key *)key->data;
+   
+   if (host1 < host2) {
+     keydata->host1=host1;
+     keydata->port1=port1;
+     keydata->host2=host2;
+     keydata->port2=port2;
+   } else {
+     keydata->host1=host2;
+     keydata->port1=port2;
+     keydata->host2=host1;
+     keydata->port2=port1;
+   }
+   
+   return key;
+ }
+ 
+ conntrack_conn *conntrack_lookup(dword host1, word port1,
+ 				 dword host2, word port2) {
+   hash_key *key=conntrack_make_key();
+   conntrack_conn **conptrptr, *con;
+   
+   conntrack_fill_key(key, host1, port1, host2, port2);
+ 
+   pthread_mutex_lock(&conntrack_mutex);
+     
+   conptrptr=(conntrack_conn **)hash_lookup(conntrack_ht, key);
+   con=*conptrptr;
+   if (!con) {
+     /* new connection */
+     con=calloc(sizeof(conntrack_conn), 1);
+     if (!con) {
+       perror("calloc");
+       exit(1);
+     }
+     *conptrptr=con;
+   }
+   
+   con->mtime=time(NULL);
+   
+   conntrack_free_key(key);
+ 
+   pthread_mutex_unlock(&conntrack_mutex);
+   
+   return con;
+ }
+ 
+ int conntrack_reaper_help(hash_key *key, void **dataptrptr) {
+   time_t now;
+   conntrack_conn *con=(conntrack_conn *)*dataptrptr;
+   
+   now=time(0);
+ 
+   if (now > con->mtime+CONNTRACK_TIMEOUT) {
+     free(con);
+     *dataptrptr=NULL;
+     return -1;
+   } else {
+     return 0;
+   }
+ }
+ 
+ void conntrack_reaper(void *ignored) {
+   conntrack_init_ht();
+ 
+   printf("CONN: Connection tracking reaper is up and running.\n");
+   up_conn=1;
+   while (!shutting_down ) {
+     pthread_mutex_lock(&conntrack_mutex);
+     hash_iterate(conntrack_ht, conntrack_reaper_help);
+     pthread_mutex_unlock(&conntrack_mutex);
+     sleep(1);
+   }
+   printf("CONN: Connection tracking reaper stopped.\n");
+   up_conn=0;
+ }
diff -c -N -r src.old/conntrack.h src/conntrack.h
*** src.old/conntrack.h	Wed Dec 31 16:00:00 1969
--- src/conntrack.h	Sun Jan 19 13:47:26 2003
***************
*** 0 ****
--- 1,25 ----
+ #ifndef _CONNTRACK_H_
+ #define _CONNTRACK_H_
+ 
+ #include <time.h>
+ #include "hash.h"
+ 
+ typedef struct conntrack_key {
+   dword host1, host2;
+   word port1, port2;
+ } conntrack_key;
+ 
+ typedef struct conntrack_conn {
+   void *data;
+   time_t mtime;
+ } conntrack_conn;
+ 
+ 
+ /* conntrack.c */
+ void conntrack_init_ht(void);
+ hash_key *conntrack_make_key(void);
+ void conntrack_free_key(hash_key *key);
+ hash_key *conntrack_fill_key(hash_key *key, dword host1, word port1, dword host2, word port2);
+ conntrack_conn *conntrack_lookup(dword host1, word port1, dword host2, word port2);
+ void conntrack_reaper(void *ignored);
+ #endif
diff -c -N -r src.old/darkstat.c src/darkstat.c
*** src.old/darkstat.c	Fri Jan  3 19:09:57 2003
--- src/darkstat.c	Sun Jan 19 13:49:00 2003
***************
*** 39,49 ****
  
  int	verbose = 0, webport = DEFAULT_WEBPORT, want_spy = 0,
  	webip = -1, shutting_down = 0, promisc = 1,
! 	up_acct=0, up_www=0, up_dns=0, up_spy=0;
  char	*errdb_file = NULL, *db_file = NULL, *daylog_file = NULL,
  	*fprefix = NULL, *acctdev = NULL, *acctexpr = NULL, *pid_file = NULL;
  pthread_t www_thread = (pthread_t)0, dns_thread = (pthread_t)0,
! 	  spy_thread = (pthread_t)0, acct_thread = (pthread_t)0;
  
  
  
--- 39,51 ----
  
  int	verbose = 0, webport = DEFAULT_WEBPORT, want_spy = 0,
  	webip = -1, shutting_down = 0, promisc = 1,
! 	up_acct=0, up_www=0, up_dns=0, up_spy=0, up_conn=0;
  char	*errdb_file = NULL, *db_file = NULL, *daylog_file = NULL,
  	*fprefix = NULL, *acctdev = NULL, *acctexpr = NULL, *pid_file = NULL;
  pthread_t www_thread = (pthread_t)0, dns_thread = (pthread_t)0,
!           spy_thread = (pthread_t)0, acct_thread = (pthread_t)0,
!           conn_thread = (pthread_t)0;
! 
  
  
  
***************
*** 97,102 ****
--- 99,105 ----
  		if (up_dns) printf("dns ");
  		if (up_www) printf("www ");
  		if (up_spy) printf("spy ");
+ 		if (up_conn) printf("conn ");
  		printf("\n");
  		return;
  	}
***************
*** 429,441 ****
  
  
  
! 	while ((i = up_acct+up_dns+up_www+up_spy))
  	{
  		printf("Syncing... %d %s", i, (i==1)?"thread":"threads");
  		if (up_acct) printf(" (acct)");
  		if (up_dns) printf(" (dns)");
  		if (up_www) printf(" (www)");
  		if (up_spy) printf(" (spy)");
  		printf("\n");
  		usleep(1000 * MSEC);
  	}
--- 432,445 ----
  
  
  
! 	while ((i = up_acct+up_dns+up_www+up_spy+up_conn))
  	{
  		printf("Syncing... %d %s", i, (i==1)?"thread":"threads");
  		if (up_acct) printf(" (acct)");
  		if (up_dns) printf(" (dns)");
  		if (up_www) printf(" (www)");
  		if (up_spy) printf(" (spy)");
+ 		if (up_conn) printf(" (conn)");
  		printf("\n");
  		usleep(1000 * MSEC);
  	}
***************
*** 444,449 ****
--- 448,454 ----
  	if (www_thread) pthread_join(www_thread, NULL);
  	if (dns_thread) pthread_join(dns_thread, NULL);
  	if (spy_thread) pthread_join(spy_thread, NULL);
+ 	if (conn_thread) pthread_join(conn_thread, NULL);
  
  	pthread_mutex_destroy(&db_mutex);
  	pthread_mutex_destroy(&graph_mutex);
diff -c -N -r src.old/darkstat.h src/darkstat.h
*** src.old/darkstat.h	Thu Jan  2 12:07:18 2003
--- src/darkstat.h	Sun Jan 19 13:48:30 2003
***************
*** 63,69 ****
  typedef enum {MAIN, IN, OUT, TOTAL} sort_type;
  
  extern int verbose, shutting_down, promisc;
! extern int up_dns, up_www, up_spy, up_acct;
  extern char *acctdev, *acctexpr;
  
  char *build_str(char *format, ...);
--- 63,69 ----
  typedef enum {MAIN, IN, OUT, TOTAL} sort_type;
  
  extern int verbose, shutting_down, promisc;
! extern int up_dns, up_www, up_spy, up_acct, up_conn;
  extern char *acctdev, *acctexpr;
  
  char *build_str(char *format, ...);
***************
*** 71,77 ****
  void print_addr(const dword ip);
  void save_db(const char *filename);
  
! 
  
  /* packet stuff */
  dword get_local_ip(char *inter);
--- 71,78 ----
  void print_addr(const dword ip);
  void save_db(const char *filename);
  
! #include <pthread.h>
! pthread_t conntrack_thread;
  
  /* packet stuff */
  dword get_local_ip(char *inter);
diff -c -N -r src.old/hash.c src/hash.c
*** src.old/hash.c	Wed Dec 31 16:00:00 1969
--- src/hash.c	Sun Jan 19 14:53:18 2003
***************
*** 0 ****
--- 1,162 ----
+ #include <malloc.h>
+ #include <assert.h>
+ #include <pthread.h>
+ #include "hash.h"
+ 
+ /* (C) 2003, Ahmon Dancy */
+ 
+ hash_table *make_hash_table(int size) {
+   hash_table *ht=calloc(sizeof(hash_table), 1);
+   
+   if (!ht) {
+     perror("calloc");
+     exit(1);
+   }
+   
+   ht->size=size;
+   pthread_mutex_init(&ht->mutex, NULL);
+   ht->buckets=calloc(sizeof(hash_bucket *), size);
+   if (!ht->buckets) {
+     perror("calloc");
+     exit(1);
+   }
+   
+   return ht;
+ }
+ 
+ unsigned int hash_compute_index(hash_key *key) {
+   unsigned int index=0, i;
+   char *p=key->data;
+ 
+   for(i=0; i<key->len; i++) {
+     index+=*p++;
+   }
+ 
+   return index;
+ }
+ 
+ int hash_keys_match(hash_key *key1, hash_key *key2) {
+   if (key1->len != key2->len)
+     return 0;
+   
+   if (memcmp(key1->data, key2->data, key1->len) == 0)
+     return 1;
+   else
+     return 0;
+ }
+   
+ void **hash_lookup(hash_table *ht, hash_key *key) {
+   unsigned int slot;
+   hash_bucket *bucket;
+   
+   pthread_mutex_lock(&ht->mutex);
+ 
+   slot=hash_compute_index(key) % ht->size;
+   bucket=ht->buckets[slot];
+   
+   while (bucket) {
+     if (hash_keys_match(key, &bucket->key))
+       goto finishup;
+     bucket=bucket->next;
+   }
+ 
+   /* Create a new bucket */
+   bucket=calloc(sizeof(hash_bucket), 1);
+   if (!bucket) {
+     perror("calloc");
+     exit(1);
+   }
+ 
+   /* populate it */
+   bucket->key.len=key->len;
+   bucket->key.data=malloc(key->len);
+   if (!bucket->key.data) {
+     perror("malloc");
+     exit(1);
+   }
+   memcpy(bucket->key.data, key->data, key->len);
+ 
+   /* Put it on the front of the chain */
+   bucket->next=ht->buckets[slot];
+   if (bucket->next) 
+     bucket->next->prev=bucket;
+   ht->buckets[slot]=bucket;
+   ht->entries++;
+  
+  finishup:
+   pthread_mutex_unlock(&ht->mutex);
+   return &bucket->data;
+ }
+ 
+ void hash_remove_help(hash_table *ht, hash_bucket *bucket, unsigned int slot) {
+   if (bucket->next) 
+     bucket->next->prev=bucket->prev;
+ 
+   if (bucket->prev)
+     bucket->prev->next=bucket->next;
+   else 
+     ht->buckets[slot]=bucket->next;
+   
+   assert(bucket->data == NULL);
+   free(bucket->key.data);
+   free(bucket);
+   
+   ht->entries--;
+   
+ }
+   
+ void hash_remove(hash_table *ht, hash_key *key) {
+   unsigned int slot;
+   hash_bucket *bucket;
+   
+   pthread_mutex_lock(&ht->mutex);
+ 
+   slot=hash_compute_index(key) % ht->size;
+   bucket=ht->buckets[slot];
+   
+   while (bucket) {
+     if (hash_keys_match(key, &bucket->key))
+       break;
+     bucket=bucket->next;
+   }
+   
+   if (!bucket) {
+     pthread_mutex_unlock(&ht->mutex);
+     return;
+   }
+  
+   hash_remove_help(ht, bucket, slot);
+  
+   pthread_mutex_unlock(&ht->mutex);  
+ 
+   return;
+ }
+ 
+ hash_key *make_hash_key_from_string(char *str) {
+   hash_key *key=calloc(sizeof(hash_key), 1);
+   
+   key->data=str;
+   key->len=strlen(str);
+ 
+   return key;
+ }
+ 
+ void hash_iterate(hash_table *ht, int (* iter)(hash_key *, void **)) {
+   int i;
+   hash_bucket *bucket, *next;
+   
+   pthread_mutex_lock(&ht->mutex);
+   
+   for(i=0; i<ht->size; i++) {
+     bucket=ht->buckets[i];
+ 
+     while (bucket) {
+       next=bucket->next;
+       if (iter(&bucket->key, &bucket->data)==-1)
+ 	hash_remove_help(ht, bucket, i);
+       bucket=next;
+     }
+   }
+  
+   pthread_mutex_unlock(&ht->mutex);
+ }
diff -c -N -r src.old/hash.h src/hash.h
*** src.old/hash.h	Wed Dec 31 16:00:00 1969
--- src/hash.h	Sun Jan 19 13:59:13 2003
***************
*** 0 ****
--- 1,30 ----
+ #ifndef _HASH_H_
+ #define _HASH_H_
+ 
+ #include <pthread.h>
+ 
+ typedef struct hash_key {
+   char *data;
+   int len;
+ } hash_key;
+ 
+ typedef struct hash_bucket {
+   hash_key key;
+   void *data;
+   struct hash_bucket *next, *prev;
+ } hash_bucket;
+ 
+ typedef struct hash_table {
+   int size, entries;
+   hash_bucket **buckets;
+   pthread_mutex_t mutex;
+ } hash_table;
+ 
+ 
+ hash_table *make_hash_table(int size);
+ unsigned int hash_compute_index(hash_key *key);
+ void **hash_lookup(hash_table *ht, hash_key *key);
+ void hash_remove(hash_table *ht, hash_key *key);
+ hash_key *make_hash_key_from_string(char *str);
+ 
+ #endif
diff -c -N -r src.old/spylog.c src/spylog.c
*** src.old/spylog.c	Sun Jan  5 21:55:17 2003
--- src/spylog.c	Sun Jan 19 15:01:06 2003
***************
*** 5,10 ****
--- 5,12 ----
  
  #include "spylog.h"
  #include "port_db.h"
+ #include "conntrack.h"
+ #include "vhost.h"
  
  #include <pthread.h>
  #include <stdio.h>
***************
*** 49,60 ****
  #define DISTANCE 12
  
  static void spylog_packet(byte *packet, word length,
! 		dword srcip, dword destip, word destport)
  {
  	char method[6], *url, *host = NULL;
  	int start = 0, end, found = 0;
  	time_t now;
  	struct tm *tm;
  
  	if (length < 6) return; /* strlen("get /") */
  
--- 51,70 ----
  #define DISTANCE 12
  
  static void spylog_packet(byte *packet, word length,
! 		dword srcip, word srcport, dword destip, word destport)
  {
  	char method[6], *url, *host = NULL;
  	int start = 0, end, found = 0;
  	time_t now;
  	struct tm *tm;
+ 	conntrack_conn *con=conntrack_lookup(srcip, srcport, 
+ 					     destip, destport);
+ 	vhost *vh;
+ 	
+ 	if (con->data) {
+ 	  vh=(vhost *)con->data;
+ 	  i64add32(vh->count, length);
+ 	}
  
  	if (length < 6) return; /* strlen("get /") */
  
***************
*** 105,110 ****
--- 115,123 ----
  		host = (char*)malloc(end - start + 1);
  		memcpy(host, packet+start, end-start);
  		host[end-start] = 0;
+ 		
+ 		vh=vhost_get(host, length);
+ 		con->data=(void *)vh;
  	}
  	else if (destport == 80)
  		host = build_str("%d.%d.%d.%d",
***************
*** 153,159 ****
  	word		pkt_type;
  	dword		pkt_srcip, pkt_destip;
  	byte		*tcp_ptr;
! 	word		destport;
  
  	/* assert(strcmp(user, "def") == 0); */
  
--- 166,172 ----
  	word		pkt_type;
  	dword		pkt_srcip, pkt_destip;
  	byte		*tcp_ptr;
! 	word		destport, srcport;
  
  	/* assert(strcmp(user, "def") == 0); */
  
***************
*** 165,174 ****
  
  	tcp_ptr = ip_pkt_strip(ptr);
  	destport = tcp_pkt_destport(tcp_ptr);
  
  	spylog_packet(tcp_ptr + 20,
  		      pdata + pheader->len - tcp_ptr - 20,
! 		      pkt_srcip, pkt_destip, destport);
  }
  
  
--- 178,188 ----
  
  	tcp_ptr = ip_pkt_strip(ptr);
  	destport = tcp_pkt_destport(tcp_ptr);
+ 	srcport = tcp_pkt_srcport(tcp_ptr);
  
  	spylog_packet(tcp_ptr + 20,
  		      pdata + pheader->len - tcp_ptr - 20,
! 		      pkt_srcip, srcport, pkt_destip, destport);
  }
  
  
***************
*** 217,222 ****
--- 231,244 ----
  	pcap_freecode(&spyprog);
  #endif
  	free(prog);
+ 
+ 	if (pthread_create(&conntrack_thread, NULL, (void*)&conntrack_reaper, 
+ 			   NULL) == -1)
+ 	  {
+ 	    printf("Error: could not create CONN thread.\n");
+ 	    exit(1);
+ 	}
+ 	init_vhost_hash();
  
  	/* spylogging loop */
  	printf("SPY: Thread is up and logging.\n");
diff -c -N -r src.old/vhost.c src/vhost.c
*** src.old/vhost.c	Wed Dec 31 16:00:00 1969
--- src/vhost.c	Sun Jan 19 15:00:10 2003
***************
*** 0 ****
--- 1,40 ----
+ #include <malloc.h>
+ #include "hash.h"
+ #include "bignum.h"
+ #include "vhost.h"
+ #include "conntrack.h"
+ 
+ hash_table *vhost_hash;
+ 
+ void init_vhost_hash() {
+   vhost_hash=make_hash_table(VHOST_HASH_SIZE);
+ }
+ 
+ vhost *vhost_get(char *vhostname, int length) {
+   hash_key *key=make_hash_key_from_string(vhostname);
+   vhost *vh, **vhostptrptr;
+ 
+   vhostptrptr=(vhost **)hash_lookup(vhost_hash, key);
+   vh=*vhostptrptr;
+ 
+   if (!vh) {
+     vh=calloc(sizeof(vhost), 1);
+     if (!vh) {
+       perror("calloc");
+       exit(1);
+     }
+     *vhostptrptr=vh;
+   }
+   
+   i64add32(vh->count, length);
+   
+   free(key);
+ 
+   return vh;
+ }
+ 
+ /* We use the data pointer of the connection to hold the vhost
+    information */
+ 
+ void set_vhost_of_connection(conntrack_conn *con, char *vhostname) {
+ }
diff -c -N -r src.old/vhost.h src/vhost.h
*** src.old/vhost.h	Wed Dec 31 16:00:00 1969
--- src/vhost.h	Thu Jan 16 18:48:20 2003
***************
*** 0 ****
--- 1,12 ----
+ #include "conntrack.h"
+ 
+ #define VHOST_HASH_SIZE 50
+ 
+ typedef struct vhost {
+   int64 count;
+ } vhost;
+ 
+ /* vhost.c */
+ void init_vhost_hash(void);
+ vhost *vhost_get(char *vhostname, int length);
+ void set_vhost_of_connection(conntrack_conn *con, char *vhostname)
