rfbsrc: make authentication handshake safe for multiple instances

Move DES key into context struct.

3 files changed, 39 insertions, 32 deletions

diff --git a/gst/librfb/d3des.c b/gst/librfb/d3des.c
index 2c536780b..785a3a80d 100644
--- a/gst/librfb/d3des.c
+++ b/gst/librfb/d3des.c
@@ -34,9 +34,9 @@
 static void scrunch (unsigned char *, unsigned long *);
 static void unscrun (unsigned long *, unsigned char *);
 static void desfunc (unsigned long *, unsigned long *);
-static void cookey (unsigned long *);
+static void cookey (DESContext * ctx, unsigned long *);
 
-static unsigned long KnL[32] = { 0L };
+//static unsigned long KnL[32] = { 0L };
 
 //static unsigned long KnR[32] = { 0L };
 //static unsigned long Kn3[32] = { 0L };
@@ -47,11 +47,11 @@ static unsigned long KnL[32] = { 0L };
  * 	0x89,0xab,0xcd,0xef,0x01,0x23,0x45,0x67 };
  */
 
-static uint16_t bytebit[8] = {
+static const uint16_t bytebit[8] = {
   01, 02, 04, 010, 020, 040, 0100, 0200
 };
 
-static unsigned long bigbyte[24] = {
+static const unsigned long bigbyte[24] = {
   0x800000L, 0x400000L, 0x200000L, 0x100000L,
   0x80000L, 0x40000L, 0x20000L, 0x10000L,
   0x8000L, 0x4000L, 0x2000L, 0x1000L,
@@ -62,18 +62,18 @@ static unsigned long bigbyte[24] = {
 
 /* Use the key schedule specified in the Standard (ANSI X3.92-1981). */
 
-static unsigned char pc1[56] = {
+static const unsigned char pc1[56] = {
   56, 48, 40, 32, 24, 16, 8, 0, 57, 49, 41, 33, 25, 17,
   9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35,
   62, 54, 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21,
   13, 5, 60, 52, 44, 36, 28, 20, 12, 4, 27, 19, 11, 3
 };
 
-static unsigned char totrot[16] = {
+static const unsigned char totrot[16] = {
   1, 2, 4, 6, 8, 10, 12, 14, 15, 17, 19, 21, 23, 25, 27, 28
 };
 
-static unsigned char pc2[48] = {
+static const unsigned char pc2[48] = {
   13, 16, 10, 23, 0, 4, 2, 27, 14, 5, 20, 9,
   22, 18, 11, 3, 25, 7, 15, 6, 26, 19, 12, 1,
   40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47,
@@ -81,7 +81,7 @@ static unsigned char pc2[48] = {
 };
 
 void
-deskey (unsigned char *key, int32_t edf)
+deskey (DESContext * ctx, unsigned char *key, int32_t edf)
 {                               /* Thanks to James Gillogly & Phil Karn! */
   register int32_t i, j, l, m, n;
   unsigned char pc1m[56], pcr[56];
@@ -120,12 +120,12 @@ deskey (unsigned char *key, int32_t edf)
         kn[n] |= bigbyte[j];
     }
   }
-  cookey (kn);
+  cookey (ctx, kn);
   return;
 }
 
 static void
-cookey (register unsigned long *raw1)
+cookey (DESContext * ctx, register unsigned long *raw1)
 {
   register unsigned long *cook, *raw0;
   unsigned long dough[32];
@@ -143,39 +143,39 @@ cookey (register unsigned long *raw1)
     *cook |= (*raw1 & 0x0003f000L) >> 4;
     *cook++ |= (*raw1 & 0x0000003fL);
   }
-  usekey (dough);
+  usekey (ctx, dough);
   return;
 }
 
 void
-cpkey (register unsigned long *into)
+cpkey (DESContext * ctx, register unsigned long *into)
 {
   register unsigned long *from, *endp;
 
-  from = KnL, endp = &KnL[32];
+  from = ctx->KnL, endp = &ctx->KnL[32];
   while (from < endp)
     *into++ = *from++;
   return;
 }
 
 void
-usekey (register unsigned long *from)
+usekey (DESContext * ctx, register unsigned long *from)
 {
   register unsigned long *to, *endp;
 
-  to = KnL, endp = &KnL[32];
+  to = ctx->KnL, endp = &ctx->KnL[32];
   while (to < endp)
     *to++ = *from++;
   return;
 }
 
 void
-des (unsigned char *inblock, unsigned char *outblock)
+des (DESContext * ctx, unsigned char *inblock, unsigned char *outblock)
 {
   unsigned long work[2];
 
   scrunch (inblock, work);
-  desfunc (work, KnL);
+  desfunc (work, ctx->KnL);
   unscrun (work, outblock);
   return;
 }
@@ -208,7 +208,7 @@ unscrun (register unsigned long *outof, register unsigned char *into)
   return;
 }
 
-static unsigned long SP1[64] = {
+static const unsigned long SP1[64] = {
   0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L,
   0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L,
   0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L,
@@ -227,7 +227,7 @@ static unsigned long SP1[64] = {
   0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L
 };
 
-static unsigned long SP2[64] = {
+static const unsigned long SP2[64] = {
   0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L,
   0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L,
   0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L,
@@ -246,7 +246,7 @@ static unsigned long SP2[64] = {
   0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L
 };
 
-static unsigned long SP3[64] = {
+static const unsigned long SP3[64] = {
   0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L,
   0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L,
   0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L,
@@ -265,7 +265,7 @@ static unsigned long SP3[64] = {
   0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L
 };
 
-static unsigned long SP4[64] = {
+static const unsigned long SP4[64] = {
   0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
   0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L,
   0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L,
@@ -284,7 +284,7 @@ static unsigned long SP4[64] = {
   0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L
 };
 
-static unsigned long SP5[64] = {
+static const unsigned long SP5[64] = {
   0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L,
   0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L,
   0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L,
@@ -303,7 +303,7 @@ static unsigned long SP5[64] = {
   0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L
 };
 
-static unsigned long SP6[64] = {
+static const unsigned long SP6[64] = {
   0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L,
   0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L,
   0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L,
@@ -322,7 +322,7 @@ static unsigned long SP6[64] = {
   0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L
 };
 
-static unsigned long SP7[64] = {
+static const unsigned long SP7[64] = {
   0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L,
   0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L,
   0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L,
@@ -341,7 +341,7 @@ static unsigned long SP7[64] = {
   0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L
 };
 
-static unsigned long SP8[64] = {
+static const unsigned long SP8[64] = {
   0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L,
   0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L,
   0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L,
diff --git a/gst/librfb/d3des.h b/gst/librfb/d3des.h
index 7988eef35..1106fc367 100644
--- a/gst/librfb/d3des.h
+++ b/gst/librfb/d3des.h
@@ -22,7 +22,12 @@
 #define EN0	0               /* MODE == encrypt */
 #define DE1	1               /* MODE == decrypt */
 
-extern void deskey (unsigned char *, int);
+/* init to 0 */
+typedef struct {
+  unsigned long KnL[32];
+} DESContext;
+
+extern void deskey (DESContext * ctx, unsigned char *, int);
 
 /*		      hexkey[8]     MODE
  * Sets the internal key register according to the hexadecimal
@@ -30,20 +35,20 @@ extern void deskey (unsigned char *, int);
  * for encryption or decryption according to MODE.
  */
 
-extern void usekey (unsigned long *);
+extern void usekey (DESContext * ctx, unsigned long *);
 
 /*		    cookedkey[32]
  * Loads the internal key register with the data in cookedkey.
  */
 
-extern void cpkey (unsigned long *);
+extern void cpkey (DESContext * ctx, unsigned long *);
 
 /*		   cookedkey[32]
  * Copies the contents of the internal key register into the storage
  * located at &cookedkey[0].
  */
 
-extern void des (unsigned char *, unsigned char *);
+extern void des (DESContext * ctx, unsigned char *, unsigned char *);
 
 /*		    from[8]	      to[8]
  * Encrypts/Decrypts (according to the key currently loaded in the
diff --git a/gst/librfb/rfbdecoder.c b/gst/librfb/rfbdecoder.c
index 70976590b..edc18ec11 100644
--- a/gst/librfb/rfbdecoder.c
+++ b/gst/librfb/rfbdecoder.c
@@ -424,6 +424,7 @@ rfb_decoder_state_wait_for_security (RfbDecoder * decoder)
       break;
     case SECURITY_VNC:{
       unsigned char key[8], *challenge;
+      DESContext des_ctx;
       gsize password_len;
 
       /*
@@ -449,9 +450,10 @@ rfb_decoder_state_wait_for_security (RfbDecoder * decoder)
         return FALSE;
 
       /* encrypt 16 challenge bytes in place using key */
-      deskey (key, EN0);
-      des (challenge, challenge);
-      des (challenge + 8, challenge + 8);
+      memset (&des_ctx, 0, sizeof (DESContext));
+      deskey (&des_ctx, key, EN0);
+      des (&des_ctx, challenge, challenge);
+      des (&des_ctx, challenge + 8, challenge + 8);
 
       /* .. and send back to server */
       rfb_decoder_send (decoder, challenge, 16);