A fix (Bug #9489: Problem with BIT_OR and MySQL 5.0.3)

heap/hp_hash.c:
  A fix (Bug #9489: Problem with BIT_OR and MySQL 5.0.3).
  Should take into account key pack length.

3 files changed, 34 insertions, 2 deletions

diff --git a/heap/hp_hash.c b/heap/hp_hash.c
index 3121ef71fb0..52a250bd7af 100644
--- a/heap/hp_hash.c
+++ b/heap/hp_hash.c
@@ -255,6 +255,9 @@ ulong hp_hashnr(register HP_KEYDEF *keydef, register const byte *key)
       if (*pos)					/* Found null */
       {
 	nr^= (nr << 1) | 1;
+	/* Add key pack length (2) to key for VARCHAR segments */
+        if (seg->type == HA_KEYTYPE_VARTEXT1)
+          key+= 2;
 	continue;
       }
       pos++;
@@ -390,6 +393,9 @@ ulong hp_hashnr(register HP_KEYDEF *keydef, register const byte *key)
       if (*pos)
       {
 	nr^= (nr << 1) | 1;
+	/* Add key pack length (2) to key for VARCHAR segments */
+        if (seg->type == HA_KEYTYPE_VARTEXT1)
+          key+= 2;
 	continue;
       }
       pos++;
@@ -584,7 +590,12 @@ int hp_key_cmp(HP_KEYDEF *keydef, const byte *rec, const byte *key)
       if (found_null != (int) *key++)
 	return 1;
       if (found_null)
+      {
+        /* Add key pack length (2) to key for VARCHAR segments */
+        if (seg->type == HA_KEYTYPE_VARTEXT1)
+          key+= 2;
 	continue;
+      }
     }
     if (seg->type == HA_KEYTYPE_TEXT)
     {
diff --git a/mysql-test/r/type_varchar.result b/mysql-test/r/type_varchar.result
index d2fe843a68b..3bd7fe6b175 100644
--- a/mysql-test/r/type_varchar.result
+++ b/mysql-test/r/type_varchar.result
@@ -1,4 +1,4 @@
-drop table if exists t1;
+drop table if exists t1, t2;
 create table t1 (v varchar(30), c char(3), e enum('abc','def','ghi'), t text);
 truncate table vchar;
 show create table t1;
@@ -383,3 +383,12 @@ select * from t1;
 pkcol	othercol
 test	somethingelse
 drop table t1;
+create table t1 (a int, b varchar(12));
+insert into t1 values (1, 'A'), (22, NULL);
+create table t2 (a int);
+insert into t2 values (22), (22);
+select t1.a, t1.b, min(t1.b) from t1 inner join t2 ON t2.a = t1.a 
+group by t1.b, t1.a;
+a	b	min(t1.b)
+22	NULL	NULL
+drop table t1, t2;
diff --git a/mysql-test/t/type_varchar.test b/mysql-test/t/type_varchar.test
index 9867cf4c057..2bffca6b889 100644
--- a/mysql-test/t/type_varchar.test
+++ b/mysql-test/t/type_varchar.test
@@ -1,5 +1,5 @@
 --disable_warnings
-drop table if exists t1;
+drop table if exists t1, t2;
 --enable_warnings
 
 create table t1 (v varchar(30), c char(3), e enum('abc','def','ghi'), t text);
@@ -106,3 +106,15 @@ insert into t1 values ('test', 'something');
 update t1 set othercol='somethingelse' where pkcol='test';
 select * from t1;
 drop table t1;
+
+#
+# Bug #9489: problems with key handling
+#
+
+create table t1 (a int, b varchar(12));
+insert into t1 values (1, 'A'), (22, NULL);
+create table t2 (a int);
+insert into t2 values (22), (22);
+select t1.a, t1.b, min(t1.b) from t1 inner join t2 ON t2.a = t1.a 
+  group by t1.b, t1.a;
+drop table t1, t2;