Introduce dict_col_t::was_not_null()

For dual-format pages, we must know whether columns were originally
NOT NULL, so that we can correctly interpret records in the old leaf
pages of the dual-format primary key index.

dict_table_t::init_instant(): Set the new DATA_WAS_NOT_NULL flags
and index->n_nullable based on it.

11 files changed, 74 insertions, 61 deletions

diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc
index 28542b61198..3a8e4fc8aa5 100644
--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@@ -2712,7 +2712,7 @@ dict_index_add_col(
 	change, which would be a disaster. */
 	compile_time_assert(DICT_MAX_FIXED_COL_LEN == 768);
 
-	if (!(col->prtype & DATA_NOT_NULL)) {
+	if (!col->was_not_null()) {
 		index->n_nullable++;
 	}
 }
@@ -5029,10 +5029,9 @@ scan_on_conditions:
 	}
 
 	for (j = 0; j < foreign->n_fields; j++) {
-		if ((dict_index_get_nth_col(foreign->foreign_index, j)->prtype)
-		    & DATA_NOT_NULL) {
-			const dict_col_t*	col
-				= dict_index_get_nth_col(foreign->foreign_index, j);
+		const dict_col_t*	col
+			= dict_index_get_nth_col(foreign->foreign_index, j);
+		if (!col->is_nullable()) {
 			const char* col_name = dict_table_get_col_name(foreign->foreign_index->table,
 				dict_col_get_no(col));
 
@@ -5500,7 +5499,7 @@ dict_index_calc_min_rec_len(
 {
 	ulint	sum	= 0;
 	ulint	i;
-	ulint	comp	= dict_table_is_comp(index->table);
+	ulint	comp	= dict_table_is_comp(index->table);// FIXME
 
 	if (comp) {
 		ulint nullable = 0;
@@ -5514,7 +5513,7 @@ dict_index_calc_min_rec_len(
 				size = col->len;
 				sum += size < 128 ? 1 : 2;
 			}
-			if (!(col->prtype & DATA_NOT_NULL)) {
+			if (col->is_nullable()) {
 				nullable++;
 			}
 		}
@@ -6764,8 +6763,7 @@ dict_foreign_qualify_index(
 			return(false);
 		}
 
-		if (check_null
-		    && (field->col->prtype & DATA_NOT_NULL)) {
+		if (check_null && !field->col->is_nullable()) {
 			if (error && err_col_no && err_index) {
 				*error = DB_FOREIGN_KEY_COL_NOT_NULL;
 				*err_col_no = i;
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index da440326231..62f6c534671 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -157,38 +157,51 @@ inline void dict_table_t::init_instant(const dict_table_t& table)
 		DBUG_ASSERT(table.instant->leaf_redundant
 			    || !instant->leaf_redundant);
 		instant->leaf_redundant= table.instant->leaf_redundant;
+		if (const auto omap = table.instant->non_pk_col_map) {
+			memcpy(non_pk_col_map, omap,
+			       (index.n_fields - u) * sizeof *omap);
+		}
 	}
 
 	ut_d(unsigned n_drop = 0);
-	ut_d(unsigned n_nullable = 0);
+	unsigned n_nullable = 0;
 	for (unsigned i = u; i < index.n_fields; i++) {
 		auto& f = index.fields[i];
 		DBUG_ASSERT(dict_col_get_fixed_size(f.col, not_redundant())
 			    <= DICT_MAX_FIXED_COL_LEN);
-#ifdef UNIV_DEBUG
-		if (!f.col->is_nullable()) {
-			ut_ad((*non_pk_col_map & 3U << 14) != 1U << 14);
-		} else if ((*non_pk_col_map & 3U << 14) != 1U << 14) {
-			n_nullable++;
-		}
-#endif
 		if (!f.col->is_dropped()) {
 			auto c = *non_pk_col_map & 3U << 14;
 			DBUG_ASSERT(!(c & 1U << 15));
-			if (!c
-			    && f.col->is_nullable()
-			    && !oindex.fields[i].col->is_nullable()) {
+			if (!f.col->is_nullable()) {
+				DBUG_ASSERT(!c);
+			} else if (oindex.fields[i].col->was_not_null()) {
 				c = 1U << 14;
-				ut_d(n_nullable--);
+				f.col->prtype |= DATA_WAS_NOT_NULL;
+				ut_ad(f.col->was_not_null());
+			} else {
+				n_nullable++;
 			}
 			*non_pk_col_map++ = c | f.col->ind;
 			continue;
 		}
 
+		if (f.col->was_not_null()) {
+			DBUG_ASSERT(!table.instant
+				    || !table.instant->non_pk_col_map
+				    || (*non_pk_col_map & 1U << 14));
+		} else if (!(*non_pk_col_map & 1U << 14)) {
+			n_nullable++;
+		} else {
+			f.col->prtype |= DATA_WAS_NOT_NULL;
+		}
+
+		DBUG_ASSERT(f.col->was_not_null()
+			    == oindex.fields[i].col->was_not_null());
+
 		auto fixed_len = dict_col_get_fixed_size(
 			f.col, not_redundant());
 		*non_pk_col_map++ = 1U << 15
-			| uint16_t(!f.col->is_nullable()) << 14
+			| uint16_t(f.col->was_not_null()) << 14
 			| (fixed_len
 			   ? uint16_t(fixed_len + 1)
 			   : f.col->len > 255);
@@ -205,7 +218,8 @@ inline void dict_table_t::init_instant(const dict_table_t& table)
 	}
 	ut_ad(n_drop == n_dropped());
 	ut_ad(non_pk_col_map == &instant->non_pk_col_map[index.n_fields - u]);
-	ut_ad(index.n_nullable == n_nullable);
+	ut_ad(index.n_nullable >= n_nullable);
+	index.n_nullable = n_nullable;
 }
 
 /** Acquire a page latch on the possible metadata record,
@@ -504,7 +518,7 @@ inline void dict_index_t::instant_add_field(const dict_index_t& instant)
 	for (unsigned i = 0; i < n_fields; i++) {
 		const dict_col_t* icol = instant.fields[i].col;
 		dict_field_t& f = fields[i];
-		ut_d(n_null += icol->is_nullable());
+		ut_d(n_null += !icol->was_not_null());
 		DBUG_ASSERT(!icol->is_virtual());
 		if (icol->is_dropped()) {
 			ut_d(n_dropped++);
@@ -1619,7 +1633,7 @@ innobase_fts_check_doc_id_col(
 			internally created FTS_DOC_ID column. */
 			ut_ad(col->mtype == DATA_INT);
 			ut_ad(col->len == 8);
-			ut_ad(col->prtype & DATA_NOT_NULL);
+			ut_ad(!col->is_nullable());
 			ut_ad(col->prtype & DATA_UNSIGNED);
 #endif /* UNIV_DEBUG */
 			*fts_doc_col_no = i;
@@ -2336,10 +2350,9 @@ innobase_check_fk_option(
 			     | DICT_FOREIGN_ON_DELETE_SET_NULL)) {
 
 		for (ulint j = 0; j < foreign->n_fields; j++) {
-			if ((dict_index_get_nth_col(
-				     foreign->foreign_index, j)->prtype)
-			    & DATA_NOT_NULL) {
-
+			const dict_col_t* col = dict_index_get_nth_col(
+				foreign->foreign_index, j);
+			if (!col->is_nullable()) {
 				/* It is not sensible to define
 				SET NULL if the column is not
 				allowed to be NULL! */
@@ -3497,7 +3510,7 @@ innobase_fts_check_doc_id_index(
 		if (strcmp(field->name, FTS_DOC_ID_COL_NAME) == 0
 		    && field->col->mtype == DATA_INT
 		    && field->col->len == 8
-		    && field->col->prtype & DATA_NOT_NULL
+		    && !field->col->is_nullable()
 		    && !field->col->is_virtual()) {
 			if (fts_doc_col_no) {
 				*fts_doc_col_no = dict_col_get_no(field->col);
@@ -4907,7 +4920,8 @@ static bool innodb_insert_sys_columns(
 	pars_info_add_int4_literal(info, "pos", pos);
 	pars_info_add_str_literal(info, "name", field_name);
 	pars_info_add_int4_literal(info, "mtype", mtype);
-	pars_info_add_int4_literal(info, "prtype", prtype);
+	pars_info_add_int4_literal(info, "prtype", prtype
+				   & ~DATA_WAS_NOT_NULL);
 	pars_info_add_int4_literal(info, "len", len);
 	pars_info_add_int4_literal(info, "base", n_base);
 
diff --git a/storage/innobase/include/data0type.h b/storage/innobase/include/data0type.h
index 3b3ac9d1885..ee22edc309c 100644
--- a/storage/innobase/include/data0type.h
+++ b/storage/innobase/include/data0type.h
@@ -195,6 +195,9 @@ be less than 256 */
 /** system-versioned user data column */
 #define DATA_VERSIONED (DATA_VERS_START|DATA_VERS_END)
 
+/** the original NOT NULL attribute of a column before instant ALTER */
+#define DATA_WAS_NOT_NULL (1U << 31)
+
 /** Check whether locking is disabled (never). */
 #define dict_table_is_locking_disabled(table) false
 
diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h
index da374b51840..0024cc874f8 100644
--- a/storage/innobase/include/dict0mem.h
+++ b/storage/innobase/include/dict0mem.h
@@ -611,6 +611,11 @@ public:
 	bool is_virtual() const { return prtype & DATA_VIRTUAL; }
 	/** @return whether NULL is an allowed value for this column */
 	bool is_nullable() const { return !(prtype & DATA_NOT_NULL); }
+	/** @return whether this column originally was NOT NULL
+	@see dict_index_t::dual_format() */
+	bool was_not_null() const {
+		return prtype & (DATA_WAS_NOT_NULL | DATA_NOT_NULL);
+	}
 
 	/** @return whether table of this system field is TRX_ID-based */
 	bool vers_native() const
@@ -697,7 +702,8 @@ public:
 			&& mbminlen == other.mbminlen
 			&& mbmaxlen == other.mbmaxlen
 			&& !((prtype ^ other.prtype)
-			     & ~(DATA_NOT_NULL | DATA_VERSIONED));
+			     & ~(DATA_NOT_NULL | DATA_VERSIONED
+				 | DATA_WAS_NOT_NULL));
 	}
 };
 
diff --git a/storage/innobase/mtr/mtr0log.cc b/storage/innobase/mtr/mtr0log.cc
index c50deb667d9..9e227249b29 100644
--- a/storage/innobase/mtr/mtr0log.cc
+++ b/storage/innobase/mtr/mtr0log.cc
@@ -583,7 +583,7 @@ mlog_open_and_write_index(
 				with maximum length > 255 */
 				len = 0x7fff;
 			}
-			if (col->prtype & DATA_NOT_NULL) {
+			if (col->was_not_null()) {
 				len |= 0x8000;
 			}
 			if (log_ptr + 2 > log_end) {
diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc
index 2a53d3e3b88..38cbc571891 100644
--- a/storage/innobase/page/page0zip.cc
+++ b/storage/innobase/page/page0zip.cc
@@ -534,13 +534,7 @@ page_zip_fields_encode(
 
 	for (i = col = 0; i < n; i++) {
 		dict_field_t*	field = dict_index_get_nth_field(index, i);
-		ulint		val;
-
-		if (dict_field_get_col(field)->prtype & DATA_NOT_NULL) {
-			val = 1; /* set the "not nullable" flag */
-		} else {
-			val = 0; /* nullable field */
-		}
+		ulint val = !field->col->is_nullable();
 
 		if (!field->fixed_len) {
 			/* variable-length field */
diff --git a/storage/innobase/rem/rem0cmp.cc b/storage/innobase/rem/rem0cmp.cc
index f8449e5443f..1788cf45cff 100644
--- a/storage/innobase/rem/rem0cmp.cc
+++ b/storage/innobase/rem/rem0cmp.cc
@@ -1055,8 +1055,7 @@ cmp_rec_rec_simple(
 		      == rec_offs_nth_sql_null(offsets2, n));
 
 		if (rec_offs_nth_sql_null(offsets1, n)) {
-			ut_ad(!(dict_index_get_nth_col(index, n)->prtype
-				& DATA_NOT_NULL));
+			ut_ad(dict_index_get_nth_col(index, n)->is_nullable());
 			null_eq = true;
 		}
 	}
diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc
index 9b7096744a6..6d456d26afd 100644
--- a/storage/innobase/rem/rem0rec.cc
+++ b/storage/innobase/rem/rem0rec.cc
@@ -190,7 +190,7 @@ rec_get_n_extern_new(
 			= dict_field_get_col(field);
 		ulint			len;
 
-		if (!(col->prtype & DATA_NOT_NULL)) {
+		if (!col->was_not_null()) {
 			/* nullable field => read the null flag */
 
 			if (UNIV_UNLIKELY(!(byte) null_mask)) {
@@ -684,8 +684,7 @@ rec_init_offsets(
 			}
 
 			field = dict_index_get_nth_field(index, i);
-			if (!(dict_field_get_col(field)->prtype
-			      & DATA_NOT_NULL)) {
+			if (!dict_field_get_col(field)->was_not_null()) {
 				/* nullable field => read the null flag */
 
 				if (UNIV_UNLIKELY(!(byte) null_mask)) {
@@ -996,7 +995,7 @@ rec_get_offsets_reverse(
 		}
 
 		field = dict_index_get_nth_field(index, i);
-		if (!(dict_field_get_col(field)->prtype & DATA_NOT_NULL)) {
+		if (!dict_field_get_col(field)->was_not_null()) {
 			/* nullable field => read the null flag */
 
 			if (UNIV_UNLIKELY(!(byte) null_mask)) {
@@ -1651,6 +1650,8 @@ start:
 
 		const dict_field_t* ifield
 			= dict_index_get_nth_field(index, i);
+		ut_ad(!(field->type.prtype & DATA_NOT_NULL)
+		      == ifield->col->is_nullable());
 		ulint fixed_len = ifield->fixed_len;
 
 		if (temp && fixed_len
@@ -1996,7 +1997,7 @@ rec_copy_prefix_to_buf(
 		if (dict_index_is_spatial(index)) {
 			ut_ad(index->n_core_null_bytes == 0);
 			ut_ad(n_fields == DICT_INDEX_SPATIAL_NODEPTR_SIZE + 1);
-			ut_ad(index->fields[0].col->prtype & DATA_NOT_NULL);
+			ut_ad(!index->fields[0].col->is_nullable());
 			ut_ad(DATA_BIG_COL(index->fields[0].col));
 			/* This is a deficiency of the format introduced
 			in MySQL 5.7. The length in the R-tree index should
@@ -2040,7 +2041,7 @@ rec_copy_prefix_to_buf(
 		field = dict_index_get_nth_field(index, i);
 		col = dict_field_get_col(field);
 
-		if (!(col->prtype & DATA_NOT_NULL)) {
+		if (!col->was_not_null()) {
 			/* nullable field => read the null flag */
 			if (UNIV_UNLIKELY(!(byte) null_mask)) {
 				nulls--;
@@ -2733,11 +2734,11 @@ wsrep_rec_get_foreign_key(
 		}
 
 		if (len == UNIV_SQL_NULL) {
-			ut_a(!(col_f->prtype & DATA_NOT_NULL));
+			ut_a(col_f->is_nullable());
 			*buf++ = 1;
 			key_len++;
 		} else if (!new_protocol) {
-			if (!(col_r->prtype & DATA_NOT_NULL)) {
+			if (col_r->is_nullable()) {
 				*buf++ = 0;
 				key_len++;
 			}
@@ -2747,7 +2748,7 @@ wsrep_rec_get_foreign_key(
 				(uint)dtype_get_charset_coll(col_f->prtype),
 				buf, len, *buf_len);
 		} else { /* new protocol */
-			if (!(col_r->prtype & DATA_NOT_NULL)) {
+			if (col_r->is_nullable()) {
 				*buf++ = 0;
 				key_len++;
 			}
diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc
index 0506992f84d..a62da58a103 100644
--- a/storage/innobase/row/row0ins.cc
+++ b/storage/innobase/row/row0ins.cc
@@ -560,7 +560,7 @@ row_ins_cascade_calc_update_vec(
 				updated as NULL */
 
 				if (dfield_is_null(&ufield->new_val)
-				    && (col->prtype & DATA_NOT_NULL)) {
+				    && !col->is_nullable()) {
 					goto err_exit;
 				}
 
@@ -3466,7 +3466,7 @@ row_ins_index_entry_set_vals(
 		} else if (col->is_dropped()) {
 			ut_ad(index->is_primary());
 
-			if (!(col->prtype & DATA_NOT_NULL)) {
+			if (col->is_nullable()) {
 				field->data = NULL;
 				field->len = UNIV_SQL_NULL;
 				field->type.prtype = DATA_BINARY_TYPE;
diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc
index 0675217a391..46cf8700ad9 100644
--- a/storage/innobase/row/row0log.cc
+++ b/storage/innobase/row/row0log.cc
@@ -1657,9 +1657,7 @@ blob_done:
 			continue;
 		}
 
-		if ((new_col->prtype & DATA_NOT_NULL)
-		    && dfield_is_null(dfield)) {
-
+		if (!new_col->is_nullable() && dfield_is_null(dfield)) {
 			const dfield_t& default_field
 				= log->defaults->fields[col_no];
 			Field* field = log->old_table->field[col_no];
diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc
index 79619e805ca..edec9c16ef2 100644
--- a/storage/innobase/row/row0merge.cc
+++ b/storage/innobase/row/row0merge.cc
@@ -721,7 +721,7 @@ row_merge_buf_add(
 		len = dfield_get_len(field);
 
 		if (dfield_is_null(field)) {
-			ut_ad(!(col->prtype & DATA_NOT_NULL));
+			ut_ad(col->is_nullable());
 			continue;
 		} else if (!ext) {
 		} else if (dict_index_is_clust(index)) {
@@ -1888,8 +1888,8 @@ row_merge_read_clustered_index(
 				  * sizeof *nonnull));
 
 		for (ulint i = 0; i < dict_table_get_n_cols(old_table); i++) {
-			if (dict_table_get_nth_col(old_table, i)->prtype
-			    & DATA_NOT_NULL) {
+			if (!dict_table_get_nth_col(old_table, i)
+			    ->is_nullable()) {
 				continue;
 			}
 
@@ -1900,8 +1900,8 @@ row_merge_read_clustered_index(
 				continue;
 			}
 
-			if (dict_table_get_nth_col(new_table, j)->prtype
-			    & DATA_NOT_NULL) {
+			if (!dict_table_get_nth_col(new_table, j)
+			    ->is_nullable()) {
 				nonnull[n_nonnull++] = j;
 			}
 		}