MDEV-11759: Encryption code in MariaDB 10.1/10.2 causes

compatibility problems

Pages that are encrypted contain post encryption checksum on
different location that normal checksum fields. Therefore,
we should before decryption check this checksum to avoid
unencrypting corrupted pages. After decryption we can use
traditional checksum check to detect if page is corrupted
or unencryption was done using incorrect key.

Pages that are page compressed do not contain any checksum,
here we need to fist unencrypt, decompress and finally
use tradional checksum check to detect page corruption
or that we used incorrect key in unencryption.

buf0buf.cc: buf_page_is_corrupted() mofified so that
compressed pages are skipped.

buf0buf.h, buf_block_init(), buf_page_init_low():
removed unnecessary page_encrypted, page_compressed,
stored_checksum, valculated_checksum fields from
buf_page_t

buf_page_get_gen(): use new buf_page_check_corrupt() function
to detect corrupted pages.

buf_page_check_corrupt(): If page was not yet decrypted
check if post encryption checksum still matches.
If page is not anymore encrypted, use buf_page_is_corrupted()
traditional checksum method.

If page is detected as corrupted and it is not encrypted
we print corruption message to error log.
If page is still encrypted or it was encrypted and now
corrupted, we will print message that page is
encrypted to error log.

buf_page_io_complete(): use new buf_page_check_corrupt()
function to detect corrupted pages.

buf_page_decrypt_after_read(): Verify post encryption
checksum before tring to decrypt.

fil0crypt.cc: fil_encrypt_buf() verify post encryption
checksum and ind fil_space_decrypt() return true
if we really decrypted the page.

fil_space_verify_crypt_checksum(): rewrite to use
the method used when calculating post encryption
checksum. We also check if post encryption checksum
matches that traditional checksum check does not
match.

fil0fil.ic: Add missed page type encrypted and page
compressed to fil_get_page_type_name()

Note that this change does not yet fix innochecksum tool,
that will be done in separate MDEV.

Fix test failures caused by buf page corruption injection.

1 files changed, 105 insertions, 64 deletions

diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 2999bea2765..fed1cf94d84 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -625,6 +625,8 @@ fil_encrypt_buf(
 	// store the post-encryption checksum after the key-version
 	mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum);
 
+	ut_ad(fil_space_verify_crypt_checksum(dst_frame, zip_size, NULL, offset));
+
 	srv_stats.pages_encrypted.inc();
 
 	return dst_frame;
@@ -676,6 +678,7 @@ fil_space_encrypt(
 		byte* comp_mem = NULL;
 		byte* uncomp_mem = NULL;
 		ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
+		fil_space_t* tspace = fil_space_found_by_id(space);
 
 		if (page_compressed_encrypted) {
 			comp_mem = (byte *)malloc(UNIV_PAGE_SIZE);
@@ -685,7 +688,7 @@ fil_space_encrypt(
 			src = uncomp_mem;
 		}
 
-		bool corrupted1 = buf_page_is_corrupted(true, src, zip_size);
+		bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, tspace);
 		bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err);
 
 		/* Need to decompress the page if it was also compressed */
@@ -694,7 +697,7 @@ fil_space_encrypt(
 			fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL);
 		}
 
-		bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size);
+		bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, tspace);
 		bool different = memcmp(src, tmp_mem, size);
 
 		if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) {
@@ -858,19 +861,25 @@ fil_space_decrypt(
 
 /******************************************************************
 Decrypt a page
-@return encrypted page, or original not encrypted page if encryption is
-not needed. */
+@param[in]	space			Tablespace id
+@param[in]	tmp_frame		Temporary buffer used for decrypting
+@param[in]	page_size		Page size
+@param[in,out]	src_frame		Page to decrypt
+@param[out]	decrypted		true if page was decrypted
+@return decrypted page, or original not encrypted page if decryption is
+not needed.*/
 UNIV_INTERN
 byte*
 fil_space_decrypt(
-/*==============*/
-	ulint		space,		/*!< in: Fil space id */
-	byte*		tmp_frame,	/*!< in: temporary buffer */
-	ulint		page_size,	/*!< in: page size */
-	byte*		src_frame)	/*!< in/out: page buffer */
+	ulint		space,
+	byte*		tmp_frame,
+	ulint		page_size,
+	byte*		src_frame,
+	bool*		decrypted)
 {
 	dberr_t err = DB_SUCCESS;
 	byte* res = NULL;
+	*decrypted = false;
 
 	bool encrypted = fil_space_decrypt(
 				fil_space_get_crypt_data(space),
@@ -881,6 +890,7 @@ fil_space_decrypt(
 
 	if (err == DB_SUCCESS) {
 		if (encrypted) {
+			*decrypted = true;
 			/* Copy the decrypted page back to page buffer, not
 			really any other options. */
 			memcpy(src_frame, tmp_frame, page_size);
@@ -934,83 +944,114 @@ fil_crypt_calculate_checksum(
 }
 
 /*********************************************************************
-Verify checksum for a page (iff it's encrypted)
-NOTE: currently this function can only be run in single threaded mode
-as it modifies srv_checksum_algorithm (temporarily)
+Verify that post encryption checksum match calculated checksum.
+This function should be called only if tablespace contains crypt_data
+metadata (this is strong indication that tablespace is encrypted).
+Function also verifies that traditional checksum does not match
+calculated checksum as if it does page could be valid unencrypted,
+encrypted, or corrupted.
+@param[in]	page		Page to verify
+@param[in]	zip_size	zip size
+@param[in]	space		Tablespace
+@param[in]	pageno		Page no
 @return true if page is encrypted AND OK, false otherwise */
 UNIV_INTERN
 bool
 fil_space_verify_crypt_checksum(
-/*============================*/
-	const byte* 	src_frame,	/*!< in: page the verify */
-	ulint		zip_size)  	/*!< in: compressed size if
-					row_format compressed */
+	byte* 			page,
+	ulint			zip_size,
+	const fil_space_t*	space,
+	ulint			pageno)
 {
-	// key version
-	uint key_version = mach_read_from_4(
-		src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+	uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
 
+	/* If page is not encrypted, return false */
 	if (key_version == 0) {
-		return false; // unencrypted page
+		return false;
 	}
 
-	/* "trick" the normal checksum routines by storing the post-encryption
-	* checksum into the normal checksum field allowing for reuse of
-	* the normal routines */
+	/* If no checksum is used, can't continue checking. */
+	if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
+		return(true);
+	}
 
-	// post encryption checksum
-	ib_uint32_t stored_post_encryption = mach_read_from_4(
-		src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
+	/* Read stored post encryption checksum. */
+	ib_uint32_t checksum = mach_read_from_4(
+		page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
 
-	// save pre encryption checksum for restore in end of this function
-	ib_uint32_t stored_pre_encryption = mach_read_from_4(
-		src_frame + FIL_PAGE_SPACE_OR_CHKSUM);
+	/* Declare empty pages non-corrupted */
+	if (checksum == 0
+	    && *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0) {
+		return (true);
+	}
 
-	ib_uint32_t checksum_field2 = mach_read_from_4(
-		src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
+	/* Compressed and encrypted pages do not have checksum. Assume not
+	corrupted. */
+	if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+		return (true);
+	}
 
-	/** prepare frame for usage of normal checksum routines */
-	mach_write_to_4(const_cast<byte*>(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM,
-			stored_post_encryption);
+	/* Compressed pages use different checksum method. We first store
+	the post encryption checksum on checksum location and after function
+	restore the original. */
+	if (zip_size) {
+		ib_uint32_t old = static_cast<ib_uint32_t>(mach_read_from_4(
+				page + FIL_PAGE_SPACE_OR_CHKSUM));
 
-	/* NOTE: this function is (currently) only run when restoring
-	* dblwr-buffer, server is single threaded so it's safe to modify
-	* srv_checksum_algorithm */
-	srv_checksum_algorithm_t save_checksum_algorithm =
-		(srv_checksum_algorithm_t)srv_checksum_algorithm;
+		mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
 
-	if (zip_size == 0 &&
-	    (save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB ||
-	     save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) {
-		/* handle ALGORITHM_INNODB specially,
-		* "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC
-		* checksum_field2 is sort of pointless anyway...
-		*/
-		srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
-		mach_write_to_4(const_cast<byte*>(src_frame) +
-				UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
-				BUF_NO_CHECKSUM_MAGIC);
+		bool valid = page_zip_verify_checksum(page, zip_size);
+
+		mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old);
+
+		return (valid);
 	}
 
-	/* verify checksums */
-	ibool corrupted = buf_page_is_corrupted(false, src_frame, zip_size);
+	/* If stored checksum matches one of the calculated checksums
+	page is not corrupted. */
 
-	/** restore frame & algorithm */
-	srv_checksum_algorithm = save_checksum_algorithm;
+	ib_uint32_t cchecksum1 = buf_calc_page_crc32(page);
+	ib_uint32_t cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum(
+				page);
+	bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
+		|| checksum == BUF_NO_CHECKSUM_MAGIC);
 
-	mach_write_to_4(const_cast<byte*>(src_frame) +
-			FIL_PAGE_SPACE_OR_CHKSUM,
-			stored_pre_encryption);
+	/* Old InnoDB versions did not initialize
+	FIL_PAGE_FILE_FLUSH_LSN field so there could be garbage
+	and above checksum check could produce false positive.
+	Thus we also check does the traditional stored
+	checksum fields match the calculated one. Both of these
+	could naturally produce false positive but then
+	we just decrypt the page and after that corrupted
+	pages very probable stay corrupted and valid
+	pages very probable stay valid.
+	*/
+	ulint checksum1 = mach_read_from_4(
+		page + FIL_PAGE_SPACE_OR_CHKSUM);
 
-	mach_write_to_4(const_cast<byte*>(src_frame) +
-			UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
-			checksum_field2);
+	ulint checksum2 = mach_read_from_4(
+		page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
 
-	if (!corrupted) {
-		return true;  // page was encrypted and checksum matched
-	} else {
-		return false; // page was encrypted but checksum didn't match
+
+	bool valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
+		|| buf_page_is_checksum_valid_none(page,checksum1,checksum2)
+		|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
+
+	if (encrypted && valid) {
+		/* If page is encrypted and traditional checksums match,
+		page could be still encrypted, or not encrypted and valid or
+		corrupted. */
+		ib_logf(IB_LOG_LEVEL_ERROR,
+			" Page %lu in space %s (%lu) maybe corrupted."
+			" Post encryption checksum %u stored [%lu:%lu] key_version %u",
+			pageno,
+			space ? space->name : "N/A",
+			mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
+			checksum, checksum1, checksum2, key_version);
+		encrypted = false;
 	}
+
+	return(encrypted);
 }
 
 /***********************************************************************/