MDEV-11684: post-10.1-merge fixes

10.1 is merged into 10.2 now. Two issues are left to fix:
(1) encryption.innochecksum test
(2) read_page0 vs page_0_crypt_read

(1) innochecksum tool did not compile after merge because
buf_page_is_corrupted uses fil_crypt_t that has been changed.

extra/CMakeLists.txt: Added fil/fil0crypt.cc as dependency
as we need to use fil_crypt_verify_checksum for encrypted pages.

innochecksum.cc: If we think page is encrypted i.e.
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION != 0 we call
fil_crypt_verify_checksum() function to compare calculated
checksum to stored checksum calculated after encryption
(this is stored on different offset i.e.
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4).
If checksum does not match we call normal buf_page_is_corrupted
to compare calculated checksum to stored checksum.

fil0crypt.cc: add #ifdef UNIV_INNOCHECKSUM to be able to compile
this file for innochecksum tool.

(2) read_page0 is not needed and thus removed.

7 files changed, 146 insertions, 90 deletions

diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt
index ee696c156f9..480cd6e7fbb 100644
--- a/extra/CMakeLists.txt
+++ b/extra/CMakeLists.txt
@@ -80,6 +80,7 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE)
       ../storage/innobase/buf/buf0buf.cc
       ../storage/innobase/page/page0zip.cc
       ../storage/innobase/os/os0file.cc
+      ../storage/innobase/fil/fil0crypt.cc
       )
 
 
diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc
index ce631c13952..9d02a545cc5 100644
--- a/extra/innochecksum.cc
+++ b/extra/innochecksum.cc
@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
-   Copyright (c) 2014, 2016, MariaDB Corporation.
+   Copyright (c) 2014, 2017, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -70,6 +70,24 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
 #define PRIuMAX   "llu"
 #endif
 
+/*********************************************************************
+Verify checksum for a page (iff it's encrypted)
+NOTE: currently this function can only be run in single threaded mode
+as it modifies srv_checksum_algorithm (temporarily)
+@param[in]	src_fame	page to verify
+@param[in]	page_size	page_size
+@param[in]	page_no		page number of given read_buf
+@param[in]	strict_check	true if strict-check option is enabled
+@return true if page is encrypted AND OK, false otherwise */
+UNIV_INTERN
+bool
+fil_space_verify_crypt_checksum(
+/*============================*/
+	const byte* 		src_frame,	/*!< in: page the verify */
+	const page_size_t&	page_size	/*!< in: page size */
+	,uintmax_t 		page_no,
+	bool			strict_check);
+
 /* Global variables */
 static bool			verbose;
 static bool			just_count;
@@ -564,9 +582,25 @@ is_page_corrupted(
 		}
 	}
 
-	is_corrupted = buf_page_is_corrupted(
-		true, buf, page_size, false, cur_page_num, strict_verify,
-		is_log_enabled, log_file);
+	/* If page is encrypted, use different checksum calculation
+	as innochecksum can't decrypt pages. Note that some old InnoDB
+	versions did not initialize FIL_PAGE_FILE_FLUSH_LSN field
+	so if crypt checksum does not match we verify checksum using
+	normal method.
+	*/
+	if (mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) {
+		is_corrupted = fil_space_verify_crypt_checksum(buf, page_size,
+			cur_page_num, strict_verify);
+	} else {
+		is_corrupted = true;
+	}
+
+	if (is_corrupted) {
+		is_corrupted = buf_page_is_corrupted(
+			true, buf, page_size, false,
+			cur_page_num, strict_verify,
+			is_log_enabled, log_file);
+	}
 
 	return(is_corrupted);
 }
diff --git a/mysql-test/suite/encryption/disabled.def b/mysql-test/suite/encryption/disabled.def
index f8d7fb51865..c4394b26947 100644
--- a/mysql-test/suite/encryption/disabled.def
+++ b/mysql-test/suite/encryption/disabled.def
@@ -13,4 +13,4 @@
 innodb_scrub : MDEV-8139
 innodb_scrub_compressed : MDEV-8139
 innodb_scrub_background : MDEV-8139
-innochecksum: see buf_page_is_corrupted()
+
diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc
index 638126f8de1..59b3ee4ca7e 100644
--- a/storage/innobase/buf/buf0buf.cc
+++ b/storage/innobase/buf/buf0buf.cc
@@ -846,7 +846,7 @@ buf_page_is_corrupted(
 	ulint		checksum_field2;
 	bool page_encrypted = false;
 
-#ifndef UNIV_INNOCHECKSUM // FIXME see also encryption.innochecksum test
+#ifndef UNIV_INNOCHECKSUM
 	ulint 		space_id = mach_read_from_4(
 		read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
 	fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
@@ -859,6 +859,12 @@ buf_page_is_corrupted(
 	    fil_page_is_encrypted(read_buf)) {
 		page_encrypted = true;
 	}
+#else
+	if (mach_read_from_4(read_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0
+	    || mach_read_from_2(read_buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+		page_encrypted = true;
+	}
+
 #endif
 
 	DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); );
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 708dbd0fd01..ff30b3a09e9 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 Copyright (C) 2013, 2015, Google Inc. All Rights Reserved.
-Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved.
+Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -24,14 +24,16 @@ Modified           Jan Lindström jan.lindstrom@mariadb.com
 *******************************************************/
 
 #include "fil0fil.h"
+#include "mach0data.h"
+#include "page0size.h"
+#include "page0zip.h"
+#ifndef UNIV_INNOCHECKSUM
 #include "fil0crypt.h"
 #include "srv0srv.h"
 #include "srv0start.h"
-#include "mach0data.h"
 #include "log0recv.h"
 #include "mtr0mtr.h"
 #include "mtr0log.h"
-#include "page0zip.h"
 #include "ut0ut.h"
 #include "btr0scrub.h"
 #include "fsp0fsp.h"
@@ -910,81 +912,6 @@ fil_crypt_calculate_checksum(
 	return checksum;
 }
 
-/*********************************************************************
-Verify checksum for a page (iff it's encrypted)
-NOTE: currently this function can only be run in single threaded mode
-as it modifies srv_checksum_algorithm (temporarily)
-@return true if page is encrypted AND OK, false otherwise */
-UNIV_INTERN
-bool
-fil_space_verify_crypt_checksum(
-/*============================*/
-	const byte* 		src_frame,	/*!< in: page the verify */
-	const page_size_t&	page_size)	/*!< in: page size */
-{
-	// key version
-	uint key_version = mach_read_from_4(
-		src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
-
-	if (key_version == 0) {
-		return false; // unencrypted page
-	}
-
-	/* "trick" the normal checksum routines by storing the post-encryption
-	* checksum into the normal checksum field allowing for reuse of
-	* the normal routines */
-
-	// post encryption checksum
-	ib_uint32_t stored_post_encryption = mach_read_from_4(
-		src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
-
-	// save pre encryption checksum for restore in end of this function
-	ib_uint32_t stored_pre_encryption = mach_read_from_4(
-		src_frame + FIL_PAGE_SPACE_OR_CHKSUM);
-
-	ib_uint32_t checksum_field2 = mach_read_from_4(
-		src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
-
-	/** prepare frame for usage of normal checksum routines */
-	mach_write_to_4(const_cast<byte*>(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM,
-			stored_post_encryption);
-
-	/* NOTE: this function is (currently) only run when restoring
-	* dblwr-buffer, server is single threaded so it's safe to modify
-	* srv_checksum_algorithm */
-	srv_checksum_algorithm_t save_checksum_algorithm =
-		(srv_checksum_algorithm_t)srv_checksum_algorithm;
-
-	if (!page_size.is_compressed() &&
-	    (save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB ||
-	     save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) {
-		/* handle ALGORITHM_INNODB specially,
-		* "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC
-		* checksum_field2 is sort of pointless anyway...
-		*/
-		srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
-		mach_write_to_4(const_cast<byte*>(src_frame) +
-				UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
-				BUF_NO_CHECKSUM_MAGIC);
-	}
-
-	/* verify checksums */
-	ibool corrupted = buf_page_is_corrupted(false, src_frame, page_size, false);
-
-	/** restore frame & algorithm */
-	srv_checksum_algorithm = save_checksum_algorithm;
-
-	mach_write_to_4(const_cast<byte*>(src_frame) +
-			FIL_PAGE_SPACE_OR_CHKSUM,
-			stored_pre_encryption);
-
-	mach_write_to_4(const_cast<byte*>(src_frame) +
-			UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
-			checksum_field2);
-
-	return (!corrupted);
-}
-
 /***********************************************************************/
 
 /** A copy of global key state */
@@ -2671,3 +2598,93 @@ fil_space_get_scrub_status(
 
 	return crypt_data == NULL ? 1 : 0;
 }
+#endif /* UNIV_INNOCHECKSUM */
+
+/*********************************************************************
+Verify checksum for a page (iff it's encrypted)
+NOTE: currently this function can only be run in single threaded mode
+as it modifies srv_checksum_algorithm (temporarily)
+@param[in]	src_fame	page to verify
+@param[in]	page_size	page_size
+@param[in]	page_no		page number of given read_buf
+@param[in]	strict_check	true if strict-check option is enabled
+@return true if page is encrypted AND OK, false otherwise */
+UNIV_INTERN
+bool
+fil_space_verify_crypt_checksum(
+/*============================*/
+	const byte* 		src_frame,	/*!< in: page the verify */
+	const page_size_t&	page_size	/*!< in: page size */
+#ifdef UNIV_INNOCHECKSUM
+	,uintmax_t 		page_no,
+	bool			strict_check
+#endif /* UNIV_INNOCHECKSUM */
+)
+{
+	// key version
+	uint key_version = mach_read_from_4(
+		src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+
+	if (key_version == 0) {
+		return false; // unencrypted page
+	}
+
+	/* "trick" the normal checksum routines by storing the post-encryption
+	* checksum into the normal checksum field allowing for reuse of
+	* the normal routines */
+
+	// post encryption checksum
+	ib_uint32_t stored_post_encryption = mach_read_from_4(
+		src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
+
+	// save pre encryption checksum for restore in end of this function
+	ib_uint32_t stored_pre_encryption = mach_read_from_4(
+		src_frame + FIL_PAGE_SPACE_OR_CHKSUM);
+
+	ib_uint32_t checksum_field2 = mach_read_from_4(
+		src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
+
+	/** prepare frame for usage of normal checksum routines */
+	mach_write_to_4(const_cast<byte*>(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM,
+			stored_post_encryption);
+
+	/* NOTE: this function is (currently) only run when restoring
+	* dblwr-buffer, server is single threaded so it's safe to modify
+	* srv_checksum_algorithm */
+	srv_checksum_algorithm_t save_checksum_algorithm =
+		(srv_checksum_algorithm_t)srv_checksum_algorithm;
+
+	if (!page_size.is_compressed() &&
+	    (save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB ||
+	     save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) {
+		/* handle ALGORITHM_INNODB specially,
+		* "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC
+		* checksum_field2 is sort of pointless anyway...
+		*/
+		srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
+		mach_write_to_4(const_cast<byte*>(src_frame) +
+				UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
+				BUF_NO_CHECKSUM_MAGIC);
+	}
+
+	/* verify checksums */
+	bool corrupted = buf_page_is_corrupted(false, src_frame,
+		page_size, false
+#ifdef UNIV_INNOCHECKSUM
+		,page_no, strict_check, false, NULL
+#endif /* UNIV_INNOCHECKSUM */
+	);
+
+	/** restore frame & algorithm */
+	srv_checksum_algorithm = save_checksum_algorithm;
+
+	mach_write_to_4(const_cast<byte*>(src_frame) +
+			FIL_PAGE_SPACE_OR_CHKSUM,
+			stored_pre_encryption);
+
+	mach_write_to_4(const_cast<byte*>(src_frame) +
+			UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
+			checksum_field2);
+
+	return (!corrupted);
+}
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index 60ed4e01422..43b31002306 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2016, Oracle and/or its affiliates.
-Copyright (c) 2013, 2016, MariaDB Corporation.
+Copyright (c) 2013, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1718,13 +1718,14 @@ fil_space_create(
 		fil_system->max_assigned_id = id;
 	}
 
+#ifdef UNIV_DEBUG
 	if (crypt_data) {
-		space->read_page0 = true;
 		/* If table could be encrypted print info */
 		ib::info() << "Tablespace ID " << id << " name " << space->name
 			   << ":" << fil_crypt_get_mode(crypt_data)
 			   << " " << fil_crypt_get_type(crypt_data);
 	}
+#endif
 
 	mutex_exit(&fil_system->mutex);
 
diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h
index 1e07c4f8fb5..30d79a52b4d 100644
--- a/storage/innobase/include/fil0fil.h
+++ b/storage/innobase/include/fil0fil.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2013, 2016, MariaDB Corporation.
+Copyright (c) 2013, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -190,9 +190,6 @@ struct fil_space_t {
 	/** True if we have already printed compression failure */
 	bool		printed_compression_failure;
 
-	/** True if page 0 of tablespace is read */
-	bool		read_page0;
-
 	/** Release the reserved free extents.
 	@param[in]	n_reserved	number of reserved extents */
 	void release_free_extents(ulint n_reserved);