5 files changed, 45 insertions, 97 deletions

diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc
index 031270a05d3..6308ea0b5dc 100644
--- a/storage/innobase/buf/buf0buf.cc
+++ b/storage/innobase/buf/buf0buf.cc
@@ -3528,9 +3528,6 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node)
   ut_ad(zip_size() == node.space->zip_size());
   ut_ad(!!zip.ssize == !!zip.data);
 
-  ut_d(auto n=) buf_pool.n_pend_reads--;
-  ut_ad(n > 0);
-
   const byte *read_frame= zip.data ? zip.data : frame;
   ut_ad(read_frame);
 
@@ -3830,9 +3827,8 @@ void buf_pool_t::print()
 		<< ", modified database pages="
 		<< UT_LIST_GET_LEN(flush_list)
 		<< ", n pending decompressions=" << n_pend_unzip
-		<< ", n pending reads=" << n_pend_reads
 		<< ", n pending flush LRU=" << n_flush()
-		<< " list=" << buf_dblwr.pending_writes()
+		<< " list=" << os_aio_pending_writes()
 		<< ", pages made young=" << stat.n_pages_made_young
 		<< ", not young=" << stat.n_pages_not_made_young
 		<< ", pages read=" << stat.n_pages_read
@@ -3945,11 +3941,11 @@ void buf_stats_get_pool_info(buf_pool_info_t *pool_info)
 
 	pool_info->n_pend_unzip = UT_LIST_GET_LEN(buf_pool.unzip_LRU);
 
-	pool_info->n_pend_reads = buf_pool.n_pend_reads;
+	pool_info->n_pend_reads = os_aio_pending_reads_approx();
 
 	pool_info->n_pending_flush_lru = buf_pool.n_flush();
 
-	pool_info->n_pending_flush_list = buf_dblwr.pending_writes();
+	pool_info->n_pending_flush_list = os_aio_pending_writes();
 	mysql_mutex_unlock(&buf_pool.flush_list_mutex);
 
 	current_time = time(NULL);
diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc
index 8d8b92ae560..c5de7a41808 100644
--- a/storage/innobase/buf/buf0dblwr.cc
+++ b/storage/innobase/buf/buf0dblwr.cc
@@ -53,7 +53,6 @@ void buf_dblwr_t::init()
     active_slot= &slots[0];
     mysql_mutex_init(buf_dblwr_mutex_key, &mutex, nullptr);
     pthread_cond_init(&cond, nullptr);
-    pthread_cond_init(&write_cond, nullptr);
   }
 }
 
@@ -467,7 +466,6 @@ void buf_dblwr_t::close()
   ut_ad(!batch_running);
 
   pthread_cond_destroy(&cond);
-  pthread_cond_destroy(&write_cond);
   for (int i= 0; i < 2; i++)
   {
     aligned_free(slots[i].write_buf);
@@ -479,38 +477,31 @@ void buf_dblwr_t::close()
 }
 
 /** Update the doublewrite buffer on write completion. */
-void buf_dblwr_t::write_completed(bool with_doublewrite)
+void buf_dblwr_t::write_completed()
 {
   ut_ad(this == &buf_dblwr);
   ut_ad(!srv_read_only_mode);
 
   mysql_mutex_lock(&mutex);
 
-  ut_ad(writes_pending);
-  if (!--writes_pending)
-    pthread_cond_broadcast(&write_cond);
+  ut_ad(is_created());
+  ut_ad(srv_use_doublewrite_buf);
+  ut_ad(batch_running);
+  slot *flush_slot= active_slot == &slots[0] ? &slots[1] : &slots[0];
+  ut_ad(flush_slot->reserved);
+  ut_ad(flush_slot->reserved <= flush_slot->first_free);
 
-  if (with_doublewrite)
+  if (!--flush_slot->reserved)
   {
-    ut_ad(is_created());
-    ut_ad(srv_use_doublewrite_buf);
-    ut_ad(batch_running);
-    slot *flush_slot= active_slot == &slots[0] ? &slots[1] : &slots[0];
-    ut_ad(flush_slot->reserved);
-    ut_ad(flush_slot->reserved <= flush_slot->first_free);
-
-    if (!--flush_slot->reserved)
-    {
-      mysql_mutex_unlock(&mutex);
-      /* This will finish the batch. Sync data files to the disk. */
-      fil_flush_file_spaces();
-      mysql_mutex_lock(&mutex);
+    mysql_mutex_unlock(&mutex);
+    /* This will finish the batch. Sync data files to the disk. */
+    fil_flush_file_spaces();
+    mysql_mutex_lock(&mutex);
 
-      /* We can now reuse the doublewrite memory buffer: */
-      flush_slot->first_free= 0;
-      batch_running= false;
-      pthread_cond_broadcast(&cond);
-    }
+    /* We can now reuse the doublewrite memory buffer: */
+    flush_slot->first_free= 0;
+    batch_running= false;
+    pthread_cond_broadcast(&cond);
   }
 
   mysql_mutex_unlock(&mutex);
@@ -754,7 +745,6 @@ void buf_dblwr_t::add_to_batch(const IORequest &request, size_t size)
   const ulint buf_size= 2 * block_size();
 
   mysql_mutex_lock(&mutex);
-  writes_pending++;
 
   for (;;)
   {
diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc
index 3fa26dab25f..b702d79df8b 100644
--- a/storage/innobase/buf/buf0flu.cc
+++ b/storage/innobase/buf/buf0flu.cc
@@ -209,7 +209,7 @@ void buf_flush_remove_pages(uint32_t id)
     if (!deferred)
       break;
 
-    buf_dblwr.wait_for_page_writes();
+    os_aio_wait_until_no_pending_writes();
   }
 }
 
@@ -339,9 +339,9 @@ void buf_page_write_complete(const IORequest &request)
   if (request.is_LRU())
   {
     const bool temp= bpage->oldest_modification() == 2;
-    if (!temp)
-      buf_dblwr.write_completed(state < buf_page_t::WRITE_FIX_REINIT &&
-                                request.node->space->use_doublewrite());
+    if (!temp && state < buf_page_t::WRITE_FIX_REINIT &&
+        request.node->space->use_doublewrite())
+      buf_dblwr.write_completed();
     /* We must hold buf_pool.mutex while releasing the block, so that
     no other thread can access it before we have freed it. */
     mysql_mutex_lock(&buf_pool.mutex);
@@ -353,8 +353,9 @@ void buf_page_write_complete(const IORequest &request)
   }
   else
   {
-    buf_dblwr.write_completed(state < buf_page_t::WRITE_FIX_REINIT &&
-                              request.node->space->use_doublewrite());
+    if (state < buf_page_t::WRITE_FIX_REINIT &&
+        request.node->space->use_doublewrite())
+      buf_dblwr.write_completed();
     bpage->write_complete(false);
   }
 }
@@ -847,8 +848,6 @@ bool buf_page_t::flush(bool evict, fil_space_t *space)
       ut_ad(lsn >= oldest_modification());
       log_write_up_to(lsn, true);
     }
-    if (UNIV_LIKELY(space->purpose != FIL_TYPE_TEMPORARY))
-      buf_dblwr.add_unbuffered();
     space->io(IORequest{type, this, slot}, physical_offset(), size,
               write_frame, this);
   }
@@ -1083,13 +1082,11 @@ static ulint buf_flush_try_neighbors(fil_space_t *space,
     mysql_mutex_unlock(&buf_pool.mutex);
   }
 
-  ut_ad(!bpage);
-
-  if (auto n= count - 1)
+  if (count > 1)
   {
     MONITOR_INC_VALUE_CUMULATIVE(MONITOR_FLUSH_NEIGHBOR_TOTAL_PAGE,
                                  MONITOR_FLUSH_NEIGHBOR_COUNT,
-                                 MONITOR_FLUSH_NEIGHBOR_PAGES, n);
+                                 MONITOR_FLUSH_NEIGHBOR_PAGES, count - 1);
   }
 
   return count;
@@ -1964,7 +1961,7 @@ static void buf_flush_wait(lsn_t lsn)
         break;
     }
     mysql_mutex_unlock(&buf_pool.flush_list_mutex);
-    buf_dblwr.wait_for_page_writes();
+    os_aio_wait_until_no_pending_writes();
     mysql_mutex_lock(&buf_pool.flush_list_mutex);
   }
 }
@@ -1985,8 +1982,6 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn)
   if (buf_pool.get_oldest_modification(sync_lsn) < sync_lsn)
   {
     MONITOR_INC(MONITOR_FLUSH_SYNC_WAITS);
-    thd_wait_begin(nullptr, THD_WAIT_DISKIO);
-    tpool::tpool_wait_begin();
 
 #if 1 /* FIXME: remove this, and guarantee that the page cleaner serves us */
     if (UNIV_UNLIKELY(!buf_page_cleaner_is_active))
@@ -2001,7 +1996,7 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn)
                                        MONITOR_FLUSH_SYNC_COUNT,
                                        MONITOR_FLUSH_SYNC_PAGES, n_pages);
         }
-        buf_dblwr.wait_for_page_writes();
+        os_aio_wait_until_no_pending_writes();
         mysql_mutex_lock(&buf_pool.flush_list_mutex);
       }
       while (buf_pool.get_oldest_modification(sync_lsn) < sync_lsn);
@@ -2010,7 +2005,6 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn)
 #endif
       buf_flush_wait(sync_lsn);
 
-    tpool::tpool_wait_end();
     thd_wait_end(nullptr);
   }
 
@@ -2375,7 +2369,7 @@ static void buf_flush_page_cleaner()
 
         if (!recv_recovery_is_on() &&
             !srv_startup_is_before_trx_rollback_phase &&
-            srv_operation == SRV_OPERATION_NORMAL)
+            srv_operation <= SRV_OPERATION_EXPORT_RESTORED)
           log_checkpoint();
       }
       while (false);
@@ -2445,7 +2439,7 @@ static void buf_flush_page_cleaner()
         last_activity_count= activity_count;
         goto maybe_unemployed;
       }
-      else if (buf_pool.page_cleaner_idle() && buf_pool.n_pend_reads == 0)
+      else if (buf_pool.page_cleaner_idle() && !os_aio_pending_reads())
       {
         /* reaching here means 3 things:
            - last_activity_count == activity_count: suggesting server is idle
@@ -2521,7 +2515,7 @@ static void buf_flush_page_cleaner()
     mysql_mutex_lock(&buf_pool.flush_list_mutex);
     buf_flush_wait_LRU_batch_end();
     mysql_mutex_unlock(&buf_pool.flush_list_mutex);
-    buf_dblwr.wait_for_page_writes();
+    os_aio_wait_until_no_pending_writes();
   }
 
   mysql_mutex_lock(&buf_pool.flush_list_mutex);
@@ -2547,7 +2541,7 @@ static void buf_flush_page_cleaner()
 ATTRIBUTE_COLD void buf_flush_page_cleaner_init()
 {
   ut_ad(!buf_page_cleaner_is_active);
-  ut_ad(srv_operation == SRV_OPERATION_NORMAL ||
+  ut_ad(srv_operation <= SRV_OPERATION_EXPORT_RESTORED ||
         srv_operation == SRV_OPERATION_RESTORE ||
         srv_operation == SRV_OPERATION_RESTORE_EXPORT);
   buf_flush_async_lsn= 0;
@@ -2571,15 +2565,7 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool()
   {
     mysql_mutex_unlock(&buf_pool.flush_list_mutex);
     buf_flush_list(srv_max_io_capacity);
-    if (const size_t pending= buf_dblwr.pending_writes())
-    {
-      timespec abstime;
-      service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL,
-                                     "Waiting to write %zu pages", pending);
-      set_timespec(abstime, INNODB_EXTEND_TIMEOUT_INTERVAL / 2);
-      buf_dblwr.wait_for_page_writes(abstime);
-    }
-
+    os_aio_wait_until_no_pending_writes();
     mysql_mutex_lock(&buf_pool.flush_list_mutex);
     service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL,
                                    "Waiting to flush " ULINTPF " pages",
@@ -2587,20 +2573,18 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool()
   }
 
   mysql_mutex_unlock(&buf_pool.flush_list_mutex);
-  ut_ad(!buf_pool.any_io_pending());
+  ut_ad(!os_aio_pending_writes());
+  ut_ad(!os_aio_pending_reads());
 }
 
 /** Synchronously flush dirty blocks during recv_sys_t::apply().
 NOTE: The calling thread is not allowed to hold any buffer page latches! */
 void buf_flush_sync_batch(lsn_t lsn)
 {
-  thd_wait_begin(nullptr, THD_WAIT_DISKIO);
-  tpool::tpool_wait_begin();
+  lsn= std::max(lsn, log_sys.get_lsn());
   mysql_mutex_lock(&buf_pool.flush_list_mutex);
   buf_flush_wait(lsn);
   mysql_mutex_unlock(&buf_pool.flush_list_mutex);
-  tpool::tpool_wait_end();
-  thd_wait_end(nullptr);
 }
 
 /** Synchronously flush dirty blocks.
diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc
index 2e9b42e9f51..d1c7f0ba666 100644
--- a/storage/innobase/buf/buf0lru.cc
+++ b/storage/innobase/buf/buf0lru.cc
@@ -1112,17 +1112,8 @@ static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, const page_id_t id,
 				break;
 			case FIL_PAGE_TYPE_ZBLOB:
 			case FIL_PAGE_TYPE_ZBLOB2:
-				break;
 			case FIL_PAGE_INDEX:
 			case FIL_PAGE_RTREE:
-#if defined UNIV_ZIP_DEBUG && defined BTR_CUR_HASH_ADAPT
-				/* During recovery, we only update the
-				compressed page, not the uncompressed one. */
-				ut_a(recv_recovery_is_on()
-				     || page_zip_validate(
-					     &bpage->zip, page,
-					     ((buf_block_t*) bpage)->index));
-#endif /* UNIV_ZIP_DEBUG && BTR_CUR_HASH_ADAPT */
 				break;
 			default:
 				ib::error() << "The compressed page to be"
diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc
index fa91939acee..8631a03644f 100644
--- a/storage/innobase/buf/buf0rea.cc
+++ b/storage/innobase/buf/buf0rea.cc
@@ -227,12 +227,9 @@ static buf_page_t* buf_page_init_for_read(ulint mode, const page_id_t page_id,
   }
 
   buf_pool.stat.n_pages_read++;
-  mysql_mutex_unlock(&buf_pool.mutex);
-  buf_pool.n_pend_reads++;
-  goto func_exit_no_mutex;
 func_exit:
   mysql_mutex_unlock(&buf_pool.mutex);
-func_exit_no_mutex:
+
   if (mode == BUF_READ_IBUF_PAGES_ONLY)
     ibuf_mtr_commit(&mtr);
 
@@ -319,8 +316,6 @@ buf_read_page_low(
 			     page_id.page_no() * len, len, dst, bpage);
 
 	if (UNIV_UNLIKELY(fio.err != DB_SUCCESS)) {
-		ut_d(auto n=) buf_pool.n_pend_reads--;
-		ut_ad(n > 0);
 		buf_pool.corrupted_evict(bpage, buf_page_t::READ_FIX);
 	} else if (sync) {
 		thd_wait_end(NULL);
@@ -367,7 +362,8 @@ buf_read_ahead_random(const page_id_t page_id, ulint zip_size, bool ibuf)
     read-ahead, as that could break the ibuf page access order */
     return 0;
 
-  if (buf_pool.n_pend_reads > buf_pool.curr_size / BUF_READ_AHEAD_PEND_LIMIT)
+  if (os_aio_pending_reads_approx() >
+      buf_pool.curr_size / BUF_READ_AHEAD_PEND_LIMIT)
     return 0;
 
   fil_space_t* space= fil_space_t::get(page_id.space());
@@ -519,7 +515,8 @@ buf_read_ahead_linear(const page_id_t page_id, ulint zip_size, bool ibuf)
     /* No read-ahead to avoid thread deadlocks */
     return 0;
 
-  if (buf_pool.n_pend_reads > buf_pool.curr_size / BUF_READ_AHEAD_PEND_LIMIT)
+  if (os_aio_pending_reads_approx() >
+      buf_pool.curr_size / BUF_READ_AHEAD_PEND_LIMIT)
     return 0;
 
   const uint32_t buf_read_ahead_area= buf_pool.read_ahead_area;
@@ -687,18 +684,8 @@ void buf_read_recv_pages(uint32_t space_id, st_::span<uint32_t> page_nos)
 			limit += buf_pool.chunks[j].size / 2;
 		}
 
-		for (ulint count = 0; buf_pool.n_pend_reads >= limit; ) {
-			std::this_thread::sleep_for(
-				std::chrono::milliseconds(10));
-
-			if (!(++count % 1000)) {
-
-				ib::error()
-					<< "Waited for " << count / 100
-					<< " seconds for "
-					<< buf_pool.n_pend_reads
-					<< " pending reads";
-			}
+		if (os_aio_pending_reads() >= limit) {
+			os_aio_wait_until_no_pending_reads();
 		}
 
 		space->reacquire();