QPID-6303 [linearstore] Roll back auto-upgrade of store directory structure

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1652486 13f79535-47bb-0310-9956-ffa450edef68

6 files changed, 46 insertions, 63 deletions

diff --git a/qpid/cpp/src/qpid/linearstore/ISSUES b/qpid/cpp/src/qpid/linearstore/ISSUES
index 2042136bac..4023ba9629 100644
--- a/qpid/cpp/src/qpid/linearstore/ISSUES
+++ b/qpid/cpp/src/qpid/linearstore/ISSUES
@@ -37,16 +37,21 @@ Current/pending:
                    ** Basic performance tests
    5464 -        [linearstore] Incompletely created journal files accumulate in EFP
    -    1088944  [Linearstore] store does not return all files to EFP after purging big queue <queue purge issue>
-   6043 1066256  [LinearStore] changing efp size after using store broke the new durable nodes creation
+*  -    1066256  [LinearStore] changing efp size after using store broke the new durable nodes creation
    -    1067480  [LinearStore] Provide a way to limit max count/size of empty files in EFP
    -    1067429  [LinearStore] last file from deleted queue is not moved to EFP  <queue delete issue>
-   -    1067482  [LinearStore] Provide a way to prealocate empty pages in EFP
-*  5671 1160367  [linearstore] Add ability to use disk partitions and select per-queue EFPs
-                   svn r.1636598 2014-11-04: WIP: New EFP and journal dir structure using symlinks
-                   svn r.1637985 2014-11-10: WIP: Auto-upgrade from old dir structure to new
-                   svn r.1649081 2015-01-02: WIP: Specify new queue using qpid-config --durable together with --efp-partition-num and/or --efp-pool-file-size. Needs testing.
-*       1148807  [linearstore] Restarting broker with empty journal raises confusing warning
-*       1066256  [linearstore] Changing EFP size after using store breaks durable queue creation
+   -    1067482  [LinearStore] Provide a way to preallocate empty pages in EFP
+*  6303 1180660  [linearstore] Roll back auto-upgrade of store directory structure 
+*  5362 1145363  Linearstore: No store tools exist for examining the journals
+                   svn r.1556888 2014-01-09: WIP checkin for linearstore version of qpid_qls_analyze. Needs testing and tidy-up.
+                   svn r.1560530 2014-01-22: Bugfixes for qpid_qls_analyze
+                   svn r.1561848 2014-01-27: Bugfixes and enhancements for qpid_qls_analyze
+                   svn r.1564808 2014-02-05: Bugfixes and enhancements for qpid_qls_analyze
+                   svn r.1578899 2014-03-18: Bugfixes and enhancements for qpid_qls_analyze
+                   svn r.1583778 2014-04-01: Bugfix for qpid_qls_analyze
+                   * Store analysis and status
+                   * Recovery/reading of message content
+                   * Empty file pool status and management
    
 
 
@@ -117,16 +122,6 @@ NO-JIRA -        Added missing Apache copyright/license text
    5651       -  [C++ broker] segfault in qpid::linearstore::journal::jdir::clear_dir when declaring durable queue
                    svn r.1582730 2014-03-28 Proposed fix by Pavel Moravec
                    * Bug introduced by r.1578899.
-   5362 1145363  Linearstore: No store tools exist for examining the journals
-                   svn r.1556888 2014-01-09: WIP checkin for linearstore version of qpid_qls_analyze. Needs testing and tidy-up.
-                   svn r.1560530 2014-01-22: Bugfixes for qpid_qls_analyze
-                   svn r.1561848 2014-01-27: Bugfixes and enhancements for qpid_qls_analyze
-                   svn r.1564808 2014-02-05: Bugfixes and enhancements for qpid_qls_analyze
-                   svn r.1578899 2014-03-18: Bugfixes and enhancements for qpid_qls_analyze
-                   svn r.1583778 2014-04-01: Bugfix for qpid_qls_analyze
-                   * Store analysis and status
-                   * Recovery/reading of message content
-                   * Empty file pool status and management
    5661       -  [linearstore] Set default cmake build to exclude linearstore
                    svn r.1584379 2014-04-03 Proposed solution.
                    * Run ccmake, select BUILD_LINEARSTORE to change its value to ON to build.
@@ -153,6 +148,12 @@ NO-JIRA -        Added missing Apache copyright/license text
                    svn r.1643053 2014-11-18: Proposed fix
    6248 1167911  [linearstore] Symlink creation fails if store dir path is not absolute
                    svn r.1641689 2014-11-25 Proposed solution
+   5671 1160367  [linearstore] Add ability to use disk partitions and select per-queue EFPs
+                   svn r.1636598 2014-11-04: WIP: New EFP and journal dir structure using symlinks
+                   svn r.1637985 2014-11-10: WIP: Auto-upgrade from old dir structure to new
+                   svn r.1649081 2015-01-02: WIP: Specify new queue using qpid-config --durable together with --efp-partition-num and/or --efp-pool-file-size. Needs testing.
+   -    1148807  [linearstore] Restarting broker with empty journal raises confusing warning
+                   Fixed by svn r.1649081 of bug 5671 / 1160367 above
                    
 
 Ordered checkin list:
@@ -196,10 +197,12 @@ no.   svn r  Q-JIRA     RHBZ       Date Alt Committer
 33. 1620426    6043  1089652 2014-08-25
 34. 1631360    6147  1152012 2014-10-13 (pmoravec)
 35. 1632504    6157  1150397 2014-10-17 (pmoravec)
-36. 1636598    5671          2014-11-04
-37. 1637985    5671          2014-11-10
+36. 1636598    5671  1160367 2014-11-04
+37. 1637985    5671  1160367 2014-11-10
 38. 1643053    6230  1165200 2014-11-18
 39. 1641689    6248  1167911 2014-11-25
+40. 1649081    5671  1160367 2015-01-02
+41. 1649082 NO-JIRA        - 2015-01-02
 
 See above sections for details on these checkins.
 
diff --git a/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp
index 68810936e1..70eac27f48 100644
--- a/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp
+++ b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp
@@ -1488,21 +1488,21 @@ std::string MessageStoreImpl::getStoreTopLevelDir() {
 std::string MessageStoreImpl::getJrnlBaseDir()
 {
     std::ostringstream dir;
-    dir << storeDir << "/" << storeTopLevelDir << "/jrnl/" ;
+    dir << storeDir << "/" << storeTopLevelDir << "/jrnl2/" ;
     return dir.str();
 }
 
 std::string MessageStoreImpl::getBdbBaseDir()
 {
     std::ostringstream dir;
-    dir << storeDir << "/" << storeTopLevelDir << "/dat/" ;
+    dir << storeDir << "/" << storeTopLevelDir << "/dat2/" ;
     return dir.str();
 }
 
 std::string MessageStoreImpl::getTplBaseDir()
 {
     std::ostringstream dir;
-    dir << storeDir << "/" << storeTopLevelDir << "/tpl/" ;
+    dir << storeDir << "/" << storeTopLevelDir << "/tpl2/" ;
     return dir.str();
 }
 
diff --git a/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolManager.cpp b/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolManager.cpp
index 33707578bf..a02679736e 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolManager.cpp
+++ b/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolManager.cpp
@@ -74,6 +74,7 @@ void EmptyFilePoolManager::findEfpPartitions() {
         if (!foundPartition) {
             std::ostringstream oss1;
             oss1 << qlsStorePath_ << "/" << EmptyFilePoolPartition::getPartionDirectoryName(defaultPartitionNumber_)
+                << "/" << EmptyFilePoolPartition::s_efpTopLevelDir_
                 << "/" << EmptyFilePool::dirNameFromDataSize(defaultEfpDataSize_kib_);
             jdir::create_dir(oss1.str());
             insertPartition(defaultPartitionNumber_, oss1.str());
diff --git a/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolPartition.cpp b/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolPartition.cpp
index 9b58e8c4ff..12d2db74b8 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolPartition.cpp
+++ b/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolPartition.cpp
@@ -32,6 +32,9 @@ namespace qpid {
 namespace linearstore {
 namespace journal {
 
+// static
+const std::string EmptyFilePoolPartition::s_efpTopLevelDir_("efp"); // Sets the top-level efp dir within a partition
+
 EmptyFilePoolPartition::EmptyFilePoolPartition(const efpPartitionNumber_t partitionNum,
                                                const std::string& partitionDir,
                                                const bool overwriteBeforeReturnFlag,
@@ -57,45 +60,18 @@ EmptyFilePoolPartition::~EmptyFilePoolPartition() {
 void
 EmptyFilePoolPartition::findEmptyFilePools() {
 //std::cout << "*** EmptyFilePoolPartition::findEmptyFilePools(): Reading " << partitionDir_ << std::endl; // DEBUG
-    std::vector<std::string> dirList;
-    bool upgradeDirStructureFlag = false;
-    std::string oldPartitionDir;
-        jdir::read_dir(partitionDir_, dirList, true, false, false, false);
-//std::cout << "*** dirList.size()=" << dirList.size() << "; dirList.front()=" << dirList.front() << std::endl; // DEBUG
-        if (dirList.size() == 1 && dirList.front().compare("efp") == 0) {
-            upgradeDirStructureFlag = true;
-            oldPartitionDir = partitionDir_ + "/efp";
-//std::cout << "*** oldPartitionDir=" << oldPartitionDir << std::endl; // DEBUG
-            dirList.clear();
-            jdir::read_dir(oldPartitionDir, dirList, true, false, false, false);
-        }
+    std::string efpDir(partitionDir_ + "/" + s_efpTopLevelDir_);
+    if (jdir::is_dir(efpDir)) {
+        std::vector<std::string> dirList;
+        jdir::read_dir(efpDir, dirList, true, false, false, true);
         for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) {
-            std::string fqFileName(partitionDir_ + "/" + *i);
-            if (upgradeDirStructureFlag) {
-                std::string fqOldFileName(partitionDir_ + "/efp/" + *i);
-                if (::rename(fqOldFileName.c_str(), fqFileName.c_str())) {
-                    // File move failed
-                    std::ostringstream oss;
-                    oss << "File \'" << fqOldFileName << "\' could not be renamed to \'" << fqFileName << "\' (" << FORMAT_SYSERR(errno) << "); file deleted";
-                    journalLogRef_.log(JournalLog::LOG_WARN, oss.str());
-                    if (::unlink(fqOldFileName.c_str())) {
-                        std::ostringstream oss;
-                        oss << "File \'" << fqOldFileName << "\' could not be deleted (" << FORMAT_SYSERR(errno) << "\'; file orphaned";
-                        journalLogRef_.log(JournalLog::LOG_WARN, oss.str());
-                    }
-                }
-            }
-            createEmptyFilePool(fqFileName);
-        }
-        if (upgradeDirStructureFlag) {
-            std::string oldEfpDir(partitionDir_ + "/efp");
-            if (::rmdir(oldEfpDir.c_str())) {
-                // Unable to delete old "efp" dir
-                std::ostringstream oss;
-                oss << "Unable to delete old EFP directory \'" << oldEfpDir << "\' (" << FORMAT_SYSERR(errno) << "\'; directory orphaned";
-                journalLogRef_.log(JournalLog::LOG_WARN, oss.str());
-            }
+            createEmptyFilePool(*i);
         }
+    } else {
+        std::ostringstream oss;
+        oss << "Partition \"" << partitionDir_ << "\" does not contain top level EFP dir \"" << s_efpTopLevelDir_ << "\"";
+        journalLogRef_.log(JournalLog::LOG_WARN, oss.str());
+    }
 }
 
 EmptyFilePool* EmptyFilePoolPartition::getEmptyFilePool(const efpDataSize_kib_t efpDataSize_kib, const bool createIfNonExistent) {
@@ -182,7 +158,7 @@ efpPartitionNumber_t EmptyFilePoolPartition::getPartitionNumber(const std::strin
 // --- protected functions ---
 
 EmptyFilePool* EmptyFilePoolPartition::createEmptyFilePool(const efpDataSize_kib_t efpDataSize_kib) {
-    std::string fqEfpDirectoryName(partitionDir_ + "/" + EmptyFilePool::dirNameFromDataSize(efpDataSize_kib));
+    std::string fqEfpDirectoryName(partitionDir_ + "/" + EmptyFilePoolPartition::s_efpTopLevelDir_ + "/" + EmptyFilePool::dirNameFromDataSize(efpDataSize_kib));
     return createEmptyFilePool(fqEfpDirectoryName);
 }
 
@@ -211,11 +187,12 @@ EmptyFilePool* EmptyFilePoolPartition::createEmptyFilePool(const std::string fqE
 }
 
 void EmptyFilePoolPartition::validatePartitionDir() {
+    std::ostringstream ss;
     if (!jdir::is_dir(partitionDir_)) {
-        std::ostringstream ss;
         ss << "Invalid partition directory: \'" << partitionDir_ << "\' is not a directory";
         throw jexception(jerrno::JERR_EFP_BADPARTITIONDIR, ss.str(), "EmptyFilePoolPartition", "validatePartitionDir");
     }
+
     // TODO: other validity checks here
 }
 
diff --git a/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolPartition.h b/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolPartition.h
index 23a541f8f4..570e2b073f 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolPartition.h
+++ b/qpid/cpp/src/qpid/linearstore/journal/EmptyFilePoolPartition.h
@@ -37,6 +37,8 @@ class JournalLog;
 
 class EmptyFilePoolPartition
 {
+public:
+    static const std::string s_efpTopLevelDir_;
 protected:
     typedef std::map<efpDataSize_kib_t, EmptyFilePool*> efpMap_t;
     typedef efpMap_t::iterator efpMapItr_t;
diff --git a/qpid/cpp/src/qpid/linearstore/journal/RecoveryManager.cpp b/qpid/cpp/src/qpid/linearstore/journal/RecoveryManager.cpp
index 198b39857c..73a16f01b7 100644
--- a/qpid/cpp/src/qpid/linearstore/journal/RecoveryManager.cpp
+++ b/qpid/cpp/src/qpid/linearstore/journal/RecoveryManager.cpp
@@ -429,7 +429,7 @@ void RecoveryManager::analyzeJournalFileHeaders(efpIdentity_t& efpIdentity) {
         }
     }
 
-std::cerr << "*** RecoveryManager::analyzeJournalFileHeaders() fileNumberMap_.size()=" << fileNumberMap_.size() << std::endl; // DEBUG
+//std::cerr << "*** RecoveryManager::analyzeJournalFileHeaders() fileNumberMap_.size()=" << fileNumberMap_.size() << std::endl; // DEBUG
     if (fileNumberMap_.empty()) {
         journalEmptyFlag_ = true;
     } else {