summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2014-09-29 13:16:04 +0000
committerGerrit Code Review <review@openstack.org>2014-09-29 13:16:04 +0000
commit8b61fdd2db831dcb651c42c9bcdafc1c8aa37451 (patch)
treec5d075892eff06fcdd2aa6136ef272bf8aff33c4
parent1c5ce73023660dfc416f7aba44710f3fcfe2d9b7 (diff)
parent125d950dd4a413801a28bfed7ac831227c0653f2 (diff)
downloadceilometer-2014.2.rc1.tar.gz
Merge "[HBase] Improves speed of unit tests on real HBase backend"2014.2.rc1
-rw-r--r--ceilometer/tests/db.py24
-rw-r--r--ceilometer/tests/mocks.py79
-rw-r--r--ceilometer/tests/storage/test_impl_hbase.py4
-rwxr-xr-xsetup-test-env.sh9
-rw-r--r--tools/test_hbase_table_utils.py38
5 files changed, 148 insertions, 6 deletions
diff --git a/ceilometer/tests/db.py b/ceilometer/tests/db.py
index d4843b01..68c869b5 100644
--- a/ceilometer/tests/db.py
+++ b/ceilometer/tests/db.py
@@ -23,6 +23,7 @@ import os
import uuid
import warnings
+import mock
from oslo.config import fixture as fixture_config
from oslotest import mockpatch
import six
@@ -32,6 +33,7 @@ from testtools import testcase
from ceilometer import storage
from ceilometer.tests import base as test_base
+from ceilometer.tests import mocks
class MongoDbManager(fixtures.Fixture):
@@ -71,12 +73,31 @@ class HBaseManager(fixtures.Fixture):
self.url, 'ceilometer.metering.storage')
self.alarm_connection = storage.get_connection(
self.url, 'ceilometer.alarm.storage')
+ # Unique prefix for each test to keep data is distinguished because
+ # all test data is stored in one table
+ data_prefix = str(uuid.uuid4().hex)
+
+ def table(conn, name):
+ return mocks.MockHBaseTable(name, conn, data_prefix)
+
+ # Mock only real HBase connection, MConnection "table" method
+ # stays origin.
+ mock.patch('happybase.Connection.table', new=table).start()
+ # We shouldn't delete data and tables after each test,
+ # because it last for too long.
+ # All tests tables will be deleted in setup-test-env.sh
+ mock.patch("happybase.Connection.disable_table",
+ new=mock.MagicMock()).start()
+ mock.patch("happybase.Connection.delete_table",
+ new=mock.MagicMock()).start()
+ mock.patch("happybase.Connection.create_table",
+ new=mock.MagicMock()).start()
@property
def url(self):
return '%s?table_prefix=%s' % (
self._url,
- uuid.uuid4().hex
+ os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", "test")
)
@@ -106,7 +127,6 @@ class TestBase(testscenarios.testcase.WithScenarios, test_base.BaseTestCase):
def setUp(self):
super(TestBase, self).setUp()
-
engine = urlparse.urlparse(self.db_url).scheme
# NOTE(Alexei_987) Shortcut to skip expensive db setUp
diff --git a/ceilometer/tests/mocks.py b/ceilometer/tests/mocks.py
new file mode 100644
index 00000000..f44682a4
--- /dev/null
+++ b/ceilometer/tests/mocks.py
@@ -0,0 +1,79 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import happybase
+
+
+class MockHBaseTable(happybase.Table):
+
+ def __init__(self, name, connection, data_prefix):
+ # data_prefix is added to all rows which are written
+ # in this test. It allows to divide data from different tests
+ self.data_prefix = data_prefix
+ # We create happybase Table with prefix from
+ # CEILOMETER_TEST_HBASE_TABLE_PREFIX
+ prefix = os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", 'test')
+ super(MockHBaseTable, self).__init__(
+ "%s_%s" % (prefix, name),
+ connection)
+
+ def put(self, row, *args, **kwargs):
+ row = self.data_prefix + row
+ return super(MockHBaseTable, self).put(row, *args,
+ **kwargs)
+
+ def scan(self, row_start=None, row_stop=None, row_prefix=None,
+ columns=None, filter=None, timestamp=None,
+ include_timestamp=False, batch_size=10, scan_batching=None,
+ limit=None, sorted_columns=False):
+ # Add data prefix for row parameters
+ # row_prefix could not be combined with row_start or row_stop
+ if not row_start and not row_stop:
+ row_prefix = self.data_prefix + (row_prefix or "")
+ row_start = None
+ row_stop = None
+ elif row_start and not row_stop:
+ # Adding data_prefix to row_start and row_stop does not work
+ # if it looks like row_start = %data_prefix%foo,
+ # row_stop = %data_prefix, because row_start > row_stop
+ filter = self._update_filter_row(filter)
+ row_start = self.data_prefix + row_start
+ else:
+ row_start = self.data_prefix + (row_start or "")
+ row_stop = self.data_prefix + (row_stop or "")
+ gen = super(MockHBaseTable, self).scan(row_start, row_stop,
+ row_prefix, columns,
+ filter, timestamp,
+ include_timestamp, batch_size,
+ scan_batching, limit,
+ sorted_columns)
+ data_prefix_len = len(self.data_prefix)
+ # Restore original row format
+ for row, data in gen:
+ yield (row[data_prefix_len:], data)
+
+ def row(self, row, *args, **kwargs):
+ row = self.data_prefix + row
+ return super(MockHBaseTable, self).row(row, *args, **kwargs)
+
+ def delete(self, row, *args, **kwargs):
+ row = self.data_prefix + row
+ return super(MockHBaseTable, self).delete(row, *args, **kwargs)
+
+ def _update_filter_row(self, filter):
+ if filter:
+ return "PrefixFilter(%s) AND %s" % (self.data_prefix, filter)
+ else:
+ return "PrefixFilter(%s)" % self.data_prefix
diff --git a/ceilometer/tests/storage/test_impl_hbase.py b/ceilometer/tests/storage/test_impl_hbase.py
index 1900c556..e5ca5bad 100644
--- a/ceilometer/tests/storage/test_impl_hbase.py
+++ b/ceilometer/tests/storage/test_impl_hbase.py
@@ -26,7 +26,6 @@
import mock
from ceilometer.alarm.storage import impl_hbase as hbase_alarm
-from ceilometer.storage.hbase import inmemory as hbase_inmemory
from ceilometer.storage import impl_hbase as hbase
from ceilometer.tests import base as test_base
from ceilometer.tests import db as tests_db
@@ -37,9 +36,6 @@ class ConnectionTest(tests_db.TestBase,
@tests_db.run_with('hbase')
def test_hbase_connection(self):
- conn = hbase.Connection(self.db_manager.url)
- self.assertIsInstance(conn.conn_pool.connection(),
- hbase_inmemory.MConnection)
class TestConn(object):
def __init__(self, host, port):
diff --git a/setup-test-env.sh b/setup-test-env.sh
index d29ac5cf..09fc9b4e 100755
--- a/setup-test-env.sh
+++ b/setup-test-env.sh
@@ -4,6 +4,10 @@ set -e
function clean_exit(){
local error_code="$?"
rm -rf ${MONGO_DATA}
+ if test -n "$CEILOMETER_TEST_HBASE_URL"
+ then
+ python tools/test_hbase_table_utils.py --clear
+ fi
kill $(jobs -p)
return $error_code
}
@@ -28,6 +32,11 @@ done < ${MONGO_DATA}/out
# Read the fifo for ever otherwise mongod would block
cat ${MONGO_DATA}/out > /dev/null &
export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:${MONGO_PORT}/ceilometer"
+if test -n "$CEILOMETER_TEST_HBASE_URL"
+then
+ export CEILOMETER_TEST_HBASE_TABLE_PREFIX=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)
+ python tools/test_hbase_table_utils.py --upgrade
+fi
# Yield execution to venv command
$*
diff --git a/tools/test_hbase_table_utils.py b/tools/test_hbase_table_utils.py
new file mode 100644
index 00000000..5ecc49ad
--- /dev/null
+++ b/tools/test_hbase_table_utils.py
@@ -0,0 +1,38 @@
+ # Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import sys
+
+from oslo.config import cfg
+
+from ceilometer import storage
+
+def main(argv):
+ cfg.CONF([], project='ceilometer')
+ if os.getenv("CEILOMETER_TEST_HBASE_URL"):
+ url = ("%s?table_prefix=%s" %
+ (os.getenv("CEILOMETER_TEST_HBASE_URL"),
+ os.getenv("CEILOMETER_TEST_HBASE_TABLE_PREFIX", "test")))
+ conn = storage.get_connection(url, 'ceilometer.metering.storage')
+ alarm_conn = storage.get_connection(url, 'ceilometer.alarm.storage')
+ for arg in argv:
+ if arg == "--upgrade":
+ conn.upgrade()
+ alarm_conn.upgrade()
+ if arg == "--clear":
+ conn.clear()
+ alarm_conn.clear()
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:]) \ No newline at end of file