diff options
57 files changed, 416 insertions, 259 deletions
diff --git a/doc/governance.rst b/doc/governance.rst index 8612bcc0c6a..4aaaf4d6269 100644 --- a/doc/governance.rst +++ b/doc/governance.rst @@ -52,6 +52,7 @@ the CLT itself. Current CLT members are: + * Adam King <adking@redhat.com> * Casey Bodley <cbodley@redhat.com> * Dan van der Ster <daniel.vanderster@cern.ch> * David Galloway <dgallowa@redhat.com> @@ -73,7 +74,6 @@ Current CLT members are: * Patrick Donnelly <pdonnell@redhat.com> * Sage Weil <sage@redhat.com> * Sam Just <sjust@redhat.com> - * Sebastian Wagner <sewagner@redhat.com> * Xie Xingguo <xie.xingguo@zte.com.cn> * Yehuda Sadeh <yehuda@redhat.com> * Yuri Weinstein <yweinste@redhat.com> diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst index 57d7fcfc902..042d1f3391e 100644 --- a/doc/man/8/radosgw-admin.rst +++ b/doc/man/8/radosgw-admin.rst @@ -997,11 +997,11 @@ Rename a bucket:: Move a bucket from the old global tenant space to a specified tenant:: - $ radosgw-admin bucket link --bucket=/foo --uid=12345678$12345678' + $ radosgw-admin bucket link --bucket=/foo --uid='12345678$12345678' Link bucket to specified user and change object ACLs:: - $ radosgw-admin bucket chown --bucket=/foo --uid=12345678$12345678' + $ radosgw-admin bucket chown --bucket=/foo --uid='12345678$12345678' Show the logs of a bucket from April 1st, 2012:: diff --git a/doc/rados/operations/crush-map.rst b/doc/rados/operations/crush-map.rst index 792bbcdf219..90a3871144f 100644 --- a/doc/rados/operations/crush-map.rst +++ b/doc/rados/operations/crush-map.rst @@ -953,8 +953,7 @@ release notes and documentation carefully before changing the profile on a running cluster, and consider throttling recovery/backfill parameters to limit the impact of a bolus of backfill. - -.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.com/wp-content/uploads/2016/08/weil-crush-sc06.pdf +.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.com/assets/pdfs/weil-crush-sc06.pdf Primary Affinity diff --git a/qa/suites/rgw/crypt/2-kms/kmip.yaml b/qa/suites/rgw/crypt/2-kms/kmip.yaml index 4b2a13f4278..0057d954e32 100644 --- a/qa/suites/rgw/crypt/2-kms/kmip.yaml +++ b/qa/suites/rgw/crypt/2-kms/kmip.yaml @@ -3,9 +3,9 @@ overrides: conf: client: rgw crypt s3 kms backend: kmip - rgw crypt kmip ca path: /home/ubuntu/cephtest/ca/kmiproot.crt - rgw crypt kmip client cert: /home/ubuntu/cephtest/ca/kmip-client.crt - rgw crypt kmip client key: /home/ubuntu/cephtest/ca/kmip-client.key + rgw crypt kmip ca path: /etc/ceph/kmiproot.crt + rgw crypt kmip client cert: /etc/ceph/kmip-client.crt + rgw crypt kmip client key: /etc/ceph/kmip-client.key rgw crypt kmip kms key template: pykmip-$keyid rgw: client.0: diff --git a/qa/tasks/cephfs/test_volumes.py b/qa/tasks/cephfs/test_volumes.py index 9052b578906..6d3d3dfda3a 100644 --- a/qa/tasks/cephfs/test_volumes.py +++ b/qa/tasks/cephfs/test_volumes.py @@ -3733,6 +3733,40 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): # verify trash dir is clean self._wait_for_trash_empty() + def test_subvolume_snapshot_clone_quota_exceeded(self): + subvolume = self._generate_random_subvolume_name() + snapshot = self._generate_random_snapshot_name() + clone = self._generate_random_clone_name() + + # create subvolume with 20MB quota + osize = self.DEFAULT_FILE_SIZE*1024*1024*20 + self._fs_cmd("subvolume", "create", self.volname, subvolume,"--mode=777", "--size", str(osize)) + + # do IO, write 50 files of 1MB each to exceed quota. This mostly succeeds as quota enforcement takes time. + self._do_subvolume_io(subvolume, number_of_files=50) + + # snapshot subvolume + self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) + + # schedule a clone + self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) + + # check clone status + self._wait_for_clone_to_complete(clone) + + # verify clone + self._verify_clone(subvolume, snapshot, clone) + + # remove snapshot + self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) + + # remove subvolumes + self._fs_cmd("subvolume", "rm", self.volname, subvolume) + self._fs_cmd("subvolume", "rm", self.volname, clone) + + # verify trash dir is clean + self._wait_for_trash_empty() + def test_subvolume_snapshot_in_complete_clone_rm(self): """ Validates the removal of clone when it is not in 'complete|cancelled|failed' state. diff --git a/qa/tasks/rgw.py b/qa/tasks/rgw.py index 693d3d4d6d7..3d2542981b1 100644 --- a/qa/tasks/rgw.py +++ b/qa/tasks/rgw.py @@ -150,6 +150,35 @@ def start_rgw(ctx, config, clients): '--rgw_crypt_kmip_addr', "{}:{}".format(*ctx.pykmip.endpoints[pykmip_role]), ]) + clientcert = ctx.ssl_certificates.get('kmip-client') + servercert = ctx.ssl_certificates.get('kmip-server') + clientca = ctx.ssl_certificates.get('kmiproot') + + clientkey = clientcert.key + clientcert = clientcert.certificate + serverkey = servercert.key + servercert = servercert.certificate + rootkey = clientca.key + rootcert = clientca.certificate + + cert_path = '/etc/ceph/' + ctx.cluster.only(client).run(args=['sudo', 'cp', clientcert, cert_path]) + ctx.cluster.only(client).run(args=['sudo', 'cp', clientkey, cert_path]) + ctx.cluster.only(client).run(args=['sudo', 'cp', servercert, cert_path]) + ctx.cluster.only(client).run(args=['sudo', 'cp', serverkey, cert_path]) + ctx.cluster.only(client).run(args=['sudo', 'cp', rootkey, cert_path]) + ctx.cluster.only(client).run(args=['sudo', 'cp', rootcert, cert_path]) + + clientcert = cert_path + 'kmip-client.crt' + clientkey = cert_path + 'kmip-client.key' + servercert = cert_path + 'kmip-server.crt' + serverkey = cert_path + 'kmip-server.key' + rootkey = cert_path + 'kmiproot.key' + rootcert = cert_path + 'kmiproot.crt' + + ctx.cluster.only(client).run(args=['sudo', 'chmod', '600', clientcert, clientkey, servercert, serverkey, rootkey, rootcert]) + ctx.cluster.only(client).run(args=['sudo', 'chown', 'ceph', clientcert, clientkey, servercert, serverkey, rootkey, rootcert]) + rgw_cmd.extend([ '--foreground', run.Raw('|'), diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini b/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini index 508d1b4c6cd..4556af9b1aa 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini @@ -34,6 +34,7 @@ changedir= commands= git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible python -m pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt + ansible-galaxy install -r {envdir}/tmp/ceph-ansible/requirements.yml -v # bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:""} {posargs:--provider=virtualbox} bash {toxinidir}/../scripts/vagrant_up.sh {posargs:--provider=virtualbox} diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini b/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini index bec30e6d7cb..9da13be2820 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini @@ -32,6 +32,7 @@ changedir= commands= git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt + ansible-galaxy install -r {envdir}/tmp/ceph-ansible/requirements.yml -v bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir} diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini b/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini index 1fdfe26a876..f5f9c8024b9 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini +++ b/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini @@ -28,6 +28,7 @@ changedir= commands= git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch https://github.com/ceph/ceph-ansible.git {envdir}/tmp/ceph-ansible pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt + ansible-galaxy install -r {envdir}/tmp/ceph-ansible/requirements.yml -v bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir} diff --git a/src/ceph-volume/ceph_volume/tests/util/test_disk.py b/src/ceph-volume/ceph_volume/tests/util/test_disk.py index 44f19e036fa..6b178eabdbb 100644 --- a/src/ceph-volume/ceph_volume/tests/util/test_disk.py +++ b/src/ceph-volume/ceph_volume/tests/util/test_disk.py @@ -331,6 +331,15 @@ class TestGetDevices(object): result = disk.get_devices(_sys_block_path=block_path) assert result[sda_path]['rotational'] == '1' + def test_is_ceph_rbd(self, tmpfile, tmpdir, patched_get_block_devs_lsblk): + rbd_path = '/dev/rbd0' + patched_get_block_devs_lsblk.return_value = [[rbd_path, rbd_path, 'disk']] + block_path = self.setup_path(tmpdir) + block_rbd_path = os.path.join(block_path, 'rbd0') + os.makedirs(block_rbd_path) + result = disk.get_devices(_sys_block_path=block_path) + assert rbd_path not in result + class TestSizeCalculations(object): diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py index 88db0513817..8a47061ef91 100644 --- a/src/ceph-volume/ceph_volume/util/disk.py +++ b/src/ceph-volume/ceph_volume/util/disk.py @@ -361,6 +361,13 @@ def is_partition(dev): return False +def is_ceph_rbd(dev): + """ + Boolean to determine if a given device is a ceph RBD device, like /dev/rbd0 + """ + return dev.startswith(('/dev/rbd')) + + class BaseFloatUnit(float): """ Base class to support float representations of size values. Suffix is @@ -764,6 +771,10 @@ def get_devices(_sys_block_path='/sys/block'): sysdir = os.path.join(_sys_block_path, devname) metadata = {} + # If the device is ceph rbd it gets excluded + if is_ceph_rbd(diskname): + continue + # If the mapper device is a logical volume it gets excluded if is_mapper_device(diskname): if lvm.get_device_lvs(diskname): diff --git a/src/ceph-volume/ceph_volume/util/system.py b/src/ceph-volume/ceph_volume/util/system.py index 2563c736fea..ed1fb8ed2a2 100644 --- a/src/ceph-volume/ceph_volume/util/system.py +++ b/src/ceph-volume/ceph_volume/util/system.py @@ -59,10 +59,10 @@ def find_executable_on_host(locations=[], executable='', binary_check='/bin/ls') stdout = as_string(process.stdout.read()) if stdout: executable_on_host = stdout.split('\n')[0] - mlogger.info('Executable {} found on the host, will use {}'.format(executable, executable_on_host)) + logger.info('Executable {} found on the host, will use {}'.format(executable, executable_on_host)) return executable_on_host else: - mlogger.warning('Executable {} not found on the host, will return {} as-is'.format(executable, executable)) + logger.warning('Executable {} not found on the host, will return {} as-is'.format(executable, executable)) return executable def which(executable, run_on_host=False): diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index 0597a0315ab..4b7ab5495af 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -842,7 +842,7 @@ void Cache::on_transaction_destruct(Transaction& t) CachedExtentRef Cache::alloc_new_extent_by_type( Transaction &t, ///< [in, out] current transaction extent_types_t type, ///< [in] type tag - segment_off_t length, ///< [in] length + seastore_off_t length, ///< [in] length bool delay ///< [in] whether to delay paddr alloc ) { @@ -974,7 +974,7 @@ record_t Cache::prepare_record(Transaction &t) : L_ADDR_NULL), i->last_committed_crc, final_crc, - (segment_off_t)i->get_length(), + (seastore_off_t)i->get_length(), i->get_version() - 1, i->get_delta() }); @@ -1386,7 +1386,7 @@ Cache::get_extent_ertr::future<CachedExtentRef> Cache::_get_extent_by_type( extent_types_t type, paddr_t offset, laddr_t laddr, - segment_off_t length, + seastore_off_t length, const Transaction::src_t* p_src, extent_init_func_t &&extent_init_func) { diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index ba8faaea4fb..0b4a50afed7 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -181,7 +181,7 @@ public: template <typename T, typename Func> get_extent_ret<T> get_extent( paddr_t offset, ///< [in] starting addr - segment_off_t length, ///< [in] length + seastore_off_t length, ///< [in] length const src_ext_t* p_metric_key, ///< [in] cache query metric key Func &&extent_init_func ///< [in] init func for extent ) { @@ -230,7 +230,7 @@ public: template <typename T> get_extent_ret<T> get_extent( paddr_t offset, ///< [in] starting addr - segment_off_t length, ///< [in] length + seastore_off_t length, ///< [in] length const src_ext_t* p_metric_key ///< [in] cache query metric key ) { return get_extent<T>( @@ -302,7 +302,7 @@ public: get_extent_iertr::future<TCachedExtentRef<T>> get_extent( Transaction &t, paddr_t offset, - segment_off_t length, + seastore_off_t length, Func &&extent_init_func) { CachedExtentRef ret; LOG_PREFIX(Cache::get_extent); @@ -343,7 +343,7 @@ public: get_extent_iertr::future<TCachedExtentRef<T>> get_extent( Transaction &t, paddr_t offset, - segment_off_t length) { + seastore_off_t length) { return get_extent<T>(t, offset, length, [](T &){}); } @@ -385,7 +385,7 @@ private: extent_types_t type, paddr_t offset, laddr_t laddr, - segment_off_t length, + seastore_off_t length, const Transaction::src_t* p_src, extent_init_func_t &&extent_init_func ); @@ -398,7 +398,7 @@ private: extent_types_t type, paddr_t offset, laddr_t laddr, - segment_off_t length, + seastore_off_t length, extent_init_func_t &&extent_init_func) { CachedExtentRef ret; auto status = t.get_extent(offset, &ret); @@ -436,7 +436,7 @@ public: extent_types_t type, ///< [in] type tag paddr_t offset, ///< [in] starting addr laddr_t laddr, ///< [in] logical address if logical - segment_off_t length, ///< [in] length + seastore_off_t length, ///< [in] length Func &&extent_init_func ///< [in] extent init func ) { return _get_extent_by_type( @@ -452,7 +452,7 @@ public: extent_types_t type, paddr_t offset, laddr_t laddr, - segment_off_t length + seastore_off_t length ) { return get_extent_by_type( t, type, offset, laddr, length, [](CachedExtent &) {}); @@ -467,7 +467,7 @@ public: template <typename T> TCachedExtentRef<T> alloc_new_extent( Transaction &t, ///< [in, out] current transaction - segment_off_t length, ///< [in] length + seastore_off_t length, ///< [in] length bool delayed = false ///< [in] whether the paddr allocation of extent is delayed ) { auto ret = CachedExtent::make_cached_extent_ref<T>( @@ -502,7 +502,7 @@ public: CachedExtentRef alloc_new_extent_by_type( Transaction &t, ///< [in, out] current transaction extent_types_t type, ///< [in] type tag - segment_off_t length, ///< [in] length + seastore_off_t length, ///< [in] length bool delayed = false ///< [in] whether delay addr allocation ); diff --git a/src/crimson/os/seastore/cached_extent.h b/src/crimson/os/seastore/cached_extent.h index 99efa039140..906a7272eb5 100644 --- a/src/crimson/os/seastore/cached_extent.h +++ b/src/crimson/os/seastore/cached_extent.h @@ -546,7 +546,7 @@ class ExtentIndex { friend class Cache; CachedExtent::index extent_index; public: - auto get_overlap(paddr_t addr, segment_off_t len) { + auto get_overlap(paddr_t addr, seastore_off_t len) { auto bottom = extent_index.upper_bound(addr, paddr_cmp()); if (bottom != extent_index.begin()) --bottom; diff --git a/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc b/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc index 6d762844eb9..7b0cb1f9c90 100644 --- a/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc +++ b/src/crimson/os/seastore/collection_manager/flat_collection_manager.cc @@ -18,8 +18,8 @@ namespace { namespace crimson::os::seastore::collection_manager { -constexpr static segment_off_t MIN_FLAT_BLOCK_SIZE = 4<<10; -[[maybe_unused]] constexpr static segment_off_t MAX_FLAT_BLOCK_SIZE = 4<<20; +constexpr static seastore_off_t MIN_FLAT_BLOCK_SIZE = 4<<10; +[[maybe_unused]] constexpr static seastore_off_t MAX_FLAT_BLOCK_SIZE = 4<<20; FlatCollectionManager::FlatCollectionManager( TransactionManager &tm) diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index c4d5d285668..0da195f2326 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -217,7 +217,7 @@ SegmentedAllocator::Writer::write( return write_func(); } -bool SegmentedAllocator::Writer::_needs_roll(segment_off_t length) const { +bool SegmentedAllocator::Writer::_needs_roll(seastore_off_t length) const { return allocated_to + length > current_segment->segment->get_write_capacity(); } diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index 687c4514684..50c2a081523 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -61,14 +61,14 @@ public: assert(extents.size() == record.extents.size()); assert(!record.deltas.size()); auto record_group = record_group_t(std::move(record), block_size); - segment_off_t extent_offset = base + record_group.size.get_mdlength(); + seastore_off_t extent_offset = base + record_group.size.get_mdlength(); for (auto& extent : extents) { extent.set_ool_paddr( paddr_t::make_seg_paddr(segment, extent_offset)); extent_offset += extent.get_bptr().length(); } assert(extent_offset == - (segment_off_t)(base + record_group.size.get_encoded_length())); + (seastore_off_t)(base + record_group.size.get_encoded_length())); return encode_records(record_group, journal_seq_t(), nonce); } void add_extent(LogicalCachedExtentRef& extent) { @@ -83,10 +83,10 @@ public: std::vector<OolExtent>& get_extents() { return extents; } - void set_base(segment_off_t b) { + void set_base(seastore_off_t b) { base = b; } - segment_off_t get_base() const { + seastore_off_t get_base() const { return base; } void clear() { @@ -102,7 +102,7 @@ private: std::vector<OolExtent> extents; record_t record; size_t block_size; - segment_off_t base = MAX_SEG_OFF; + seastore_off_t base = MAX_SEG_OFF; }; /** @@ -209,7 +209,7 @@ class SegmentedAllocator : public ExtentAllocator { finish_record_ret finish_write( Transaction& t, ool_record_t& record); - bool _needs_roll(segment_off_t length) const; + bool _needs_roll(seastore_off_t length) const; write_iertr::future<> _write( Transaction& t, @@ -231,7 +231,7 @@ class SegmentedAllocator : public ExtentAllocator { SegmentManager& segment_manager; open_segment_wrapper_ref current_segment; std::list<open_segment_wrapper_ref> open_segments; - segment_off_t allocated_to = 0; + seastore_off_t allocated_to = 0; LBAManager& lba_manager; Journal& journal; crimson::condition_variable segment_rotation_guard; @@ -301,7 +301,7 @@ public: CachedExtentRef alloc_new_extent_by_type( Transaction& t, extent_types_t type, - segment_off_t length, + seastore_off_t length, placement_hint_t hint) { // only logical extents should fall in this path assert(is_logical_type(type)); @@ -322,7 +322,7 @@ public: std::enable_if_t<std::is_base_of_v<LogicalCachedExtent, T>, int> = 0> TCachedExtentRef<T> alloc_new_extent( Transaction& t, - segment_off_t length, + seastore_off_t length, placement_hint_t hint) { // only logical extents should fall in this path static_assert(is_logical_type(T::TYPE)); diff --git a/src/crimson/os/seastore/extent_reader.cc b/src/crimson/os/seastore/extent_reader.cc index 8b66ffffd31..15ca6cad765 100644 --- a/src/crimson/os/seastore/extent_reader.cc +++ b/src/crimson/os/seastore/extent_reader.cc @@ -253,7 +253,7 @@ ExtentReader::read_validate_record_metadata( auto rest_start = paddr_t::make_seg_paddr( seg_addr.get_segment_id(), - seg_addr.get_segment_off() + (segment_off_t)block_size + seg_addr.get_segment_off() + (seastore_off_t)block_size ); auto rest_len = header.mdlength - block_size; TRACE("reading record group header rest {}~{}", rest_start, rest_len); @@ -315,7 +315,7 @@ ExtentReader::consume_next_records( cursor.seq.segment_seq, next.offset }, - static_cast<segment_off_t>(total_length) + static_cast<seastore_off_t>(total_length) } }; DEBUG("processing {} at {}, budget_used={}", diff --git a/src/crimson/os/seastore/extent_reader.h b/src/crimson/os/seastore/extent_reader.h index b6e5013cba7..697adb47aa8 100644 --- a/src/crimson/os/seastore/extent_reader.h +++ b/src/crimson/os/seastore/extent_reader.h @@ -6,6 +6,7 @@ #include "crimson/common/errorator.h" #include "crimson/os/seastore/seastore_types.h" #include "crimson/os/seastore/segment_manager.h" +#include "crimson/os/seastore/logging.h" namespace crimson::os::seastore { @@ -14,7 +15,7 @@ class TransactionManager; class ExtentReader { public: - segment_off_t get_block_size() const { + seastore_off_t get_block_size() const { assert(segment_managers.size()); // assume all segment managers have the same block size return segment_managers[0]->get_block_size(); diff --git a/src/crimson/os/seastore/journal.cc b/src/crimson/os/seastore/journal.cc index 39a9063d62a..173a6c8124c 100644 --- a/src/crimson/os/seastore/journal.cc +++ b/src/crimson/os/seastore/journal.cc @@ -424,7 +424,7 @@ Journal::JournalSegmentManager::write(ceph::bufferlist to_write) written_to += write_length; auto write_result = write_result_t{ write_start_seq, - static_cast<segment_off_t>(write_length) + static_cast<seastore_off_t>(write_length) }; return current_journal_segment->write( write_start_offset, to_write diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h index 4a68a9a9308..538e8f7a973 100644 --- a/src/crimson/os/seastore/journal.h +++ b/src/crimson/os/seastore/journal.h @@ -140,7 +140,7 @@ private: return segment_manager.get_device_id(); } - segment_off_t get_block_size() const { + seastore_off_t get_block_size() const { return segment_manager.get_block_size(); } @@ -219,7 +219,7 @@ private: segment_nonce_t current_segment_nonce; SegmentRef current_journal_segment; - segment_off_t written_to; + seastore_off_t written_to; // committed_to may be in a previous journal segment journal_seq_t committed_to; }; @@ -332,12 +332,12 @@ private: record_group_t pending; std::size_t submitting_size = 0; - segment_off_t submitting_length = 0; - segment_off_t submitting_mdlength = 0; + seastore_off_t submitting_length = 0; + seastore_off_t submitting_mdlength = 0; struct promise_result_t { write_result_t write_result; - segment_off_t mdlength; + seastore_off_t mdlength; }; using maybe_promise_result_t = std::optional<promise_result_t>; std::optional<seastar::shared_promise<maybe_promise_result_t> > io_promise; diff --git a/src/crimson/os/seastore/lba_manager.h b/src/crimson/os/seastore/lba_manager.h index 406cf8a35ff..c98302371ed 100644 --- a/src/crimson/os/seastore/lba_manager.h +++ b/src/crimson/os/seastore/lba_manager.h @@ -190,7 +190,7 @@ public: extent_types_t type, paddr_t addr, laddr_t laddr, - segment_off_t len) = 0; + seastore_off_t len) = 0; virtual void add_pin(LBAPin &pin) = 0; diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc index 971d018a319..f15582733f7 100644 --- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc +++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.cc @@ -402,7 +402,7 @@ BtreeLBAManager::get_physical_extent_if_live( extent_types_t type, paddr_t addr, laddr_t laddr, - segment_off_t len) + seastore_off_t len) { ceph_assert(is_lba_node(type)); auto c = get_context(t); diff --git a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h index a036b337b81..7194019d62a 100644 --- a/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h +++ b/src/crimson/os/seastore/lba_manager/btree/btree_lba_manager.h @@ -113,7 +113,7 @@ public: extent_types_t type, paddr_t addr, laddr_t laddr, - segment_off_t len) final; + seastore_off_t len) final; void add_pin(LBAPin &pin) final { auto *bpin = reinterpret_cast<BtreeLBAPin*>(&pin); diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc index 4bfb5edf474..84358df08da 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc @@ -343,7 +343,7 @@ LBABtree::get_internal_if_live( op_context_t c, paddr_t addr, laddr_t laddr, - segment_off_t len) + seastore_off_t len) { LOG_PREFIX(BtreeLBAManager::get_leaf_if_live); return lower_bound( @@ -379,7 +379,7 @@ LBABtree::get_leaf_if_live( op_context_t c, paddr_t addr, laddr_t laddr, - segment_off_t len) + seastore_off_t len) { LOG_PREFIX(BtreeLBAManager::get_leaf_if_live); return lower_bound( diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h index ab604e302ae..5caf0940474 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h @@ -391,7 +391,7 @@ public: op_context_t c, paddr_t addr, laddr_t laddr, - segment_off_t len); + seastore_off_t len); /// get_internal_if_live: get internal node at laddr/addr if still live using get_internal_if_live_iertr = base_iertr; @@ -400,7 +400,7 @@ public: op_context_t c, paddr_t addr, laddr_t laddr, - segment_off_t len); + seastore_off_t len); /** * rewrite_lba_extent diff --git a/src/crimson/os/seastore/random_block_manager.h b/src/crimson/os/seastore/random_block_manager.h index ee707478a2b..14d6f569601 100644 --- a/src/crimson/os/seastore/random_block_manager.h +++ b/src/crimson/os/seastore/random_block_manager.h @@ -32,7 +32,6 @@ public: paddr_t end; size_t block_size = 0; size_t total_size = 0; - uint32_t blocks_per_segment = 1 << 18; device_id_t device_id = 0; seastore_meta_t meta; }; @@ -47,7 +46,7 @@ public: crimson::ct_error::invarg, crimson::ct_error::enoent, crimson::ct_error::erange>; - virtual read_ertr::future<> read(uint64_t addr, bufferptr &buffer) = 0; + virtual read_ertr::future<> read(paddr_t addr, bufferptr &buffer) = 0; using write_ertr = crimson::errorator< crimson::ct_error::input_output_error, @@ -56,7 +55,7 @@ public: crimson::ct_error::enospc, crimson::ct_error::erange >; - virtual write_ertr::future<> write(uint64_t addr, bufferptr &buf) = 0; + virtual write_ertr::future<> write(paddr_t addr, bufferptr &buf) = 0; using open_ertr = crimson::errorator< crimson::ct_error::input_output_error, @@ -95,10 +94,19 @@ public: virtual size_t get_size() const = 0; virtual size_t get_block_size() const = 0; virtual uint64_t get_free_blocks() const = 0; - virtual uint32_t get_blocks_per_segment() const = 0; virtual device_id_t get_device_id() const = 0; virtual ~RandomBlockManager() {} }; using RandomBlockManagerRef = std::unique_ptr<RandomBlockManager>; +using blk_no_t = uint64_t; +using rbm_abs_addr = uint64_t; +inline rbm_abs_addr convert_paddr_to_abs_addr(paddr_t& paddr) { + blk_paddr_t& blk_addr = paddr.as_blk_paddr(); + return blk_addr.get_block_off(); +} + +inline paddr_t convert_abs_addr_to_paddr(rbm_abs_addr addr, device_id_t d_id) { + return paddr_t::make_blk_paddr(d_id, addr); +} } diff --git a/src/crimson/os/seastore/random_block_manager/nvme_manager.cc b/src/crimson/os/seastore/random_block_manager/nvme_manager.cc index 4906cad30bc..1d53f10d216 100644 --- a/src/crimson/os/seastore/random_block_manager/nvme_manager.cc +++ b/src/crimson/os/seastore/random_block_manager/nvme_manager.cc @@ -21,7 +21,7 @@ namespace { namespace crimson::os::seastore { NVMeManager::write_ertr::future<> NVMeManager::rbm_sync_block_bitmap( - rbm_bitmap_block_t &block, blk_id_t block_no) + rbm_bitmap_block_t &block, blk_no_t block_no) { bufferptr bptr; try { @@ -67,7 +67,7 @@ NVMeManager::mkfs_ertr::future<> NVMeManager::initialize_blk_alloc_area() { /* initialize bitmap blocks as unused */ auto max = max_block_by_bitmap_block(); auto max_block = super.size / super.block_size; - blk_id_t end = round_up_to(max_block, max) - 1; + blk_no_t end = round_up_to(max_block, max) - 1; logger().debug(" init start {} end {} ", start, end); return rbm_sync_block_bitmap_by_range( start, @@ -123,10 +123,8 @@ NVMeManager::mkfs_ertr::future<> NVMeManager::mkfs(mkfs_config_t config) { logger().debug("path {}", path); return _open_device(path).safe_then([this, &config]() { - blk_paddr_t addr = convert_paddr_to_blk_paddr( - config.start, - config.block_size, - config.blocks_per_segment); + rbm_abs_addr addr = convert_paddr_to_abs_addr( + config.start); return read_rbm_header(addr).safe_then([](auto super) { logger().debug(" already exists "); return mkfs_ertr::now(); @@ -134,14 +132,10 @@ NVMeManager::mkfs_ertr::future<> NVMeManager::mkfs(mkfs_config_t config) crimson::ct_error::enoent::handle([this, &config] (auto) { super.uuid = uuid_d(); // TODO super.magic = 0xFF; // TODO - super.start = convert_paddr_to_blk_paddr( - config.start, - config.block_size, - config.blocks_per_segment); - super.end = convert_paddr_to_blk_paddr( - config.end, - config.block_size, - config.blocks_per_segment); + super.start = convert_paddr_to_abs_addr( + config.start); + super.end = convert_paddr_to_abs_addr( + config.end); super.block_size = config.block_size; super.size = config.total_size; super.free_block_count = config.total_size/config.block_size - 2; @@ -151,7 +145,6 @@ NVMeManager::mkfs_ertr::future<> NVMeManager::mkfs(mkfs_config_t config) super.start_alloc_area + super.alloc_area_size; super.crc = 0; super.feature |= RBM_BITMAP_BLOCK_CRC; - super.blocks_per_segment = config.blocks_per_segment; super.device_id = config.device_id; logger().debug(" super {} ", super); @@ -187,7 +180,7 @@ NVMeManager::find_block_ret NVMeManager::find_free_block(Transaction &t, size_t auto bp = bufferptr(ceph::buffer::create_page_aligned(super.block_size)); return seastar::do_with(uint64_t(0), uint64_t(super.start_alloc_area), - interval_set<blk_id_t>(), + interval_set<blk_no_t>(), bp, [&, this] (auto &allocated, auto &addr, auto &alloc_extent, auto &bp) mutable { return crimson::repeat( @@ -275,10 +268,8 @@ NVMeManager::allocate_ret NVMeManager::alloc_extent( if (!alloc_extent.empty()) { rbm_alloc_delta_t alloc_info; for (auto p : alloc_extent) { - paddr_t paddr = convert_blk_paddr_to_paddr( + paddr_t paddr = convert_abs_addr_to_paddr( p.first * super.block_size, - super.block_size, - super.blocks_per_segment, super.device_id); size_t len = p.second * super.block_size; alloc_info.alloc_blk_ranges.push_back(std::make_pair(paddr, len)); @@ -288,10 +279,8 @@ NVMeManager::allocate_ret NVMeManager::alloc_extent( } else { return crimson::ct_error::enospc::make(); } - paddr_t paddr = convert_blk_paddr_to_paddr( + paddr_t paddr = convert_abs_addr_to_paddr( alloc_extent.range_start() * super.block_size, - super.block_size, - super.blocks_per_segment, super.device_id); return allocate_ret( allocate_ertr::ready_future_marker{}, @@ -306,13 +295,11 @@ NVMeManager::allocate_ret NVMeManager::alloc_extent( } void NVMeManager::add_free_extent( - std::vector<rbm_alloc_delta_t>& v, blk_paddr_t from, size_t len) + std::vector<rbm_alloc_delta_t>& v, rbm_abs_addr from, size_t len) { ceph_assert(!(len % super.block_size)); - paddr_t paddr = convert_blk_paddr_to_paddr( + paddr_t paddr = convert_abs_addr_to_paddr( from, - super.block_size, - super.blocks_per_segment, super.device_id); rbm_alloc_delta_t alloc_info; alloc_info.alloc_blk_ranges.push_back(std::make_pair(paddr, len)); @@ -321,7 +308,7 @@ void NVMeManager::add_free_extent( } NVMeManager::write_ertr::future<> NVMeManager::rbm_sync_block_bitmap_by_range( - blk_id_t start, blk_id_t end, bitmap_op_types_t op) + blk_no_t start, blk_no_t end, bitmap_op_types_t op) { auto addr = super.start_alloc_area + (start / max_block_by_bitmap_block()) @@ -455,12 +442,10 @@ NVMeManager::write_ertr::future<> NVMeManager::sync_allocation( (alloc.op == rbm_alloc_delta_t::op_types_t::SET) ? bitmap_op_types_t::ALL_SET : bitmap_op_types_t::ALL_CLEAR; - blk_paddr_t addr = convert_paddr_to_blk_paddr( - range.first, - super.block_size, - super.blocks_per_segment); - blk_id_t start = addr / super.block_size; - blk_id_t end = start + + rbm_abs_addr addr = convert_paddr_to_abs_addr( + range.first); + blk_no_t start = addr / super.block_size; + blk_no_t end = start + (round_up_to(range.second, super.block_size)) / super.block_size - 1; return rbm_sync_block_bitmap_by_range( @@ -498,10 +483,7 @@ NVMeManager::open_ertr::future<> NVMeManager::open( { logger().debug("open: path{}", path); - blk_paddr_t addr = convert_paddr_to_blk_paddr( - paddr, - super.block_size, - super.blocks_per_segment); + rbm_abs_addr addr = convert_paddr_to_abs_addr(paddr); return _open_device(path ).safe_then([this, addr]() { return read_rbm_header(addr).safe_then([&](auto s) @@ -524,10 +506,11 @@ NVMeManager::open_ertr::future<> NVMeManager::open( } NVMeManager::write_ertr::future<> NVMeManager::write( - blk_paddr_t addr, + paddr_t paddr, bufferptr &bptr) { ceph_assert(device); + rbm_abs_addr addr = convert_paddr_to_abs_addr(paddr); if (addr > super.end || addr < super.start || bptr.length() > super.end - super.start) { return crimson::ct_error::erange::make(); @@ -538,10 +521,11 @@ NVMeManager::write_ertr::future<> NVMeManager::write( } NVMeManager::read_ertr::future<> NVMeManager::read( - blk_paddr_t addr, + paddr_t paddr, bufferptr &bptr) { ceph_assert(device); + rbm_abs_addr addr = convert_paddr_to_abs_addr(paddr); if (addr > super.end || addr < super.start || bptr.length() > super.end - super.start) { return crimson::ct_error::erange::make(); @@ -590,7 +574,7 @@ NVMeManager::write_ertr::future<> NVMeManager::write_rbm_header() } NVMeManager::read_ertr::future<rbm_metadata_header_t> NVMeManager::read_rbm_header( - blk_paddr_t addr) + rbm_abs_addr addr) { ceph_assert(device); bufferptr bptr = @@ -679,7 +663,7 @@ NVMeManager::check_bitmap_blocks_ertr::future<> NVMeManager::check_bitmap_blocks } NVMeManager::write_ertr::future<> NVMeManager::write( - blk_paddr_t addr, + rbm_abs_addr addr, bufferlist &bl) { ceph_assert(device); diff --git a/src/crimson/os/seastore/random_block_manager/nvme_manager.h b/src/crimson/os/seastore/random_block_manager/nvme_manager.h index fa94cfea272..cf865513df6 100644 --- a/src/crimson/os/seastore/random_block_manager/nvme_manager.h +++ b/src/crimson/os/seastore/random_block_manager/nvme_manager.h @@ -54,7 +54,6 @@ struct rbm_metadata_header_t { uint32_t start_data_area; uint64_t flag; // reserved uint64_t feature; - uint32_t blocks_per_segment; // the number of blocks in segment device_id_t device_id; checksum_t crc; @@ -72,7 +71,6 @@ struct rbm_metadata_header_t { denc(v.start_data_area, p); denc(v.flag, p); denc(v.feature, p); - denc(v.blocks_per_segment, p); denc(v.device_id, p); denc(v.crc, p); @@ -186,8 +184,8 @@ public: */ mkfs_ertr::future<> mkfs(mkfs_config_t) final; - read_ertr::future<> read(uint64_t addr, bufferptr &buffer) final; - write_ertr::future<> write(uint64_t addr, bufferptr &buf) final; + read_ertr::future<> read(paddr_t addr, bufferptr &buffer) final; + write_ertr::future<> write(paddr_t addr, bufferptr &buf) final; open_ertr::future<> open(const std::string &path, paddr_t start) final; close_ertr::future<> close() final; @@ -218,7 +216,7 @@ public: write_ertr::future<> complete_allocation(Transaction &t) final; open_ertr::future<> _open_device(const std::string path); - read_ertr::future<rbm_metadata_header_t> read_rbm_header(blk_paddr_t addr); + read_ertr::future<rbm_metadata_header_t> read_rbm_header(rbm_abs_addr addr); write_ertr::future<> write_rbm_header(); size_t get_size() const final { return super.size; }; @@ -229,7 +227,7 @@ public: return (super.block_size - ceph::encoded_sizeof_bounded<rbm_bitmap_block_t>()) * 8; } - uint64_t convert_block_no_to_bitmap_block(blk_id_t block_no) + uint64_t convert_block_no_to_bitmap_block(blk_no_t block_no) { ceph_assert(super.block_size); return block_no / max_block_by_bitmap_block(); @@ -240,7 +238,7 @@ public: * * return block id using address where freebitmap is stored and offset */ - blk_id_t convert_bitmap_block_no_to_block_id(uint64_t offset, blk_paddr_t addr) + blk_no_t convert_bitmap_block_no_to_block_id(uint64_t offset, rbm_abs_addr addr) { ceph_assert(super.block_size); // freebitmap begins at block 1 @@ -261,7 +259,7 @@ public: using find_block_ertr = crimson::errorator< crimson::ct_error::input_output_error, crimson::ct_error::enoent>; - using find_block_ret = find_block_ertr::future<interval_set<blk_id_t>>; + using find_block_ret = find_block_ertr::future<interval_set<blk_no_t>>; /* * find_free_block * @@ -282,7 +280,7 @@ public: * */ write_ertr::future<> rbm_sync_block_bitmap( - rbm_bitmap_block_t &block, blk_id_t block_no); + rbm_bitmap_block_t &block, blk_no_t block_no); using check_bitmap_blocks_ertr = crimson::errorator< crimson::ct_error::input_output_error, @@ -328,11 +326,11 @@ public: b_block.buf.append(bitmap_blk); } - blk_paddr_t get_blk_paddr_by_block_no(blk_id_t id) { + rbm_abs_addr get_blk_paddr_by_block_no(blk_no_t id) { return (id * super.block_size) + super.start; } - int num_block_between_blk_ids(blk_id_t start, blk_id_t end) { + int num_block_between_blk_ids(blk_no_t start, blk_no_t end) { auto max = max_block_by_bitmap_block(); auto block_start = start / max; auto block_end = end / max; @@ -340,7 +338,7 @@ public: } write_ertr::future<> rbm_sync_block_bitmap_by_range( - blk_id_t start, blk_id_t end, bitmap_op_types_t op); + blk_no_t start, blk_no_t end, bitmap_op_types_t op); void add_cont_bitmap_blocks_to_buf( bufferlist& buf, int num_block, bitmap_op_types_t op) { rbm_bitmap_block_t b_block(super.block_size); @@ -355,15 +353,12 @@ public: } } - write_ertr::future<> write(blk_paddr_t addr, bufferlist &bl); + write_ertr::future<> write(rbm_abs_addr addr, bufferlist &bl); write_ertr::future<> sync_allocation( std::vector<rbm_alloc_delta_t>& alloc_blocks); void add_free_extent( - std::vector<rbm_alloc_delta_t>& v, blk_paddr_t from, size_t len); + std::vector<rbm_alloc_delta_t>& v, rbm_abs_addr from, size_t len); - uint32_t get_blocks_per_segment() const final { - return super.blocks_per_segment; - } device_id_t get_device_id() const final { return super.device_id; } diff --git a/src/crimson/os/seastore/root_block.h b/src/crimson/os/seastore/root_block.h index 3144edb8251..e0484c1cb05 100644 --- a/src/crimson/os/seastore/root_block.h +++ b/src/crimson/os/seastore/root_block.h @@ -33,7 +33,7 @@ namespace crimson::os::seastore { * mutation which changes the journal trim bound. */ struct RootBlock : CachedExtent { - constexpr static segment_off_t SIZE = 4<<10; + constexpr static seastore_off_t SIZE = 4<<10; using Ref = TCachedExtentRef<RootBlock>; root_t root; diff --git a/src/crimson/os/seastore/seastore_types.cc b/src/crimson/os/seastore/seastore_types.cc index 8befb9f1eed..f738870544c 100644 --- a/src/crimson/os/seastore/seastore_types.cc +++ b/src/crimson/os/seastore/seastore_types.cc @@ -29,7 +29,7 @@ std::ostream &segment_to_stream(std::ostream &out, const segment_id_t &t) return out << t; } -std::ostream &offset_to_stream(std::ostream &out, const segment_off_t &t) +std::ostream &offset_to_stream(std::ostream &out, const seastore_off_t &t) { if (t == NULL_SEG_OFF) return out << "NULL_OFF"; @@ -43,6 +43,11 @@ std::ostream &operator<<(std::ostream &out, const segment_id_t& segment) << segment.device_segment_id() << "]"; } +std::ostream &block_offset_to_stream(std::ostream &out, const block_off_t &t) +{ + return out << t; +} + std::ostream &operator<<(std::ostream &out, const paddr_t &rhs) { out << "paddr_t<"; @@ -61,6 +66,9 @@ std::ostream &operator<<(std::ostream &out, const paddr_t &rhs) segment_to_stream(out, s.get_segment_id()); out << ", "; offset_to_stream(out, s.get_segment_off()); + } else if (rhs.get_addr_type() == addr_types_t::RANDOM_BLOCK) { + const blk_paddr_t& s = rhs.as_blk_paddr(); + block_offset_to_stream(out, s.get_block_off()); } else { out << "INVALID"; } @@ -539,25 +547,6 @@ std::ostream& operator<<(std::ostream& out, device_type_t t) } } -paddr_t convert_blk_paddr_to_paddr(blk_paddr_t addr, size_t block_size, - uint32_t blocks_per_segment, device_id_t d_id) -{ - segment_id_t id = segment_id_t { - d_id, - (device_segment_id_t)(addr / (block_size * blocks_per_segment)) - }; - segment_off_t off = addr % (block_size * blocks_per_segment); - return paddr_t::make_seg_paddr(id, off); -} - -blk_paddr_t convert_paddr_to_blk_paddr(paddr_t addr, size_t block_size, - uint32_t blocks_per_segment) -{ - seg_paddr_t& s = addr.as_seg_paddr(); - return (blk_paddr_t)(s.get_segment_id().device_segment_id() * - (block_size * blocks_per_segment) + s.get_segment_off()); -} - std::ostream& operator<<(std::ostream& out, const write_result_t& w) { return out << "write_result_t(" diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h index 29ea8159d76..db7c907a29a 100644 --- a/src/crimson/os/seastore/seastore_types.h +++ b/src/crimson/os/seastore/seastore_types.h @@ -78,7 +78,7 @@ private: // mask for segment manager id static constexpr internal_segment_id_t SM_ID_MASK = - 0xF << (std::numeric_limits<internal_segment_id_t>::digits - DEVICE_ID_LEN_BITS); + 0xFF << (std::numeric_limits<internal_segment_id_t>::digits - DEVICE_ID_LEN_BITS); // default internal segment id static constexpr internal_segment_id_t DEFAULT_INTERNAL_SEG_ID = (std::numeric_limits<internal_segment_id_t>::max() >> 1) - 1; @@ -178,13 +178,13 @@ std::ostream &segment_to_stream(std::ostream &, const segment_id_t &t); // Offset within a segment on disk, see SegmentManager // may be negative for relative offsets -using segment_off_t = int32_t; -constexpr segment_off_t NULL_SEG_OFF = - std::numeric_limits<segment_off_t>::max(); -constexpr segment_off_t MAX_SEG_OFF = - std::numeric_limits<segment_off_t>::max(); +using seastore_off_t = int32_t; +constexpr seastore_off_t NULL_SEG_OFF = + std::numeric_limits<seastore_off_t>::max(); +constexpr seastore_off_t MAX_SEG_OFF = + std::numeric_limits<seastore_off_t>::max(); -std::ostream &offset_to_stream(std::ostream &, const segment_off_t &t); +std::ostream &offset_to_stream(std::ostream &, const seastore_off_t &t); /* Monotonically increasing segment seq, uniquely identifies * the incarnation of a segment */ @@ -390,6 +390,7 @@ private: size_t total_segments = 0; }; +using block_off_t = uint64_t; /** * paddr_t * @@ -414,35 +415,48 @@ enum class addr_types_t : uint8_t { RANDOM_BLOCK = 1 }; struct seg_paddr_t; +struct blk_paddr_t; struct paddr_t { protected: using common_addr_t = uint64_t; common_addr_t dev_addr; private: - constexpr paddr_t(segment_id_t seg, segment_off_t offset) + constexpr paddr_t(segment_id_t seg, seastore_off_t offset) : dev_addr((static_cast<common_addr_t>(seg.segment) << SEG_OFF_LEN_BITS) | static_cast<uint32_t>(offset)) {} constexpr paddr_t(common_addr_t val) : dev_addr(val) {} + constexpr paddr_t(device_id_t d_id, block_off_t offset) + : dev_addr( + (static_cast<common_addr_t>(d_id) << + (std::numeric_limits<block_off_t>::digits - DEVICE_ID_LEN_BITS)) | + (offset & (std::numeric_limits<block_off_t>::max() >> DEVICE_ID_LEN_BITS))) + {} public: static constexpr paddr_t make_seg_paddr( - segment_id_t seg, segment_off_t offset) { + segment_id_t seg, seastore_off_t offset) { return paddr_t(seg, offset); } static constexpr paddr_t make_seg_paddr( device_id_t device, device_segment_id_t seg, - segment_off_t offset) { + seastore_off_t offset) { return paddr_t(segment_id_t(device, seg), offset); } constexpr paddr_t() : paddr_t(NULL_SEG_ID, 0) {} + static constexpr paddr_t make_blk_paddr( + device_id_t device, + block_off_t offset) { + return paddr_t(device, offset); + } // use 1bit in device_id_t for address type void set_device_id(device_id_t id, addr_types_t type = addr_types_t::SEGMENT) { dev_addr &= static_cast<common_addr_t>( - std::numeric_limits<device_segment_id_t>::max()); - dev_addr |= static_cast<common_addr_t>(id & 0x8) << DEV_ADDR_LEN_BITS; - dev_addr |= static_cast<common_addr_t>(type) - << (std::numeric_limits<common_addr_t>::digits - 1); + std::numeric_limits<common_addr_t>::max() >> DEVICE_ID_LEN_BITS); + dev_addr |= (static_cast<common_addr_t>(id & + std::numeric_limits<device_id_t>::max() >> 1) << DEV_ADDR_LEN_BITS); + dev_addr |= (static_cast<common_addr_t>(type) + << (std::numeric_limits<common_addr_t>::digits - 1)); } device_id_t get_device_id() const { @@ -461,6 +475,8 @@ public: seg_paddr_t& as_seg_paddr(); const seg_paddr_t& as_seg_paddr() const; + blk_paddr_t& as_blk_paddr(); + const blk_paddr_t& as_blk_paddr() const; paddr_t operator-(paddr_t rhs) const; @@ -524,21 +540,21 @@ struct seg_paddr_t : public paddr_t { segment_id_t get_segment_id() const { return segment_id_t((dev_addr & SEG_ID_MASK) >> SEG_OFF_LEN_BITS); } - segment_off_t get_segment_off() const { - return segment_off_t(dev_addr & SEG_OFF_MASK); + seastore_off_t get_segment_off() const { + return seastore_off_t(dev_addr & SEG_OFF_MASK); } void set_segment_id(const segment_id_t id) { dev_addr &= static_cast<common_addr_t>( std::numeric_limits<device_segment_id_t>::max()); dev_addr |= static_cast<common_addr_t>(id.segment) << SEG_OFF_LEN_BITS; } - void set_segment_off(const segment_off_t off) { + void set_segment_off(const seastore_off_t off) { dev_addr &= static_cast<common_addr_t>( std::numeric_limits<device_segment_id_t>::max()) << SEG_OFF_LEN_BITS; dev_addr |= (uint32_t)off; } - paddr_t add_offset(segment_off_t o) const { + paddr_t add_offset(seastore_off_t o) const { return paddr_t::make_seg_paddr(get_segment_id(), get_segment_off() + o); } @@ -594,28 +610,62 @@ struct seg_paddr_t : public paddr_t { return *this; } }; + +constexpr block_off_t BLK_OFF_MAX = + std::numeric_limits<block_off_t>::max() >> DEVICE_ID_LEN_BITS; +struct blk_paddr_t : public paddr_t { + + blk_paddr_t(const blk_paddr_t&) = delete; + blk_paddr_t(blk_paddr_t&) = delete; + blk_paddr_t& operator=(const blk_paddr_t&) = delete; + blk_paddr_t& operator=(blk_paddr_t&) = delete; + + static constexpr uint64_t BLK_OFF_MASK = std::numeric_limits<uint64_t>::max() + >> DEVICE_ID_LEN_BITS; + void set_block_off(const block_off_t off) { + check_blk_off_valid(off); + uint64_t val = off & BLK_OFF_MASK; + dev_addr |= val; + } + block_off_t get_block_off() const { + const block_off_t ret = static_cast<block_off_t>( + dev_addr & BLK_OFF_MASK); + check_blk_off_valid(ret); + return ret; + } + + paddr_t add_offset(seastore_off_t o) const { + return paddr_t::make_blk_paddr(get_device_id(), get_block_off() + o); + } + +private: + void check_blk_off_valid(const block_off_t offset) const { + assert(offset <= BLK_OFF_MAX); + } +}; + constexpr paddr_t P_ADDR_NULL = paddr_t{}; constexpr paddr_t P_ADDR_MIN = paddr_t::make_seg_paddr(segment_id_t(0, 0), 0); constexpr paddr_t P_ADDR_MAX = paddr_t::make_seg_paddr( segment_id_t(DEVICE_ID_MAX, DEVICE_SEGMENT_ID_MAX), - std::numeric_limits<segment_off_t>::max()); + std::numeric_limits<seastore_off_t>::max()); constexpr paddr_t P_ADDR_ZERO = paddr_t::make_seg_paddr( DEVICE_ID_ZERO, 0, 0); -constexpr paddr_t make_record_relative_paddr(segment_off_t off) { +constexpr paddr_t make_record_relative_paddr(seastore_off_t off) { return paddr_t::make_seg_paddr( segment_id_t{DEVICE_ID_RECORD_RELATIVE, 0}, off); } -constexpr paddr_t make_block_relative_paddr(segment_off_t off) { +constexpr paddr_t make_block_relative_paddr(seastore_off_t off) { return paddr_t::make_seg_paddr( segment_id_t{DEVICE_ID_BLOCK_RELATIVE, 0}, off); } -constexpr paddr_t make_fake_paddr(segment_off_t off) { +constexpr paddr_t make_fake_paddr(seastore_off_t off) { return paddr_t::make_seg_paddr(FAKE_SEG_ID, off); } -constexpr paddr_t delayed_temp_paddr(segment_off_t off) { +constexpr paddr_t delayed_temp_paddr(seastore_off_t off) { return paddr_t::make_seg_paddr( segment_id_t{DEVICE_ID_DELAYED, 0}, off); @@ -666,7 +716,7 @@ struct journal_seq_t { segment_seq_t segment_seq = 0; paddr_t offset; - journal_seq_t add_offset(segment_off_t o) const { + journal_seq_t add_offset(seastore_off_t o) const { return {segment_seq, offset.add_offset(o)}; } @@ -807,7 +857,7 @@ struct delta_info_t { laddr_t laddr = L_ADDR_NULL; ///< logical address uint32_t prev_crc = 0; uint32_t final_crc = 0; - segment_off_t length = NULL_SEG_OFF; ///< extent length + seastore_off_t length = NULL_SEG_OFF; ///< extent length extent_version_t pversion; ///< prior version ceph::bufferlist bl; ///< payload @@ -1066,7 +1116,7 @@ class __attribute__((packed)) coll_root_le_t { public: coll_root_le_t() = default; - coll_root_le_t(laddr_t laddr, segment_off_t size) + coll_root_le_t(laddr_t laddr, seastore_off_t size) : addr(laddr), size(init_extent_len_le(size)) {} @@ -1131,12 +1181,7 @@ struct __attribute__((packed)) root_t { } }; -using blk_id_t = uint64_t; -constexpr blk_id_t NULL_BLK_ID = - std::numeric_limits<blk_id_t>::max(); - // use absolute address -using blk_paddr_t = uint64_t; struct rbm_alloc_delta_t { enum class op_types_t : uint8_t { NONE = 0, @@ -1156,11 +1201,6 @@ struct rbm_alloc_delta_t { } }; -paddr_t convert_blk_paddr_to_paddr(blk_paddr_t addr, size_t block_size, - uint32_t blocks_per_segment, device_id_t d_id); -blk_paddr_t convert_paddr_to_blk_paddr(paddr_t addr, size_t block_size, - uint32_t blocks_per_segment); - struct extent_info_t { extent_types_t type = extent_types_t::NONE; laddr_t addr = L_ADDR_NULL; @@ -1434,7 +1474,7 @@ try_decode_deltas( struct write_result_t { journal_seq_t start_seq; - segment_off_t length; + seastore_off_t length; journal_seq_t get_end_seq() const { return start_seq.add_offset(length); @@ -1476,11 +1516,11 @@ struct scan_valid_records_cursor { return seq.offset.as_seg_paddr().get_segment_id(); } - segment_off_t get_segment_offset() const { + seastore_off_t get_segment_offset() const { return seq.offset.as_seg_paddr().get_segment_off(); } - void increment_seq(segment_off_t off) { + void increment_seq(seastore_off_t off) { auto& seg_addr = seq.offset.as_seg_paddr(); seg_addr.set_segment_off( seg_addr.get_segment_off() + off); @@ -1510,6 +1550,16 @@ inline seg_paddr_t& paddr_t::as_seg_paddr() { return *static_cast<seg_paddr_t*>(this); } +inline const blk_paddr_t& paddr_t::as_blk_paddr() const { + assert(get_addr_type() == addr_types_t::RANDOM_BLOCK); + return *static_cast<const blk_paddr_t*>(this); +} + +inline blk_paddr_t& paddr_t::as_blk_paddr() { + assert(get_addr_type() == addr_types_t::RANDOM_BLOCK); + return *static_cast<blk_paddr_t*>(this); +} + inline paddr_t paddr_t::operator-(paddr_t rhs) const { if (get_addr_type() == addr_types_t::SEGMENT) { auto& seg_addr = as_seg_paddr(); @@ -1526,6 +1576,7 @@ inline paddr_t paddr_t::operator-(paddr_t rhs) const { inline paddr_t paddr_t::add_offset(int32_t o) const { PADDR_OPERATION(addr_types_t::SEGMENT, seg_paddr_t, add_offset(o)) + PADDR_OPERATION(addr_types_t::RANDOM_BLOCK, blk_paddr_t, add_offset(o)) ceph_assert(0 == "not supported type"); return paddr_t{}; } diff --git a/src/crimson/os/seastore/segment_cleaner.cc b/src/crimson/os/seastore/segment_cleaner.cc index d296a958562..42ca4d0a64a 100644 --- a/src/crimson/os/seastore/segment_cleaner.cc +++ b/src/crimson/os/seastore/segment_cleaner.cc @@ -59,7 +59,7 @@ bool SpaceTrackerSimple::equals(const SpaceTrackerI &_other) const int64_t SpaceTrackerDetailed::SegmentMap::allocate( device_segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len, const extent_len_t block_size) { @@ -91,7 +91,7 @@ int64_t SpaceTrackerDetailed::SegmentMap::allocate( int64_t SpaceTrackerDetailed::SegmentMap::release( device_segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len, const extent_len_t block_size) { diff --git a/src/crimson/os/seastore/segment_cleaner.h b/src/crimson/os/seastore/segment_cleaner.h index f958cd31826..accd3baece1 100644 --- a/src/crimson/os/seastore/segment_cleaner.h +++ b/src/crimson/os/seastore/segment_cleaner.h @@ -28,8 +28,8 @@ class segment_info_set_t { segment_manager_info_t( device_id_t device_id, device_segment_id_t num_segments, - segment_off_t segment_size, - segment_off_t block_size, + seastore_off_t segment_size, + seastore_off_t block_size, size_t empty_segments, size_t size) : device_id(device_id), @@ -43,12 +43,12 @@ class segment_info_set_t { device_id_t device_id = 0; device_segment_id_t num_segments = 0; - segment_off_t segment_size = 0; - segment_off_t block_size = 0; + seastore_off_t segment_size = 0; + seastore_off_t block_size = 0; size_t empty_segments = 0; size_t size = 0; size_t avail_bytes = 0; - std::map<segment_id_t, segment_off_t> open_segment_avails; + std::map<segment_id_t, seastore_off_t> open_segment_avails; }; struct segment_info_t { @@ -301,12 +301,12 @@ class SpaceTrackerI { public: virtual int64_t allocate( segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len) = 0; virtual int64_t release( segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len) = 0; virtual int64_t get_usage( @@ -352,14 +352,14 @@ public: int64_t allocate( segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len) final { return update_usage(segment, len); } int64_t release( segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len) final { return update_usage(segment, -(int64_t)len); } @@ -400,13 +400,13 @@ class SpaceTrackerDetailed : public SpaceTrackerI { int64_t allocate( device_segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len, const extent_len_t block_size); int64_t release( device_segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len, const extent_len_t block_size); @@ -451,7 +451,7 @@ public: int64_t allocate( segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len) final { return segment_usage[segment].allocate( segment.device_segment_id(), @@ -462,7 +462,7 @@ public: int64_t release( segment_id_t segment, - segment_off_t offset, + seastore_off_t offset, extent_len_t len) final { return segment_usage[segment].release( segment.device_segment_id(), @@ -605,7 +605,7 @@ public: extent_types_t type, paddr_t addr, laddr_t laddr, - segment_off_t len) = 0; + seastore_off_t len) = 0; /** * release_segment diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h index 5a7c48a00a5..c0c421e5fd6 100644 --- a/src/crimson/os/seastore/segment_manager.h +++ b/src/crimson/os/seastore/segment_manager.h @@ -131,12 +131,12 @@ public: /** * min next write location */ - virtual segment_off_t get_write_ptr() const = 0; + virtual seastore_off_t get_write_ptr() const = 0; /** * max capacity */ - virtual segment_off_t get_write_capacity() const = 0; + virtual seastore_off_t get_write_capacity() const = 0; /** * close @@ -165,7 +165,7 @@ public: crimson::ct_error::enospc // write exceeds segment size >; virtual write_ertr::future<> write( - segment_off_t offset, ceph::bufferlist bl) = 0; + seastore_off_t offset, ceph::bufferlist bl) = 0; virtual ~Segment() {} }; @@ -232,8 +232,8 @@ public: /* Methods for discovering device geometry, segmentid set, etc */ virtual size_t get_size() const = 0; - virtual segment_off_t get_block_size() const = 0; - virtual segment_off_t get_segment_size() const = 0; + virtual seastore_off_t get_block_size() const = 0; + virtual seastore_off_t get_segment_size() const = 0; virtual device_segment_id_t get_num_segments() const { ceph_assert(get_size() % get_segment_size() == 0); return ((device_segment_id_t)(get_size() / get_segment_size())); diff --git a/src/crimson/os/seastore/segment_manager/block.cc b/src/crimson/os/seastore/segment_manager/block.cc index f27942d5d3e..aac9376cc7f 100644 --- a/src/crimson/os/seastore/segment_manager/block.cc +++ b/src/crimson/os/seastore/segment_manager/block.cc @@ -360,7 +360,7 @@ BlockSegment::BlockSegment( BlockSegmentManager &manager, segment_id_t id) : manager(manager), id(id) {} -segment_off_t BlockSegment::get_write_capacity() const +seastore_off_t BlockSegment::get_write_capacity() const { return manager.get_segment_size(); } @@ -371,7 +371,7 @@ Segment::close_ertr::future<> BlockSegment::close() } Segment::write_ertr::future<> BlockSegment::write( - segment_off_t offset, ceph::bufferlist bl) + seastore_off_t offset, ceph::bufferlist bl) { LOG_PREFIX(BlockSegment::write); auto paddr = paddr_t::make_seg_paddr(id, offset); @@ -410,7 +410,7 @@ Segment::write_ertr::future<> BlockSegment::write( } Segment::close_ertr::future<> BlockSegmentManager::segment_close( - segment_id_t id, segment_off_t write_pointer) + segment_id_t id, seastore_off_t write_pointer) { LOG_PREFIX(BlockSegmentManager::segment_close); auto s_id = id.device_segment_id(); diff --git a/src/crimson/os/seastore/segment_manager/block.h b/src/crimson/os/seastore/segment_manager/block.h index 7dc7d99141b..8008c4bd4e3 100644 --- a/src/crimson/os/seastore/segment_manager/block.h +++ b/src/crimson/os/seastore/segment_manager/block.h @@ -88,15 +88,15 @@ class BlockSegment final : public Segment { friend class BlockSegmentManager; BlockSegmentManager &manager; const segment_id_t id; - segment_off_t write_pointer = 0; + seastore_off_t write_pointer = 0; public: BlockSegment(BlockSegmentManager &manager, segment_id_t id); segment_id_t get_segment_id() const final { return id; } - segment_off_t get_write_capacity() const final; - segment_off_t get_write_ptr() const final { return write_pointer; } + seastore_off_t get_write_capacity() const final; + seastore_off_t get_write_ptr() const final { return write_pointer; } close_ertr::future<> close() final; - write_ertr::future<> write(segment_off_t offset, ceph::bufferlist bl) final; + write_ertr::future<> write(seastore_off_t offset, ceph::bufferlist bl) final; ~BlockSegment() {} }; @@ -134,10 +134,10 @@ public: size_t get_size() const final { return superblock.size; } - segment_off_t get_block_size() const { + seastore_off_t get_block_size() const { return superblock.block_size; } - segment_off_t get_segment_size() const { + seastore_off_t get_segment_size() const { return superblock.segment_size; } @@ -230,7 +230,7 @@ private: char *buffer = nullptr; Segment::close_ertr::future<> segment_close( - segment_id_t id, segment_off_t write_pointer); + segment_id_t id, seastore_off_t write_pointer); }; } diff --git a/src/crimson/os/seastore/segment_manager/ephemeral.cc b/src/crimson/os/seastore/segment_manager/ephemeral.cc index 4aff7a3e34c..2123e3afb2c 100644 --- a/src/crimson/os/seastore/segment_manager/ephemeral.cc +++ b/src/crimson/os/seastore/segment_manager/ephemeral.cc @@ -33,7 +33,7 @@ EphemeralSegment::EphemeralSegment( EphemeralSegmentManager &manager, segment_id_t id) : manager(manager), id(id) {} -segment_off_t EphemeralSegment::get_write_capacity() const +seastore_off_t EphemeralSegment::get_write_capacity() const { return manager.get_segment_size(); } @@ -46,7 +46,7 @@ Segment::close_ertr::future<> EphemeralSegment::close() } Segment::write_ertr::future<> EphemeralSegment::write( - segment_off_t offset, ceph::bufferlist bl) + seastore_off_t offset, ceph::bufferlist bl) { if (offset < write_pointer || offset % manager.config.block_size != 0) return crimson::ct_error::invarg::make(); diff --git a/src/crimson/os/seastore/segment_manager/ephemeral.h b/src/crimson/os/seastore/segment_manager/ephemeral.h index 7083c08c89b..d9c786815ff 100644 --- a/src/crimson/os/seastore/segment_manager/ephemeral.h +++ b/src/crimson/os/seastore/segment_manager/ephemeral.h @@ -41,15 +41,15 @@ class EphemeralSegment final : public Segment { friend class EphemeralSegmentManager; EphemeralSegmentManager &manager; const segment_id_t id; - segment_off_t write_pointer = 0; + seastore_off_t write_pointer = 0; public: EphemeralSegment(EphemeralSegmentManager &manager, segment_id_t id); segment_id_t get_segment_id() const final { return id; } - segment_off_t get_write_capacity() const final; - segment_off_t get_write_ptr() const final { return write_pointer; } + seastore_off_t get_write_capacity() const final; + seastore_off_t get_write_ptr() const final { return write_pointer; } close_ertr::future<> close() final; - write_ertr::future<> write(segment_off_t offset, ceph::bufferlist bl) final; + write_ertr::future<> write(seastore_off_t offset, ceph::bufferlist bl) final; ~EphemeralSegment() {} }; @@ -115,10 +115,10 @@ public: size_t get_size() const final { return config.size; } - segment_off_t get_block_size() const final { + seastore_off_t get_block_size() const final { return config.block_size; } - segment_off_t get_segment_size() const final { + seastore_off_t get_segment_size() const final { return config.segment_size; } diff --git a/src/crimson/os/seastore/transaction.h b/src/crimson/os/seastore/transaction.h index fd14acbd515..92971b44df6 100644 --- a/src/crimson/os/seastore/transaction.h +++ b/src/crimson/os/seastore/transaction.h @@ -352,8 +352,8 @@ private: RootBlockRef root; ///< ref to root if read or written by transaction - segment_off_t offset = 0; ///< relative offset of next block - segment_off_t delayed_temp_offset = 0; + seastore_off_t offset = 0; ///< relative offset of next block + seastore_off_t delayed_temp_offset = 0; /** * read_set diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index c9dc00e4ead..3dfb8977af6 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -418,7 +418,7 @@ TransactionManager::get_extent_if_live_ret TransactionManager::get_extent_if_liv extent_types_t type, paddr_t addr, laddr_t laddr, - segment_off_t len) + seastore_off_t len) { LOG_PREFIX(TransactionManager::get_extent_if_live); DEBUGT("type {}, addr {}, laddr {}, len {}", t, type, addr, laddr, len); diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 82c825c284c..82d08ea40b6 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -400,7 +400,7 @@ public: extent_types_t type, paddr_t addr, laddr_t laddr, - segment_off_t len) final; + seastore_off_t len) final; using release_segment_ret = SegmentCleaner::ExtentCallbackInterface::release_segment_ret; diff --git a/src/mgr/CMakeLists.txt b/src/mgr/CMakeLists.txt index a53d1d8c2a0..b11c40c42bc 100644 --- a/src/mgr/CMakeLists.txt +++ b/src/mgr/CMakeLists.txt @@ -34,6 +34,7 @@ if(WITH_MGR) mgr_commands.cc $<TARGET_OBJECTS:mgr_cap_obj>) add_executable(ceph-mgr ${mgr_srcs}) + target_compile_definitions(ceph-mgr PRIVATE PY_SSIZE_T_CLEAN) if(WITH_LIBCEPHSQLITE) target_link_libraries(ceph-mgr cephsqlite SQLite3::SQLite3) endif() diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index 2b1b5cba3c1..86fa755bc5a 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -106,7 +106,6 @@ const static std::map<uint32_t, std::set<std::string>> always_on_modules = { "progress", "balancer", "devicehealth", - "feedback", "orchestrator", "rbd_support", "volumes", @@ -462,7 +461,7 @@ public: mm(a), op(c) {} void finish(int r) override { if (r >= 0) { - // Success + // Success } else if (r == -ECANCELED) { mm->mon.no_reply(op); } else { diff --git a/src/pybind/mgr/dashboard/controllers/feedback.py b/src/pybind/mgr/dashboard/controllers/feedback.py index 3b3474d39ea..c75ffa94a89 100644 --- a/src/pybind/mgr/dashboard/controllers/feedback.py +++ b/src/pybind/mgr/dashboard/controllers/feedback.py @@ -8,7 +8,7 @@ from ._version import APIVersion @APIRouter('/feedback', Scope.CONFIG_OPT) -@APIDoc("Feedback", "Report") +@APIDoc("Feedback API", "Report") class FeedbackController(RESTController): @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL) @@ -52,7 +52,7 @@ class FeedbackController(RESTController): @APIRouter('/feedback/api_key', Scope.CONFIG_OPT) -@APIDoc("Feedback API", "Report") +@APIDoc(group="Report") class FeedbackApiController(RESTController): @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL) diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml index bc4292876a9..c05831eb77f 100644 --- a/src/pybind/mgr/dashboard/openapi.yaml +++ b/src/pybind/mgr/dashboard/openapi.yaml @@ -10517,7 +10517,7 @@ tags: name: RbdSnapshot - description: RBD Trash Management API name: RbdTrash -- description: Feedback +- description: Feedback API name: Report - description: RGW Management API name: Rgw diff --git a/src/pybind/mgr/volumes/fs/async_cloner.py b/src/pybind/mgr/volumes/fs/async_cloner.py index 6bdb5fd72be..68ebb9e263a 100644 --- a/src/pybind/mgr/volumes/fs/async_cloner.py +++ b/src/pybind/mgr/volumes/fs/async_cloner.py @@ -4,6 +4,7 @@ import time import errno import logging from contextlib import contextmanager +from typing import Dict, Union import cephfs from mgr_util import lock_timeout_log @@ -185,12 +186,35 @@ def bulk_copy(fs_handle, source_path, dst_path, should_cancel): if should_cancel(): raise VolumeException(-errno.EINTR, "clone operation interrupted") +def set_quota_on_clone(fs_handle, clone_volumes_pair): + attrs = {} # type: Dict[str, Union[int, str, None]] + src_path = clone_volumes_pair[1].snapshot_data_path(clone_volumes_pair[2]) + dst_path = clone_volumes_pair[0].path + try: + attrs["quota"] = int(fs_handle.getxattr(src_path, + 'ceph.quota.max_bytes' + ).decode('utf-8')) + except cephfs.NoData: + attrs["quota"] = None + + if attrs["quota"] is not None: + clone_volumes_pair[0].set_attrs(dst_path, attrs) + def do_clone(fs_client, volspec, volname, groupname, subvolname, should_cancel): with open_volume_lockless(fs_client, volname) as fs_handle: with open_clone_subvolume_pair(fs_client, fs_handle, volspec, volname, groupname, subvolname) as clone_volumes: src_path = clone_volumes[1].snapshot_data_path(clone_volumes[2]) dst_path = clone_volumes[0].path bulk_copy(fs_handle, src_path, dst_path, should_cancel) + set_quota_on_clone(fs_handle, clone_volumes) + +def log_clone_failure(volname, groupname, subvolname, ve): + if ve.errno == -errno.EINTR: + log.info("Clone cancelled: ({0}, {1}, {2})".format(volname, groupname, subvolname)) + elif ve.errno == -errno.EDQUOT: + log.error("Clone failed: ({0}, {1}, {2}, reason -> Disk quota exceeded)".format(volname, groupname, subvolname)) + else: + log.error("Clone failed: ({0}, {1}, {2}, reason -> {3})".format(volname, groupname, subvolname, ve)) def handle_clone_in_progress(fs_client, volspec, volname, index, groupname, subvolname, should_cancel): try: @@ -199,6 +223,7 @@ def handle_clone_in_progress(fs_client, volspec, volname, index, groupname, subv SubvolumeStates.STATE_INPROGRESS, SubvolumeActions.ACTION_SUCCESS) except VolumeException as ve: + log_clone_failure(volname, groupname, subvolname, ve) next_state = get_next_state_on_error(ve.errno) except OpSmException as oe: raise VolumeException(oe.errno, oe.error_str) diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py index d62effd995b..42c08e04712 100644 --- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py +++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v1.py @@ -152,6 +152,13 @@ class SubvolumeV1(SubvolumeBase, SubvolumeTemplate): # attributes of subvolume's content though, are synced during the cloning process. attrs = source_subvolume.get_attrs(source_subvolume.snapshot_data_path(snapname)) + # The source of the clone may have exceeded its quota limit as + # CephFS quotas are imprecise. Cloning such a source may fail if + # the quota on the destination is set before starting the clone + # copy. So always set the quota on destination after cloning is + # successful. + attrs["quota"] = None + # override snapshot pool setting, if one is provided for the clone if pool is not None: attrs["data_pool"] = pool diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py index 3937daba61f..a827bb7a00e 100644 --- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py +++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_v2.py @@ -219,6 +219,13 @@ class SubvolumeV2(SubvolumeV1): # attributes of subvolume's content though, are synced during the cloning process. attrs = source_subvolume.get_attrs(source_subvolume.snapshot_data_path(snapname)) + # The source of the clone may have exceeded its quota limit as + # CephFS quotas are imprecise. Cloning such a source may fail if + # the quota on the destination is set before starting the clone + # copy. So always set the quota on destination after cloning is + # successful. + attrs["quota"] = None + # override snapshot pool setting, if one is provided for the clone if pool is not None: attrs["data_pool"] = pool diff --git a/src/test/crimson/seastore/test_block.h b/src/test/crimson/seastore/test_block.h index 4df99784ad4..c6f88b8cd88 100644 --- a/src/test/crimson/seastore/test_block.h +++ b/src/test/crimson/seastore/test_block.h @@ -44,7 +44,7 @@ inline std::ostream &operator<<( } struct TestBlock : crimson::os::seastore::LogicalCachedExtent { - constexpr static segment_off_t SIZE = 4<<10; + constexpr static seastore_off_t SIZE = 4<<10; using Ref = TCachedExtentRef<TestBlock>; std::vector<test_block_delta_t> delta = {}; @@ -83,7 +83,7 @@ struct TestBlock : crimson::os::seastore::LogicalCachedExtent { using TestBlockRef = TCachedExtentRef<TestBlock>; struct TestBlockPhysical : crimson::os::seastore::CachedExtent{ - constexpr static segment_off_t SIZE = 4<<10; + constexpr static seastore_off_t SIZE = 4<<10; using Ref = TCachedExtentRef<TestBlockPhysical>; std::vector<test_block_delta_t> delta = {}; diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 19c36edb492..f9d5d195057 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -347,7 +347,7 @@ struct btree_lba_manager_test : btree_test_base { ); } - segment_off_t next_off = 0; + seastore_off_t next_off = 0; paddr_t get_paddr() { next_off += block_size; return make_fake_paddr(next_off); diff --git a/src/test/crimson/seastore/test_randomblock_manager.cc b/src/test/crimson/seastore/test_randomblock_manager.cc index 02f266454da..f5a598cd389 100644 --- a/src/test/crimson/seastore/test_randomblock_manager.cc +++ b/src/test/crimson/seastore/test_randomblock_manager.cc @@ -55,10 +55,12 @@ struct rbm_test_t : seastar::future<> set_up_fut() final { device.reset(new nvme_device::TestMemory(DEFAULT_TEST_SIZE)); rbm_manager.reset(new NVMeManager(device.get(), std::string())); - config.start = paddr_t::make_seg_paddr(0, 0, 0); - config.end = paddr_t::make_seg_paddr(0, 0, DEFAULT_TEST_SIZE); + device_id_t d_id = 1 << (std::numeric_limits<device_id_t>::digits - 1); + config.start = paddr_t::make_blk_paddr(d_id, 0); + config.end = paddr_t::make_blk_paddr(d_id, DEFAULT_TEST_SIZE); config.block_size = DEFAULT_BLOCK_SIZE; config.total_size = DEFAULT_TEST_SIZE; + config.device_id = d_id; return tm_setup(); } @@ -73,10 +75,7 @@ struct rbm_test_t : } auto read_rbm_header() { - blk_paddr_t addr = convert_paddr_to_blk_paddr( - config.start, - config.block_size, - config.blocks_per_segment); + rbm_abs_addr addr = convert_paddr_to_abs_addr(config.start); return rbm_manager->read_rbm_header(addr).unsafe_get0(); } @@ -85,11 +84,17 @@ struct rbm_test_t : } auto write(uint64_t addr, bufferptr &ptr) { - return rbm_manager->write(addr, ptr).unsafe_get0(); + paddr_t paddr = convert_abs_addr_to_paddr( + addr, + rbm_manager->get_device_id()); + return rbm_manager->write(paddr, ptr).unsafe_get0(); } auto read(uint64_t addr, bufferptr &ptr) { - return rbm_manager->read(addr, ptr).unsafe_get0(); + paddr_t paddr = convert_abs_addr_to_paddr( + addr, + rbm_manager->get_device_id()); + return rbm_manager->read(paddr, ptr).unsafe_get0(); } auto create_rbm_transaction() { @@ -102,11 +107,9 @@ struct rbm_test_t : if (!extent.empty()) { rbm_alloc_delta_t alloc_info; for (auto p : extent) { - paddr_t paddr = convert_blk_paddr_to_paddr( + paddr_t paddr = convert_abs_addr_to_paddr( p.first * block_size, - block_size, - config.blocks_per_segment, - 0); + rbm_manager->get_device_id()); size_t len = p.second * block_size; alloc_info.alloc_blk_ranges.push_back(std::make_pair(paddr, len)); alloc_info.op = rbm_alloc_delta_t::op_types_t::SET; @@ -115,7 +118,7 @@ struct rbm_test_t : } } - void free_extent(rbm_transaction &t, interval_set<blk_id_t> range) { + void free_extent(rbm_transaction &t, interval_set<blk_no_t> range) { for (auto [off, len] : range) { logger().debug("free_extent: start {} len {}", off * DEFAULT_BLOCK_SIZE, len * DEFAULT_BLOCK_SIZE); @@ -124,16 +127,13 @@ struct rbm_test_t : } } - interval_set<blk_id_t> get_allocated_blk_ids(rbm_transaction &t) { + interval_set<blk_no_t> get_allocated_blk_ids(rbm_transaction &t) { auto allocated_blocks = t.get_rbm_allocated_blocks(); - interval_set<blk_id_t> alloc_ids; + interval_set<blk_no_t> alloc_ids; for (auto p : allocated_blocks) { for (auto b : p.alloc_blk_ranges) { - blk_paddr_t addr = - convert_paddr_to_blk_paddr( - b.first, - block_size, - config.blocks_per_segment); + rbm_abs_addr addr = + convert_paddr_to_abs_addr(b.first); alloc_ids.insert(addr / block_size, b.second / block_size); } } @@ -141,16 +141,19 @@ struct rbm_test_t : return alloc_ids; } - bool check_ids_are_allocated(interval_set<blk_id_t> &ids, bool allocated = true) { + bool check_ids_are_allocated(interval_set<blk_no_t> &ids, bool allocated = true) { bool ret = true; for (auto r : ids) { - for (blk_id_t id = r.first; id < r.first + r.second; id++) { + for (blk_no_t id = r.first; id < r.first + r.second; id++) { auto addr = rbm_manager->get_start_block_alloc_area() + (id / rbm_manager->max_block_by_bitmap_block()) * DEFAULT_BLOCK_SIZE; logger().debug(" addr {} id {} ", addr, id); auto bp = bufferptr(ceph::buffer::create_page_aligned(DEFAULT_BLOCK_SIZE)); - rbm_manager->read(addr, bp).unsafe_get0(); + paddr_t paddr = convert_abs_addr_to_paddr( + addr, + rbm_manager->get_device_id()); + rbm_manager->read(paddr, bp).unsafe_get0(); rbm_bitmap_block_t b_block(DEFAULT_BLOCK_SIZE); bufferlist bl; bl.append(bp); @@ -287,15 +290,16 @@ TEST_F(rbm_test_t, block_alloc_free_test) TEST_F(rbm_test_t, many_block_alloc) { run_async([this] { - config.start = paddr_t::make_seg_paddr(0, 0, 0); - config.end = paddr_t::make_seg_paddr(0, 0, DEFAULT_TEST_SIZE * 1024); + device_id_t d_id = 1 << (std::numeric_limits<device_id_t>::digits - 1); + config.start = paddr_t::make_blk_paddr(d_id, 0); + config.end = paddr_t::make_blk_paddr(d_id, (DEFAULT_TEST_SIZE * 1024)); config.block_size = DEFAULT_BLOCK_SIZE; config.total_size = DEFAULT_TEST_SIZE * 1024; mkfs(); open(); auto max = rbm_manager->max_block_by_bitmap_block(); rbm_manager->rbm_sync_block_bitmap_by_range(max + 10, max + 14, bitmap_op_types_t::ALL_SET).unsafe_get0(); - interval_set<blk_id_t> alloc_ids; + interval_set<blk_no_t> alloc_ids; alloc_ids.insert(max + 12, 2); ASSERT_TRUE(check_ids_are_allocated(alloc_ids)); alloc_ids.clear(); @@ -349,7 +353,7 @@ TEST_F(rbm_test_t, check_free_blocks) rbm_manager->check_bitmap_blocks().unsafe_get0(); ASSERT_TRUE(rbm_manager->get_free_blocks() == DEFAULT_TEST_SIZE/DEFAULT_BLOCK_SIZE - 5); auto free = rbm_manager->get_free_blocks(); - interval_set<blk_id_t> alloc_ids; + interval_set<blk_no_t> alloc_ids; auto t = create_rbm_transaction(); alloc_extent(*t, DEFAULT_BLOCK_SIZE * 4); alloc_ids = get_allocated_blk_ids(*t); diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc index 0bb8213786b..4b49f020acf 100644 --- a/src/test/crimson/seastore/test_seastore_cache.cc +++ b/src/test/crimson/seastore/test_seastore_cache.cc @@ -37,9 +37,9 @@ struct cache_test_t : public seastar_test_suite_t { bl.append(block.bl); } - ceph_assert((segment_off_t)bl.length() < + ceph_assert((seastore_off_t)bl.length() < segment_manager->get_segment_size()); - if (current.as_seg_paddr().get_segment_off() + (segment_off_t)bl.length() > + if (current.as_seg_paddr().get_segment_off() + (seastore_off_t)bl.length() > segment_manager->get_segment_size()) current = paddr_t::make_seg_paddr( segment_id_t( diff --git a/src/test/crimson/seastore/test_seastore_journal.cc b/src/test/crimson/seastore/test_seastore_journal.cc index eaeb8cd2d0c..00e1944b790 100644 --- a/src/test/crimson/seastore/test_seastore_journal.cc +++ b/src/test/crimson/seastore/test_seastore_journal.cc @@ -74,7 +74,7 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { std::default_random_engine generator; - segment_off_t block_size; + seastore_off_t block_size; ExtentReaderRef scanner; diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 6323f72729e..9213d2b8243 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -251,8 +251,8 @@ public: } size_t get_size() const final { return sm.get_size(); } - segment_off_t get_block_size() const final { return sm.get_block_size(); } - segment_off_t get_segment_size() const final { + seastore_off_t get_block_size() const final { return sm.get_block_size(); } + seastore_off_t get_segment_size() const final { return sm.get_segment_size(); } const seastore_meta_t &get_meta() const final { diff --git a/src/tools/osdmaptool.cc b/src/tools/osdmaptool.cc index 3b4cbff3578..15264645b94 100644 --- a/src/tools/osdmaptool.cc +++ b/src/tools/osdmaptool.cc @@ -693,9 +693,10 @@ skip_upmap: for (unsigned i=0; i<osds.size(); i++) { //cout << " rep " << i << " on " << osds[i] << std::endl; - count[osds[i]]++; + if (osds[i] != CRUSH_ITEM_NONE) + count[osds[i]]++; } - if (osds.size()) + if (osds.size() && osds[0] != CRUSH_ITEM_NONE) first_count[osds[0]]++; if (primary >= 0) primary_count[primary]++; |