path: root/erts/emulator/asmjit/core/codeholder.cpp

// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib

#include "../core/api-build_p.h"
#include "../core/assembler.h"
#include "../core/codewriter_p.h"
#include "../core/logger.h"
#include "../core/support.h"

#include <algorithm>
#include <tuple>

ASMJIT_BEGIN_NAMESPACE

// Globals
// =======

static const char CodeHolder_addrTabName[] = ".addrtab";

//! Encode MOD byte.
static inline uint32_t x86EncodeMod(uint32_t m, uint32_t o, uint32_t rm) noexcept {
  return (m << 6) | (o << 3) | rm;
}

// LabelLinkIterator
// =================

class LabelLinkIterator {
public:
  inline LabelLinkIterator(LabelEntry* le) noexcept { reset(le); }

  inline explicit operator bool() const noexcept { return isValid(); }
  inline bool isValid() const noexcept { return _link != nullptr; }

  inline LabelLink* link() const noexcept { return _link; }
  inline LabelLink* operator->() const noexcept { return _link; }

  inline void reset(LabelEntry* le) noexcept {
    _pPrev = &le->_links;
    _link = *_pPrev;
  }

  inline void next() noexcept {
    _pPrev = &_link->next;
    _link = *_pPrev;
  }

  inline void resolveAndNext(CodeHolder* code) noexcept {
    LabelLink* linkToDelete = _link;

    _link = _link->next;
    *_pPrev = _link;

    code->_unresolvedLinkCount--;
    code->_allocator.release(linkToDelete, sizeof(LabelLink));
  }

  LabelLink** _pPrev;
  LabelLink* _link;
};

// CodeHolder - Utilities
// ======================

static void CodeHolder_resetInternal(CodeHolder* self, ResetPolicy resetPolicy) noexcept {
  uint32_t i;
  const ZoneVector<BaseEmitter*>& emitters = self->emitters();

  i = emitters.size();
  while (i)
    self->detach(emitters[--i]);

  // Reset everything into its construction state.
  self->_environment.reset();
  self->_cpuFeatures.reset();
  self->_baseAddress = Globals::kNoBaseAddress;
  self->_logger = nullptr;
  self->_errorHandler = nullptr;

  // Reset all sections.
  uint32_t numSections = self->_sections.size();
  for (i = 0; i < numSections; i++) {
    Section* section = self->_sections[i];
    if (section->_buffer.data() && !section->_buffer.isExternal())
      ::free(section->_buffer._data);
    section->_buffer._data = nullptr;
    section->_buffer._capacity = 0;
  }

  // Reset zone allocator and all containers using it.
  ZoneAllocator* allocator = self->allocator();

  self->_emitters.reset();
  self->_namedLabels.reset();
  self->_relocations.reset();
  self->_labelEntries.reset();
  self->_sections.reset();
  self->_sectionsByOrder.reset();

  self->_unresolvedLinkCount = 0;
  self->_addressTableSection = nullptr;
  self->_addressTableEntries.reset();

  allocator->reset(&self->_zone);
  self->_zone.reset(resetPolicy);
}

static void CodeHolder_onSettingsUpdated(CodeHolder* self) noexcept {
  // Notify all attached emitters about a settings update.
  for (BaseEmitter* emitter : self->emitters()) {
    emitter->onSettingsUpdated();
  }
}

// CodeHolder - Construction & Destruction
// =======================================

CodeHolder::CodeHolder(const Support::Temporary* temporary) noexcept
  : _environment(),
    _cpuFeatures{},
    _baseAddress(Globals::kNoBaseAddress),
    _logger(nullptr),
    _errorHandler(nullptr),
    _zone(16384 - Zone::kBlockOverhead, 1, temporary),
    _allocator(&_zone),
    _unresolvedLinkCount(0),
    _addressTableSection(nullptr) {}

CodeHolder::~CodeHolder() noexcept {
  CodeHolder_resetInternal(this, ResetPolicy::kHard);
}

// CodeHolder - Init & Reset
// =========================

inline void CodeHolder_setSectionDefaultName(
  Section* section,
  char c0 = 0, char c1 = 0, char c2 = 0, char c3 = 0,
  char c4 = 0, char c5 = 0, char c6 = 0, char c7 = 0) noexcept {

  section->_name.u32[0] = Support::bytepack32_4x8(uint8_t(c0), uint8_t(c1), uint8_t(c2), uint8_t(c3));
  section->_name.u32[1] = Support::bytepack32_4x8(uint8_t(c4), uint8_t(c5), uint8_t(c6), uint8_t(c7));
}

Error CodeHolder::init(const Environment& environment, uint64_t baseAddress) noexcept {
  return init(environment, CpuFeatures{}, baseAddress);
}

Error CodeHolder::init(const Environment& environment, const CpuFeatures& cpuFeatures, uint64_t baseAddress) noexcept {
  // Cannot reinitialize if it's locked or there is one or more emitter attached.
  if (isInitialized())
    return DebugUtils::errored(kErrorAlreadyInitialized);

  // If we are just initializing there should be no emitters attached.
  ASMJIT_ASSERT(_emitters.empty());

  // Create a default section and insert it to the `_sections` array.
  Error err = _sections.willGrow(&_allocator) |
              _sectionsByOrder.willGrow(&_allocator);
  if (err == kErrorOk) {
    Section* section = _allocator.allocZeroedT<Section>();
    if (ASMJIT_LIKELY(section)) {
      section->_flags = SectionFlags::kExecutable | SectionFlags::kReadOnly;
      CodeHolder_setSectionDefaultName(section, '.', 't', 'e', 'x', 't');
      _sections.appendUnsafe(section);
      _sectionsByOrder.appendUnsafe(section);
    }
    else {
      err = DebugUtils::errored(kErrorOutOfMemory);
    }
  }

  if (ASMJIT_UNLIKELY(err)) {
    _zone.reset();
    return err;
  }
  else {
    _environment = environment;
    _cpuFeatures = cpuFeatures;
    _baseAddress = baseAddress;
    return kErrorOk;
  }
}

void CodeHolder::reset(ResetPolicy resetPolicy) noexcept {
  CodeHolder_resetInternal(this, resetPolicy);
}

// CodeHolder - Attach / Detach
// ============================

Error CodeHolder::attach(BaseEmitter* emitter) noexcept {
  // Catch a possible misuse of the API.
  if (ASMJIT_UNLIKELY(!emitter))
    return DebugUtils::errored(kErrorInvalidArgument);

  // Invalid emitter, this should not be possible.
  EmitterType type = emitter->emitterType();
  if (ASMJIT_UNLIKELY(type == EmitterType::kNone || uint32_t(type) > uint32_t(EmitterType::kMaxValue)))
    return DebugUtils::errored(kErrorInvalidState);

  uint64_t archMask = emitter->_archMask;
  if (ASMJIT_UNLIKELY(!(archMask & (uint64_t(1) << uint32_t(arch())))))
    return DebugUtils::errored(kErrorInvalidArch);

  // This is suspicious, but don't fail if `emitter` is already attached
  // to this code holder. This is not error, but it's not recommended.
  if (emitter->_code != nullptr) {
    if (emitter->_code == this)
      return kErrorOk;
    return DebugUtils::errored(kErrorInvalidState);
  }

  // Reserve the space now as we cannot fail after `onAttach()` succeeded.
  ASMJIT_PROPAGATE(_emitters.willGrow(&_allocator, 1));
  ASMJIT_PROPAGATE(emitter->onAttach(this));

  // Connect CodeHolder <-> BaseEmitter.
  ASMJIT_ASSERT(emitter->_code == this);
  _emitters.appendUnsafe(emitter);

  return kErrorOk;
}

Error CodeHolder::detach(BaseEmitter* emitter) noexcept {
  if (ASMJIT_UNLIKELY(!emitter))
    return DebugUtils::errored(kErrorInvalidArgument);

  if (ASMJIT_UNLIKELY(emitter->_code != this))
    return DebugUtils::errored(kErrorInvalidState);

  // NOTE: We always detach if we were asked to, if error happens during
  // `emitter->onDetach()` we just propagate it, but the BaseEmitter will
  // be detached.
  Error err = kErrorOk;
  if (!emitter->isDestroyed())
    err = emitter->onDetach(this);

  // Disconnect CodeHolder <-> BaseEmitter.
  uint32_t index = _emitters.indexOf(emitter);
  ASMJIT_ASSERT(index != Globals::kNotFound);

  _emitters.removeAt(index);
  emitter->_code = nullptr;

  return err;
}

// CodeHolder - Logging
// ====================

void CodeHolder::setLogger(Logger* logger) noexcept {
#ifndef ASMJIT_NO_LOGGING
  _logger = logger;
  CodeHolder_onSettingsUpdated(this);
#else
  DebugUtils::unused(logger);
#endif
}

// CodeHolder - Error Handling
// ===========================

void CodeHolder::setErrorHandler(ErrorHandler* errorHandler) noexcept {
  _errorHandler = errorHandler;
  CodeHolder_onSettingsUpdated(this);
}

// CodeHolder - Code Buffer
// ========================

static Error CodeHolder_reserveInternal(CodeHolder* self, CodeBuffer* cb, size_t n) noexcept {
  uint8_t* oldData = cb->_data;
  uint8_t* newData;

  if (oldData && !cb->isExternal())
    newData = static_cast<uint8_t*>(::realloc(oldData, n));
  else
    newData = static_cast<uint8_t*>(::malloc(n));

  if (ASMJIT_UNLIKELY(!newData))
    return DebugUtils::errored(kErrorOutOfMemory);

  cb->_data = newData;
  cb->_capacity = n;

  // Update pointers used by assemblers, if attached.
  for (BaseEmitter* emitter : self->emitters()) {
    if (emitter->isAssembler()) {
      BaseAssembler* a = static_cast<BaseAssembler*>(emitter);
      if (&a->_section->_buffer == cb) {
        size_t offset = a->offset();

        a->_bufferData = newData;
        a->_bufferEnd  = newData + n;
        a->_bufferPtr  = newData + offset;
      }
    }
  }

  return kErrorOk;
}

Error CodeHolder::growBuffer(CodeBuffer* cb, size_t n) noexcept {
  // The size of the section must be valid.
  size_t size = cb->size();
  if (ASMJIT_UNLIKELY(n > std::numeric_limits<uintptr_t>::max() - size))
    return DebugUtils::errored(kErrorOutOfMemory);

  // We can now check if growing the buffer is really necessary. It's unlikely
  // that this function is called while there is still room for `n` bytes.
  size_t capacity = cb->capacity();
  size_t required = cb->size() + n;
  if (ASMJIT_UNLIKELY(required <= capacity))
    return kErrorOk;

  if (cb->isFixed())
    return DebugUtils::errored(kErrorTooLarge);

  size_t kInitialCapacity = 8096;
  if (capacity < kInitialCapacity)
    capacity = kInitialCapacity;
  else
    capacity += Globals::kAllocOverhead;

  do {
    size_t old = capacity;
    if (capacity < Globals::kGrowThreshold)
      capacity *= 2;
    else
      capacity += Globals::kGrowThreshold;

    // Overflow.
    if (ASMJIT_UNLIKELY(old > capacity))
      return DebugUtils::errored(kErrorOutOfMemory);
  } while (capacity - Globals::kAllocOverhead < required);

  return CodeHolder_reserveInternal(this, cb, capacity - Globals::kAllocOverhead);
}

Error CodeHolder::reserveBuffer(CodeBuffer* cb, size_t n) noexcept {
  size_t capacity = cb->capacity();

  if (n <= capacity)
    return kErrorOk;

  if (cb->isFixed())
    return DebugUtils::errored(kErrorTooLarge);

  return CodeHolder_reserveInternal(this, cb, n);
}

// CodeHolder - Sections
// =====================

Error CodeHolder::newSection(Section** sectionOut, const char* name, size_t nameSize, SectionFlags flags, uint32_t alignment, int32_t order) noexcept {
  *sectionOut = nullptr;

  if (nameSize == SIZE_MAX)
    nameSize = strlen(name);

  if (alignment == 0)
    alignment = 1;

  if (ASMJIT_UNLIKELY(!Support::isPowerOf2(alignment)))
    return DebugUtils::errored(kErrorInvalidArgument);

  if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxSectionNameSize))
    return DebugUtils::errored(kErrorInvalidSectionName);

  uint32_t sectionId = _sections.size();
  if (ASMJIT_UNLIKELY(sectionId == Globals::kInvalidId))
    return DebugUtils::errored(kErrorTooManySections);

  ASMJIT_PROPAGATE(_sections.willGrow(&_allocator));
  ASMJIT_PROPAGATE(_sectionsByOrder.willGrow(&_allocator));

  Section* section = _allocator.allocZeroedT<Section>();
  if (ASMJIT_UNLIKELY(!section))
    return DebugUtils::errored(kErrorOutOfMemory);

  section->_id = sectionId;
  section->_flags = flags;
  section->_alignment = alignment;
  section->_order = order;
  memcpy(section->_name.str, name, nameSize);

  Section** insertPosition = std::lower_bound(_sectionsByOrder.begin(), _sectionsByOrder.end(), section, [](const Section* a, const Section* b) {
    return std::make_tuple(a->order(), a->id()) < std::make_tuple(b->order(), b->id());
  });

  _sections.appendUnsafe(section);
  _sectionsByOrder.insertUnsafe((size_t)(insertPosition - _sectionsByOrder.data()), section);

  *sectionOut = section;
  return kErrorOk;
}

Section* CodeHolder::sectionByName(const char* name, size_t nameSize) const noexcept {
  if (nameSize == SIZE_MAX)
    nameSize = strlen(name);

  // This could be also put in a hash-table similarly like we do with labels,
  // however it's questionable as the number of sections should be pretty low
  // in general. Create an issue if this becomes a problem.
  if (nameSize <= Globals::kMaxSectionNameSize) {
    for (Section* section : _sections)
      if (memcmp(section->_name.str, name, nameSize) == 0 && section->_name.str[nameSize] == '\0')
        return section;
  }

  return nullptr;
}

Section* CodeHolder::ensureAddressTableSection() noexcept {
  if (_addressTableSection)
    return _addressTableSection;

  newSection(&_addressTableSection,
             CodeHolder_addrTabName,
             sizeof(CodeHolder_addrTabName) - 1,
             SectionFlags::kNone,
             _environment.registerSize(),
             std::numeric_limits<int32_t>::max());
  return _addressTableSection;
}

Error CodeHolder::addAddressToAddressTable(uint64_t address) noexcept {
  AddressTableEntry* entry = _addressTableEntries.get(address);
  if (entry)
    return kErrorOk;

  Section* section = ensureAddressTableSection();
  if (ASMJIT_UNLIKELY(!section))
    return DebugUtils::errored(kErrorOutOfMemory);

  entry = _zone.newT<AddressTableEntry>(address);
  if (ASMJIT_UNLIKELY(!entry))
    return DebugUtils::errored(kErrorOutOfMemory);

  _addressTableEntries.insert(entry);
  section->_virtualSize += _environment.registerSize();

  return kErrorOk;
}

// CodeHolder - Labels & Symbols
// =============================

//! Only used to lookup a label from `_namedLabels`.
class LabelByName {
public:
  inline LabelByName(const char* key, size_t keySize, uint32_t hashCode, uint32_t parentId) noexcept
    : _key(key),
      _keySize(uint32_t(keySize)),
      _hashCode(hashCode),
      _parentId(parentId) {}

  inline uint32_t hashCode() const noexcept { return _hashCode; }

  inline bool matches(const LabelEntry* entry) const noexcept {
    return entry->nameSize() == _keySize &&
           entry->parentId() == _parentId &&
           ::memcmp(entry->name(), _key, _keySize) == 0;
  }

  const char* _key;
  uint32_t _keySize;
  uint32_t _hashCode;
  uint32_t _parentId;
};

// Returns a hash of `name` and fixes `nameSize` if it's `SIZE_MAX`.
static uint32_t CodeHolder_hashNameAndGetSize(const char* name, size_t& nameSize) noexcept {
  uint32_t hashCode = 0;
  if (nameSize == SIZE_MAX) {
    size_t i = 0;
    for (;;) {
      uint8_t c = uint8_t(name[i]);
      if (!c) break;
      hashCode = Support::hashRound(hashCode, c);
      i++;
    }
    nameSize = i;
  }
  else {
    for (size_t i = 0; i < nameSize; i++) {
      uint8_t c = uint8_t(name[i]);
      if (ASMJIT_UNLIKELY(!c)) return DebugUtils::errored(kErrorInvalidLabelName);
      hashCode = Support::hashRound(hashCode, c);
    }
  }
  return hashCode;
}

LabelLink* CodeHolder::newLabelLink(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept {
  LabelLink* link = _allocator.allocT<LabelLink>();
  if (ASMJIT_UNLIKELY(!link)) return nullptr;

  link->next = le->_links;
  le->_links = link;

  link->sectionId = sectionId;
  link->relocId = Globals::kInvalidId;
  link->offset = offset;
  link->rel = rel;
  link->format = format;

  _unresolvedLinkCount++;
  return link;
}

Error CodeHolder::newLabelEntry(LabelEntry** entryOut) noexcept {
  *entryOut = nullptr;

  uint32_t labelId = _labelEntries.size();
  if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId))
    return DebugUtils::errored(kErrorTooManyLabels);

  ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator));
  LabelEntry* le = _allocator.allocZeroedT<LabelEntry>();

  if (ASMJIT_UNLIKELY(!le))
    return DebugUtils::errored(kErrorOutOfMemory);

  le->_setId(labelId);
  le->_parentId = Globals::kInvalidId;
  le->_offset = 0;
  _labelEntries.appendUnsafe(le);

  *entryOut = le;
  return kErrorOk;
}

Error CodeHolder::newNamedLabelEntry(LabelEntry** entryOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId) noexcept {
  *entryOut = nullptr;
  uint32_t hashCode = CodeHolder_hashNameAndGetSize(name, nameSize);

  if (ASMJIT_UNLIKELY(nameSize == 0)) {
    if (type == LabelType::kAnonymous)
      return newLabelEntry(entryOut);
    else
      return DebugUtils::errored(kErrorInvalidLabelName);
  }

  if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxLabelNameSize))
    return DebugUtils::errored(kErrorLabelNameTooLong);

  switch (type) {
    case LabelType::kAnonymous: {
      // Anonymous labels cannot have a parent (or more specifically, parent is useless here).
      if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId))
        return DebugUtils::errored(kErrorInvalidParentLabel);

      uint32_t labelId = _labelEntries.size();
      if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId))
        return DebugUtils::errored(kErrorTooManyLabels);

      ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator));
      LabelEntry* le = _allocator.allocZeroedT<LabelEntry>();

      if (ASMJIT_UNLIKELY(!le))
        return DebugUtils::errored(kErrorOutOfMemory);

      // NOTE: This LabelEntry has a name, but we leave its hashCode as zero as it's anonymous.
      le->_setId(labelId);
      le->_parentId = Globals::kInvalidId;
      le->_offset = 0;
      ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize));

      _labelEntries.appendUnsafe(le);

      *entryOut = le;
      return kErrorOk;
    }

    case LabelType::kLocal: {
      if (ASMJIT_UNLIKELY(parentId >= _labelEntries.size()))
        return DebugUtils::errored(kErrorInvalidParentLabel);

      hashCode ^= parentId;
      break;
    }

    case LabelType::kGlobal:
    case LabelType::kExternal: {
      if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId))
        return DebugUtils::errored(kErrorInvalidParentLabel);
      break;
    }

    default: {
      return DebugUtils::errored(kErrorInvalidArgument);
    }
  }

  // Don't allow to insert duplicates. Local labels allow duplicates that have
  // different id, this is already accomplished by having a different hashes
  // between the same label names having different parent labels.
  LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId));
  if (ASMJIT_UNLIKELY(le))
    return DebugUtils::errored(kErrorLabelAlreadyDefined);

  Error err = kErrorOk;
  uint32_t labelId = _labelEntries.size();

  if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId))
    return DebugUtils::errored(kErrorTooManyLabels);

  ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator));
  le = _allocator.allocZeroedT<LabelEntry>();

  if (ASMJIT_UNLIKELY(!le))
    return DebugUtils::errored(kErrorOutOfMemory);

  le->_hashCode = hashCode;
  le->_setId(labelId);
  le->_type = type;
  le->_parentId = parentId;
  le->_offset = 0;
  ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize));

  _labelEntries.appendUnsafe(le);
  _namedLabels.insert(allocator(), le);

  *entryOut = le;
  return err;
}

uint32_t CodeHolder::labelIdByName(const char* name, size_t nameSize, uint32_t parentId) noexcept {
  uint32_t hashCode = CodeHolder_hashNameAndGetSize(name, nameSize);
  if (ASMJIT_UNLIKELY(!nameSize))
    return 0;

  if (parentId != Globals::kInvalidId)
    hashCode ^= parentId;

  LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId));
  return le ? le->id() : uint32_t(Globals::kInvalidId);
}

ASMJIT_API Error CodeHolder::resolveUnresolvedLinks() noexcept {
  if (!hasUnresolvedLinks())
    return kErrorOk;

  Error err = kErrorOk;
  for (LabelEntry* le : labelEntries()) {
    if (!le->isBound())
      continue;

    LabelLinkIterator link(le);
    if (link) {
      Support::FastUInt8 of = 0;
      Section* toSection = le->section();
      uint64_t toOffset = Support::addOverflow(toSection->offset(), le->offset(), &of);

      do {
        uint32_t linkSectionId = link->sectionId;
        if (link->relocId == Globals::kInvalidId) {
          Section* fromSection = sectionById(linkSectionId);
          size_t linkOffset = link->offset;

          CodeBuffer& buf = _sections[linkSectionId]->buffer();
          ASMJIT_ASSERT(linkOffset < buf.size());

          // Calculate the offset relative to the start of the virtual base.
          Support::FastUInt8 localOF = of;
          uint64_t fromOffset = Support::addOverflow<uint64_t>(fromSection->offset(), linkOffset, &localOF);
          int64_t displacement = int64_t(toOffset - fromOffset + uint64_t(int64_t(link->rel)));

          if (!localOF) {
            ASMJIT_ASSERT(size_t(linkOffset) < buf.size());
            ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.valueSize());

            // Overwrite a real displacement in the CodeBuffer.
            if (CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) {
              link.resolveAndNext(this);
              continue;
            }
          }

          err = DebugUtils::errored(kErrorInvalidDisplacement);
          // Falls through to `link.next()`.
        }

        link.next();
      } while (link);
    }
  }

  return err;
}

ASMJIT_API Error CodeHolder::bindLabel(const Label& label, uint32_t toSectionId, uint64_t toOffset) noexcept {
  LabelEntry* le = labelEntry(label);
  if (ASMJIT_UNLIKELY(!le))
    return DebugUtils::errored(kErrorInvalidLabel);

  if (ASMJIT_UNLIKELY(toSectionId > _sections.size()))
    return DebugUtils::errored(kErrorInvalidSection);

  // Label can be bound only once.
  if (ASMJIT_UNLIKELY(le->isBound()))
    return DebugUtils::errored(kErrorLabelAlreadyBound);

  // Bind the label.
  Section* section = _sections[toSectionId];
  le->_section = section;
  le->_offset = toOffset;

  Error err = kErrorOk;
  CodeBuffer& buf = section->buffer();

  // Fix all links to this label we have collected so far if they are within
  // the same section. We ignore any inter-section links as these have to be
  // fixed later.
  LabelLinkIterator link(le);
  while (link) {
    uint32_t linkSectionId = link->sectionId;
    size_t linkOffset = link->offset;

    uint32_t relocId = link->relocId;
    if (relocId != Globals::kInvalidId) {
      // Adjust relocation data only.
      RelocEntry* re = _relocations[relocId];
      re->_payload += toOffset;
      re->_targetSectionId = toSectionId;
    }
    else {
      if (linkSectionId != toSectionId) {
        link.next();
        continue;
      }

      ASMJIT_ASSERT(linkOffset < buf.size());
      int64_t displacement = int64_t(toOffset - uint64_t(linkOffset) + uint64_t(int64_t(link->rel)));

      // Size of the value we are going to patch. Only BYTE/DWORD is allowed.
      ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.regionSize());

      // Overwrite a real displacement in the CodeBuffer.
      if (!CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) {
        err = DebugUtils::errored(kErrorInvalidDisplacement);
        link.next();
        continue;
      }
    }

    link.resolveAndNext(this);
  }

  return err;
}

// CodeHolder - Relocations
// ========================

Error CodeHolder::newRelocEntry(RelocEntry** dst, RelocType relocType) noexcept {
  ASMJIT_PROPAGATE(_relocations.willGrow(&_allocator));

  uint32_t relocId = _relocations.size();
  if (ASMJIT_UNLIKELY(relocId == Globals::kInvalidId))
    return DebugUtils::errored(kErrorTooManyRelocations);

  RelocEntry* re = _allocator.allocZeroedT<RelocEntry>();
  if (ASMJIT_UNLIKELY(!re))
    return DebugUtils::errored(kErrorOutOfMemory);

  re->_id = relocId;
  re->_relocType = relocType;
  re->_sourceSectionId = Globals::kInvalidId;
  re->_targetSectionId = Globals::kInvalidId;
  _relocations.appendUnsafe(re);

  *dst = re;
  return kErrorOk;
}

// CodeHolder - Expression Evaluation
// ==================================

static Error CodeHolder_evaluateExpression(CodeHolder* self, Expression* exp, uint64_t* out) noexcept {
  uint64_t value[2];
  for (size_t i = 0; i < 2; i++) {
    uint64_t v;
    switch (exp->valueType[i]) {
      case ExpressionValueType::kNone: {
        v = 0;
        break;
      }

      case ExpressionValueType::kConstant: {
        v = exp->value[i].constant;
        break;
      }

      case ExpressionValueType::kLabel: {
        LabelEntry* le = exp->value[i].label;
        if (!le->isBound())
          return DebugUtils::errored(kErrorExpressionLabelNotBound);
        v = le->section()->offset() + le->offset();
        break;
      }

      case ExpressionValueType::kExpression: {
        Expression* nested = exp->value[i].expression;
        ASMJIT_PROPAGATE(CodeHolder_evaluateExpression(self, nested, &v));
        break;
      }

      default:
        return DebugUtils::errored(kErrorInvalidState);
    }

    value[i] = v;
  }

  uint64_t result;
  uint64_t& a = value[0];
  uint64_t& b = value[1];

  switch (exp->opType) {
    case ExpressionOpType::kAdd:
      result = a + b;
      break;

    case ExpressionOpType::kSub:
      result = a - b;
      break;

    case ExpressionOpType::kMul:
      result = a * b;
      break;

    case ExpressionOpType::kSll:
      result = (b > 63) ? uint64_t(0) : uint64_t(a << b);
      break;

    case ExpressionOpType::kSrl:
      result = (b > 63) ? uint64_t(0) : uint64_t(a >> b);
      break;

    case ExpressionOpType::kSra:
      result = Support::sar(a, Support::min<uint64_t>(b, 63));
      break;

    default:
      return DebugUtils::errored(kErrorInvalidState);
  }

  *out = result;
  return kErrorOk;
}

// CodeHolder - Utilities
// ======================

Error CodeHolder::flatten() noexcept {
  uint64_t offset = 0;
  for (Section* section : _sectionsByOrder) {
    uint64_t realSize = section->realSize();
    if (realSize) {
      uint64_t alignedOffset = Support::alignUp(offset, section->alignment());
      if (ASMJIT_UNLIKELY(alignedOffset < offset))
        return DebugUtils::errored(kErrorTooLarge);

      Support::FastUInt8 of = 0;
      offset = Support::addOverflow(alignedOffset, realSize, &of);

      if (ASMJIT_UNLIKELY(of))
        return DebugUtils::errored(kErrorTooLarge);
    }
  }

  // Now we know that we can assign offsets of all sections properly.
  Section* prev = nullptr;
  offset = 0;
  for (Section* section : _sectionsByOrder) {
    uint64_t realSize = section->realSize();
    if (realSize)
      offset = Support::alignUp(offset, section->alignment());
    section->_offset = offset;

    // Make sure the previous section extends a bit to cover the alignment.
    if (prev)
      prev->_virtualSize = offset - prev->_offset;

    prev = section;
    offset += realSize;
  }

  return kErrorOk;
}

size_t CodeHolder::codeSize() const noexcept {
  Support::FastUInt8 of = 0;
  uint64_t offset = 0;

  for (Section* section : _sectionsByOrder) {
    uint64_t realSize = section->realSize();

    if (realSize) {
      uint64_t alignedOffset = Support::alignUp(offset, section->alignment());
      ASMJIT_ASSERT(alignedOffset >= offset);
      offset = Support::addOverflow(alignedOffset, realSize, &of);
    }
  }

  if ((sizeof(uint64_t) > sizeof(size_t) && offset > SIZE_MAX) || of)
    return SIZE_MAX;

  return size_t(offset);
}

Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept {
  // Base address must be provided.
  if (ASMJIT_UNLIKELY(baseAddress == Globals::kNoBaseAddress))
    return DebugUtils::errored(kErrorInvalidArgument);

  _baseAddress = baseAddress;
  uint32_t addressSize = _environment.registerSize();

  Section* addressTableSection = _addressTableSection;
  uint32_t addressTableEntryCount = 0;
  uint8_t* addressTableEntryData = nullptr;

  if (addressTableSection) {
    ASMJIT_PROPAGATE(
      reserveBuffer(&addressTableSection->_buffer, size_t(addressTableSection->virtualSize())));
    addressTableEntryData = addressTableSection->_buffer.data();
  }

  // Relocate all recorded locations.
  for (const RelocEntry* re : _relocations) {
    // Possibly deleted or optimized-out entry.
    if (re->relocType() == RelocType::kNone)
      continue;

    Section* sourceSection = sectionById(re->sourceSectionId());
    Section* targetSection = nullptr;

    if (re->targetSectionId() != Globals::kInvalidId)
      targetSection = sectionById(re->targetSectionId());

    uint64_t value = re->payload();
    uint64_t sectionOffset = sourceSection->offset();
    uint64_t sourceOffset = re->sourceOffset();

    // Make sure that the `RelocEntry` doesn't go out of bounds.
    size_t regionSize = re->format().regionSize();
    if (ASMJIT_UNLIKELY(re->sourceOffset() >= sourceSection->bufferSize() ||
                        sourceSection->bufferSize() - size_t(re->sourceOffset()) < regionSize))
      return DebugUtils::errored(kErrorInvalidRelocEntry);

    uint8_t* buffer = sourceSection->data();

    switch (re->relocType()) {
      case RelocType::kExpression: {
        Expression* expression = (Expression*)(uintptr_t(value));
        ASMJIT_PROPAGATE(CodeHolder_evaluateExpression(this, expression, &value));
        break;
      }

      case RelocType::kAbsToAbs: {
        break;
      }

      case RelocType::kRelToAbs: {
        // Value is currently a relative offset from the start of its section.
        // We have to convert it to an absolute offset (including base address).
        if (ASMJIT_UNLIKELY(!targetSection))
          return DebugUtils::errored(kErrorInvalidRelocEntry);

        //value += baseAddress + sectionOffset + sourceOffset + regionSize;
        value += baseAddress + targetSection->offset();
        break;
      }

      case RelocType::kAbsToRel: {
        value -= baseAddress + sectionOffset + sourceOffset + regionSize;

        // Sign extend as we are not interested in the high 32-bit word in a 32-bit address space.
        if (addressSize <= 4)
          value = uint64_t(int64_t(int32_t(value & 0xFFFFFFFFu)));
        else if (!Support::isInt32(int64_t(value)))
          return DebugUtils::errored(kErrorRelocOffsetOutOfRange);

        break;
      }

      case RelocType::kX64AddressEntry: {
        size_t valueOffset = size_t(re->sourceOffset()) + re->format().valueOffset();
        if (re->format().valueSize() != 4 || valueOffset < 2)
          return DebugUtils::errored(kErrorInvalidRelocEntry);

        // First try whether a relative 32-bit displacement would work.
        value -= baseAddress + sectionOffset + sourceOffset + regionSize;
        if (!Support::isInt32(int64_t(value))) {
          // Relative 32-bit displacement is not possible, use '.addrtab' section.
          AddressTableEntry* atEntry = _addressTableEntries.get(re->payload());
          if (ASMJIT_UNLIKELY(!atEntry))
            return DebugUtils::errored(kErrorInvalidRelocEntry);

          // Cannot be null as we have just matched the `AddressTableEntry`.
          ASMJIT_ASSERT(addressTableSection != nullptr);

          if (!atEntry->hasAssignedSlot())
            atEntry->_slot = addressTableEntryCount++;

          size_t atEntryIndex = size_t(atEntry->slot()) * addressSize;
          uint64_t addrSrc = sectionOffset + sourceOffset + regionSize;
          uint64_t addrDst = addressTableSection->offset() + uint64_t(atEntryIndex);

          value = addrDst - addrSrc;
          if (!Support::isInt32(int64_t(value)))
            return DebugUtils::errored(kErrorRelocOffsetOutOfRange);

          // Bytes that replace [REX, OPCODE] bytes.
          uint32_t byte0 = 0xFF;
          uint32_t byte1 = buffer[valueOffset - 1];

          if (byte1 == 0xE8) {
            // Patch CALL/MOD byte to FF /2 (-> 0x15).
            byte1 = x86EncodeMod(0, 2, 5);
          }
          else if (byte1 == 0xE9) {
            // Patch JMP/MOD byte to FF /4 (-> 0x25).
            byte1 = x86EncodeMod(0, 4, 5);
          }
          else {
            return DebugUtils::errored(kErrorInvalidRelocEntry);
          }

          // Patch `jmp/call` instruction.
          buffer[valueOffset - 2] = uint8_t(byte0);
          buffer[valueOffset - 1] = uint8_t(byte1);

          Support::writeU64uLE(addressTableEntryData + atEntryIndex, re->payload());
        }
        break;
      }

      default:
        return DebugUtils::errored(kErrorInvalidRelocEntry);
    }

    if (!CodeWriterUtils::writeOffset(buffer + re->sourceOffset(), int64_t(value), re->format())) {
      return DebugUtils::errored(kErrorInvalidRelocEntry);
    }
  }

  // Fixup the virtual size of the address table if it's the last section.
  if (_sectionsByOrder.last() == addressTableSection) {
    ASMJIT_ASSERT(addressTableSection != nullptr);

    size_t addressTableSize = addressTableEntryCount * addressSize;
    addressTableSection->_buffer._size = addressTableSize;
    addressTableSection->_virtualSize = addressTableSize;
  }

  return kErrorOk;
}

Error CodeHolder::copySectionData(void* dst, size_t dstSize, uint32_t sectionId, CopySectionFlags copyFlags) noexcept {
  if (ASMJIT_UNLIKELY(!isSectionValid(sectionId)))
    return DebugUtils::errored(kErrorInvalidSection);

  Section* section = sectionById(sectionId);
  size_t bufferSize = section->bufferSize();

  if (ASMJIT_UNLIKELY(dstSize < bufferSize))
    return DebugUtils::errored(kErrorInvalidArgument);

  memcpy(dst, section->data(), bufferSize);

  if (bufferSize < dstSize && Support::test(copyFlags, CopySectionFlags::kPadSectionBuffer)) {
    size_t paddingSize = dstSize - bufferSize;
    memset(static_cast<uint8_t*>(dst) + bufferSize, 0, paddingSize);
  }

  return kErrorOk;
}

Error CodeHolder::copyFlattenedData(void* dst, size_t dstSize, CopySectionFlags copyFlags) noexcept {
  size_t end = 0;
  for (Section* section : _sectionsByOrder) {
    if (section->offset() > dstSize)
      return DebugUtils::errored(kErrorInvalidArgument);

    size_t bufferSize = section->bufferSize();
    size_t offset = size_t(section->offset());

    if (ASMJIT_UNLIKELY(dstSize - offset < bufferSize))
      return DebugUtils::errored(kErrorInvalidArgument);

    uint8_t* dstTarget = static_cast<uint8_t*>(dst) + offset;
    size_t paddingSize = 0;
    memcpy(dstTarget, section->data(), bufferSize);

    if (Support::test(copyFlags, CopySectionFlags::kPadSectionBuffer) && bufferSize < section->virtualSize()) {
      paddingSize = Support::min<size_t>(dstSize - offset, size_t(section->virtualSize())) - bufferSize;
      memset(dstTarget + bufferSize, 0, paddingSize);
    }

    end = Support::max(end, offset + bufferSize + paddingSize);
  }

  if (end < dstSize && Support::test(copyFlags, CopySectionFlags::kPadTargetBuffer)) {
    memset(static_cast<uint8_t*>(dst) + end, 0, dstSize - end);
  }

  return kErrorOk;
}

// CodeHolder - Tests
// ==================

#if defined(ASMJIT_TEST)
UNIT(code_holder) {
  CodeHolder code;

  INFO("Verifying CodeHolder::init()");
  Environment env;
  env.init(Arch::kX86);

  code.init(env);
  EXPECT(code.arch() == Arch::kX86);

  INFO("Verifying named labels");
  LabelEntry* le;
  EXPECT(code.newNamedLabelEntry(&le, "NamedLabel", SIZE_MAX, LabelType::kGlobal) == kErrorOk);
  EXPECT(strcmp(le->name(), "NamedLabel") == 0);
  EXPECT(code.labelIdByName("NamedLabel") == le->id());

  INFO("Verifying section ordering");
  Section* section1;
  EXPECT(code.newSection(&section1, "high-priority", SIZE_MAX, SectionFlags::kNone, 1, -1) == kErrorOk);
  EXPECT(code.sections()[1] == section1);
  EXPECT(code.sectionsByOrder()[0] == section1);

  Section* section0;
  EXPECT(code.newSection(&section0, "higher-priority", SIZE_MAX, SectionFlags::kNone, 1, -2) == kErrorOk);
  EXPECT(code.sections()[2] == section0);
  EXPECT(code.sectionsByOrder()[0] == section0);
  EXPECT(code.sectionsByOrder()[1] == section1);

  Section* section3;
  EXPECT(code.newSection(&section3, "low-priority", SIZE_MAX, SectionFlags::kNone, 1, 2) == kErrorOk);
  EXPECT(code.sections()[3] == section3);
  EXPECT(code.sectionsByOrder()[3] == section3);
}
#endif

ASMJIT_END_NAMESPACE