ceph Buffer#yyds干货盘点#

class buffer::raw { public: char *data; //数据指针 unsigned len; //数据长度 atomic_t nref; //引用计数

mutable simple_spinlock_t crc_spinlock; //读写锁，保护 crc_map map<pair<size_t, size_t>, pair<uint32_t, uint32_t> > crc_map; //crc 校验信息，第一个pair为数据段的起始和结束（from，to），第二个pair是crc32校验码，pair的第一个字段为base crc32 校验码，第二个字段为加上数据段后计算出的crc32 校验码

explicit raw(unsigned l) : data(NULL), len(l), nref(0), crc_spinlock(SIMPLE_SPINLOCK_INITIALIZER) { } raw(char *c, unsigned l) : data(c), len(l), nref(0), crc_spinlock(SIMPLE_SPINLOCK_INITIALIZER) { } virtual ~raw() {}

// no copying. // cppcheck-suppress noExplicitConstructor raw(const raw &other); const raw& operator=(const raw &other);

virtual char *get_data() { return data; } virtual raw* clone_empty() = 0; raw *clone() { raw *c = clone_empty(); memcpy(c->data, data, len); return c; } virtual bool can_zero_copy() const { return false; } virtual int zero_copy_to_fd(int fd, loff_t *offset) { return -ENOTSUP; } virtual bool is_page_aligned() { return ((long)data & ~CEPH_PAGE_MASK) == 0; } bool is_n_page_sized() { return (len & ~CEPH_PAGE_MASK) == 0; } virtual bool is_shareable() { // true if safe to reference/share the existing buffer copy // false if it is not safe to share the buffer, e.g., due to special // and/or registered memory that is scarce return true; } bool get_crc(const pair<size_t, size_t> &fromto, pair<uint32_t, uint32_t> *crc) const { simple_spin_lock(&crc_spinlock); map<pair<size_t, size_t>, pair<uint32_t, uint32_t> >::const_iterator i = crc_map.find(fromto); if (i == crc_map.end()) { simple_spin_unlock(&crc_spinlock); return false; } *crc = i->second; simple_spin_unlock(&crc_spinlock); return true; } void set_crc(const pair<size_t, size_t> &fromto, const pair<uint32_t, uint32_t> &crc) { simple_spin_lock(&crc_spinlock); crc_map[fromto] = crc; simple_spin_unlock(&crc_spinlock); } void invalidate_crc() { simple_spin_lock(&crc_spinlock); if (crc_map.size() != 0) { crc_map.clear(); } simple_spin_unlock(&crc_spinlock); } };

class CEPH_BUFFER_API ptr { raw *_raw; unsigned _off, _len;

void release();

public: ptr() : _raw(0), _off(0), _len(0) {} // cppcheck-suppress noExplicitConstructor ptr(raw *r); // cppcheck-suppress noExplicitConstructor ptr(unsigned l); ptr(const char *d, unsigned l); ptr(const ptr& p); ptr(ptr&& p); ptr(const ptr& p, unsigned o, unsigned l); ptr& operator= (const ptr& p); ptr& operator= (ptr&& p); ~ptr() { release(); }

bool have_raw() const { return _raw ? true:false; }

raw *clone(); void swap(ptr& other); ptr& make_shareable();

class CEPH_BUFFER_API list { // my private bits std::list<ptr> _buffers; unsigned _len; unsigned _memcopy_count; //the total of memcopy using rebuild(). ptr append_buffer; // where i put small appends.

public: class iterator;

private: template <bool is_const> class CEPH_BUFFER_API iterator_impl : public std::iterator<std::forward_iterator_tag, char> { protected: typedef typename std::conditional<is_const, const list, list>::type bl_t; typedef typename std::conditional<is_const, const std::list<ptr>, std::list<ptr> >::type list_t; typedef typename std::conditional<is_const, typename std::list<ptr>::const_iterator, typename std::list<ptr>::iterator>::type list_iter_t; bl_t* bl; list_t* ls; // meh.. just here to avoid an extra pointer dereference.. unsigned off; // in bl list_iter_t p; unsigned p_off; // in *p friend class iterator_impl<true>;

public: // constructor. position. iterator_impl() : bl(0), ls(0), off(0), p_off(0) {} iterator_impl(bl_t *l, unsigned o=0); iterator_impl(bl_t *l, unsigned o, list_iter_t ip, unsigned po) : bl(l), ls(&bl->_buffers), off(o), p(ip), p_off(po) {} iterator_impl(const list::iterator& i);

/// get current iterator offset in buffer::list unsigned get_off() const { return off; } /// get number of bytes remaining from iterator position to the end of the buffer::list unsigned get_remaining() const { return bl->length() - off; }

/// true if iterator is at the end of the buffer::list bool end() const { return p == ls->end(); //return off == bl->length(); }

void advance(int o); void seek(unsigned o); char operator*() const; iterator_impl& operator++(); ptr get_current_ptr() const;

bl_t& get_bl() const { return *bl; }

// copy data out. // note that these all _append_ to dest! void copy(unsigned len, char *dest); void copy(unsigned len, ptr &dest); void copy(unsigned len, list &dest); void copy(unsigned len, std::string &dest); void copy_all(list &dest);

friend bool operator==(const iterator_impl& lhs, const iterator_impl& rhs) { return &lhs.get_bl() == &rhs.get_bl() && lhs.get_off() == rhs.get_off(); } friend bool operator!=(const iterator_impl& lhs, const iterator_impl& rhs) { return &lhs.get_bl() != &rhs.get_bl() || lhs.get_off() != rhs.get_off(); } };

public: typedef iterator_impl<true> const_iterator;

class CEPH_BUFFER_API iterator : public iterator_impl<false> { public: iterator() = default; iterator(bl_t *l, unsigned o=0); iterator(bl_t *l, unsigned o, list_iter_t ip, unsigned po);

void advance(int o); void seek(unsigned o); char operator*(); iterator& operator++(); ptr get_current_ptr();

// copy data out void copy(unsigned len, char *dest); void copy(unsigned len, ptr &dest); void copy(unsigned len, list &dest); void copy(unsigned len, std::string &dest); void copy_all(list &dest);

// copy data in void copy_in(unsigned len, const char *src); void copy_in(unsigned len, const char *src, bool crc_reset); void copy_in(unsigned len, const list& otherl);

bool operator==(const iterator& rhs) const { return bl == rhs.bl && off == rhs.off; } bool operator!=(const iterator& rhs) const { return bl != rhs.bl || off != rhs.off; } };

private: mutable iterator last_p; int zero_copy_to_fd(int fd) const;

public: // cons/des list() : _len(0), _memcopy_count(0), last_p(this) {} // cppcheck-suppress noExplicitConstructor list(unsigned prealloc) : _len(0), _memcopy_count(0), last_p(this) { append_buffer = buffer::create(prealloc); append_buffer.set_length(0); // unused, so far. }

list(const list& other) : _buffers(other._buffers), _len(other._len), _memcopy_count(other._memcopy_count), last_p(this) { make_shareable(); } list(list&& other); list& operator= (const list& other) { if (this != &other) { _buffers = other._buffers; _len = other._len; make_shareable(); } return *this; }

list& operator= (list&& other) { _buffers = std::move(other._buffers); _len = other._len; _memcopy_count = other._memcopy_count; last_p = begin(); append_buffer.swap(other.append_buffer); other.clear(); return *this; }

unsigned get_num_buffers() const { return _buffers.size(); } const ptr& front() const { return _buffers.front(); } const ptr& back() const { return _buffers.back(); }

unsigned get_memcopy_count() const {return _memcopy_count; } const std::list<ptr>& buffers() const { return _buffers; } void swap(list& other); unsigned length() const {#if 0 // DEBUG: verify _len unsigned len = 0; for (std::list<ptr>::const_iterator it = _buffers.begin(); it != _buffers.end(); it++) { len += (*it).length(); } assert(len == _len);#endif return _len; }

bool contents_equal(buffer::list& other); bool contents_equal(const buffer::list& other) const;

bool can_zero_copy() const; bool is_provided_buffer(const char *dst) const; bool is_aligned(unsigned align) const; bool is_page_aligned() const; bool is_n_align_sized(unsigned align) const; bool is_n_page_sized() const;

bool is_zero() const;

// modifiers void clear() { _buffers.clear(); _len = 0; _memcopy_count = 0; last_p = begin(); } void push_front(ptr& bp) { if (bp.length() == 0) return; _buffers.push_front(bp); _len += bp.length(); } void push_front(ptr&& bp) { if (bp.length() == 0) return; _len += bp.length(); _buffers.push_front(std::move(bp)); } void push_front(raw *r) { push_front(ptr(r)); } void push_back(const ptr& bp) { if (bp.length() == 0) return; _buffers.push_back(bp); _len += bp.length(); } void push_back(ptr&& bp) { if (bp.length() == 0) return; _len += bp.length(); _buffers.push_back(std::move(bp)); } void push_back(raw *r) { push_back(ptr(r)); }

void zero(); void zero(unsigned o, unsigned l);

bool is_contiguous() const; void rebuild(); void rebuild(ptr& nb); void rebuild_aligned(unsigned align); void rebuild_aligned_size_and_memory(unsigned align_size, unsigned align_memory); void rebuild_page_aligned();

// assignment-op with move semantics const static unsigned int CLAIM_DEFAULT = 0; const static unsigned int CLAIM_ALLOW_NONSHAREABLE = 1;

void claim(list& bl, unsigned int flags = CLAIM_DEFAULT); void claim_append(list& bl, unsigned int flags = CLAIM_DEFAULT); void claim_prepend(list& bl, unsigned int flags = CLAIM_DEFAULT);

// clone non-shareable buffers (make shareable) void make_shareable() { std::list<buffer::ptr>::iterator pb; for (pb = _buffers.begin(); pb != _buffers.end(); ++pb) { (void) pb->make_shareable(); } }

// copy with explicit volatile-sharing semantics void share(const list& bl) { if (this != &bl) { clear(); std::list<buffer::ptr>::const_iterator pb; for (pb = bl._buffers.begin(); pb != bl._buffers.end(); ++pb) { push_back(*pb); } } }

iterator begin() { return iterator(this, 0); } iterator end() { return iterator(this, _len, _buffers.end(), 0); }

const_iterator begin() const { return const_iterator(this, 0); } const_iterator end() const { return const_iterator(this, _len, _buffers.end(), 0); }

// crope lookalikes. // **** WARNING: this are horribly inefficient for large bufferlists. **** void copy(unsigned off, unsigned len, char *dest) const; void copy(unsigned off, unsigned len, list &dest) const; void copy(unsigned off, unsigned len, std::string& dest) const; void copy_in(unsigned off, unsigned len, const char *src); void copy_in(unsigned off, unsigned len, const char *src, bool crc_reset); void copy_in(unsigned off, unsigned len, const list& src);

void append(char c); void append(const char *data, unsigned len); void append(const std::string& s) { append(s.data(), s.length()); } void append(const ptr& bp); void append(ptr&& bp); void append(const ptr& bp, unsigned off, unsigned len); void append(const list& bl); void append(std::istream& in); void append_zero(unsigned len); /* * get a char */ const char& operator[](unsigned n) const; char *c_str(); std::string to_str() const;

void substr_of(const list& other, unsigned off, unsigned len);

/// return a pointer to a contiguous extent of the buffer, /// reallocating as needed char *get_contiguous(unsigned off, ///< offset unsigned len); ///< length

// funky modifer void splice(unsigned off, unsigned len, list *claim_by=0 /*, bufferlist& replace_with */); void write(int off, int len, std::ostream& out) const;

void encode_base64(list& o); void decode_base64(list& o);

void write_stream(std::ostream &out) const; void hexdump(std::ostream &out) const; int read_file(const char *fn, std::string *error); ssize_t read_fd(int fd, size_t len); int read_fd_zero_copy(int fd, size_t len); int write_file(const char *fn, int mode=0644); int write_fd(int fd) const; int write_fd(int fd, uint64_t offset) const; int write_fd_zero_copy(int fd) const; void prepare_iov(std::vector<iovec> *piov) const; uint32_t crc32c(uint32_t crc) const; void invalidate_crc(); }

void push_front(ptr&& bp) { if (bp.length() == 0) return; _len += bp.length(); _buffers.push_front(std::move(bp)); }

void push_front(raw *r) { push_front(ptr(r)); }

template <typename T>bool is_aligned(T* ptr) { const auto align_mask = alignof(T) - 1; return (reinterpret_cast<uintptr_t>(ptr) & align_mask) == 0;}