threaddb  2.0
A file mapped memory container extension
threaddbSTLWrapper.h
Go to the documentation of this file.
1 /*
2  Copyright (c) 2019 by The ThreadDB Project
3  All Rights Reserved.
4 
5  ThreadDB undergoes the BSD License 2.0. You should have received a copy along with this program; if not, write to the ThreadDB Project.
6  To obtain a full unlimited version contact thethreaddbproject(at)gmail.com.
7 
8  threaddbSTLWraper.h - standard library container support wrapper classes
9 */
10 
11 #pragma once
12 
13 #include "threaddbCPP.h"
14 #include <atomic>
15 #include <mutex>
16 #include <vector>
17 #include <deque>
18 #include <memory>
19 #include <stdio.h>
20 #include <string.h>
21 
33 namespace tdb
34 {
39  template<class T>
40  struct size_
41  {
42  size_() {}
43  size_(const T&) {}
44  uint32_t size() const { return sizeof(T); }
45  };
46 
51  template<>
52  struct size_<std::string>
53  {
54  size_() : m_size(0) {}
55  size_(const std::string& rS_p) : m_size((uint32_t)rS_p.size()) {}
56  uint32_t size() const { return m_size; }
57  uint32_t m_size;
58  };
59 
64  template<class T>
65  class tostream
66  {
67  public:
68  tostream(const T& rVal_p) :
69  m_cptr((const char*)&rVal_p)
70  {
71  }
72 
73  tostream(const T& rVal_p, uint32_t /*size_p*/) :
74  m_cptr((const char*)&rVal_p)
75  {
76  }
77 
78  const char* get() const
79  {
80  return m_cptr;
81  }
82 
83  uint32_t size() const
84  {
85  return sizeof(T);
86  }
87 
88  private:
89  const char* m_cptr;
90  };
91 
96  template<class T>
97  class fromstream
98  {
99  public:
101  {
102  }
103 
105  {
106  }
107 
108  fromstream(database* /*pDb_p*/, ReadInfo* /*pReadInfo_p*/)
109  {
110  }
111 
112  std::unique_ptr<T> get(database* pDb_p, ItemInfo* pItemInfo_p, uint64_t Package_p) const
113  {
114  std::unique_ptr<T> pT(new T());
115  pDb_p->Recover(sizeof(T), (char*)pT.get(), *pItemInfo_p, Package_p);
116  return pT;
117  }
118 
119  std::unique_ptr<T> get(database* pDb_p, ItemInfo* pItemInfo_p, uint64_t Package_p, size_<T> /*Size_p*/) const
120  {
121  return get(pDb_p, pItemInfo_p, Package_p);
122  }
123 
124  std::unique_ptr<T> get(database* pDb_p, ReadInfo* pReadInfo_p) const
125  {
126  std::unique_ptr<T> pT(new T());
127  pDb_p->Recover(sizeof(T), (char*)pT.get(), *pReadInfo_p);
128  return pT;
129  }
130 
131  std::unique_ptr<std::string> get() const
132  {
133  return std::unique_ptr<std::string>(new std::string());
134  }
135 
136  uint32_t size() const
137  {
138  return sizeof(T);
139  }
140  };
141 
146  template<>
147  class tostream<std::string>
148  {
149  public:
150  tostream(const std::string& rVal_p) :
151  m_size(sizeof(uint32_t) + (uint32_t)rVal_p.size()),
152  m_cptr((char*)malloc(m_size)),
153  m_alloc(true)
154  {
155  *((uint32_t*)m_cptr) = (uint32_t)rVal_p.size();
156  memcpy((void*)(m_cptr + sizeof(uint32_t)), (void*)rVal_p.c_str(), rVal_p.size());
157  }
158 
159  tostream(const std::string& rVal_p, size_<std::string> size_p) :
160  m_size(size_p.size()),
161  m_cptr(rVal_p.c_str()),
162  m_alloc(false)
163  {
164  }
165 
167  {
168  if (m_alloc)
169  free((void*)m_cptr);
170  }
171 
172  const char* get() const
173  {
174  return m_cptr;
175  }
176 
177  uint32_t size() const
178  {
179  return m_size;
180  }
181 
182  private:
183  const char* m_cptr;
184  uint32_t m_size;
185  bool m_alloc;
186  };
187 
192  template<>
193  class fromstream<std::string>
194  {
195  public:
197  m_size()
198  {
199  }
200 
202  m_size(size_p.size())
203  {
204  }
205 
206  fromstream(database* pDb_p, ReadInfo* pReadInfo_p) :
207  m_size()
208  {
209  pDb_p->Recover(sizeof(m_size), (char*)&m_size, *pReadInfo_p);
210  }
211 
212  std::unique_ptr<std::string> get(database* pDb_p, ItemInfo* pItemInfo_p, uint64_t Package_p) const
213  {
214  uint32_t size_ = m_size + sizeof(m_size);
215 
216  std::vector<char> cbuf(size_);
217  pDb_p->Recover(size_, (char*)&cbuf[0], *pItemInfo_p, Package_p);
218 
219  std::unique_ptr<std::string> pstr(new std::string());
220  (*pstr).insert((*pstr).begin(), &cbuf[sizeof(m_size)], &cbuf[size_]);
221 
222  return pstr;
223  }
224 
225  std::unique_ptr<std::string> get(database* pDb_p, ItemInfo* pItemInfo_p, uint64_t Package_p, size_<std::string> Size_p) const
226  {
227  std::unique_ptr<std::string> pstr(new std::string());
228  pstr->resize(Size_p.size());
229  pDb_p->Recover(Size_p.size(), (char*)pstr->c_str(), *pItemInfo_p, Package_p);
230  return pstr;
231  }
232 
233  std::unique_ptr<std::string> get(database* pDb_p, ReadInfo* pReadInfo_p) const
234  {
235  std::unique_ptr<std::string> pstr(new std::string());
236  pstr->resize(m_size);
237  pDb_p->Recover(m_size, (char*)pstr->c_str(), *pReadInfo_p);
238  return pstr;
239  }
240 
241  std::unique_ptr<std::string> get() const
242  {
243  return std::unique_ptr<std::string>(new std::string());
244  }
245 
246  uint32_t size() const
247  {
248  return m_size;
249  }
250 
251  private:
252  uint32_t m_size;
253  };
254 
255  template<class T>
256  class ritem;
257 
258  template<class T>
259  class sitem;
260 
261  typedef std::pair<ItemInfo, std::atomic<uint8_t>> ItemHandle;
262 
267  template<class T>
268  class pool
269  {
270  public:
271  pool(database& rDb_p, uint32_t ItemLimit_p = std::numeric_limits<uint32_t>::max()) :
272  m_DeleteLock(),
273  m_DeletedItems(),
274  m_rDb(rDb_p),
275  m_ItemIndex(0),
276  m_ItemLimit(ItemLimit_p),
277  m_ThreadCount((uint32_t)rDb_p.GetThreadCount()),
278  m_Pidx(new std::pair<uint64_t, uint32_t>[m_ThreadCount])
279  {
280  for (uint32_t idx(0); idx < m_ThreadCount; ++idx)
281  {
282  m_Pidx[idx] = std::make_pair(m_rDb.NewPackage(), 0);
283  }
284  }
285 
287  {
288  std::lock_guard<std::mutex> deleteLock(m_DeleteLock);
289  do
290  {
291  for (auto iter(m_DeletedItems.begin()); iter != m_DeletedItems.end(); )
292  {
293  if ((*iter).first->first.Unregistered())
294  {
295  delete (*iter).first;
296  iter = m_DeletedItems.erase(iter);
297  continue;
298  }
299  ++iter;
300  }
301 
302  if (!m_DeletedItems.empty())
303  {
304  m_rDb.Synchronize(m_DeletedItems.front().second);
305  }
306  } while (m_DeletedItems.empty() == false);
307 
308  delete[] m_Pidx;
309  }
310 
311  private:
312  friend class ritem<T>;
313  friend class sitem<T>;
314 
315  const std::pair<uint64_t, uint32_t>&
316  Store(uint32_t Size_p, const char* pData_p, ItemInfo* pItemInfo_p = 0)
317  {
318  std::pair<uint64_t, uint32_t>& rPidx(m_Pidx[increment()]);
319  m_rDb.Store(rPidx.first, Size_p, pData_p, pItemInfo_p);
320  return rPidx;
321  }
322 
323  ItemHandle* Alloc()
324  {
325  ItemHandle* pItem = 0;
326 
327  std::lock_guard<std::mutex> deleteLock(m_DeleteLock);
328 
329  auto iter = m_DeletedItems.begin();
330  if (iter != m_DeletedItems.end())
331  {
332  if ((*iter).first->first.m_PackageSize != std::numeric_limits<uint32_t>::max())
333  {
334  pItem = (*iter).first;
335  iter = m_DeletedItems.erase(iter);
336  }
337  }
338 
339  return pItem != 0 ? pItem : new ItemHandle();
340  }
341 
342  void Delete(ItemHandle* pItemInfo_p, uint64_t PackageID_p)
343  {
344  std::lock_guard<std::mutex> deleteLock(m_DeleteLock);
345  if (pItemInfo_p != 0)
346  {
347  if (pItemInfo_p->first.m_PackageSize != std::numeric_limits<uint32_t>::max())
348  m_DeletedItems.push_front(std::make_pair(pItemInfo_p, PackageID_p));
349  else
350  m_DeletedItems.push_back(std::make_pair(pItemInfo_p, PackageID_p));
351  }
352  }
353 
354  database& Db() { return m_rDb; }
355 
356  uint32_t increment()
357  {
358  ++m_ItemIndex;
359 
360  const uint32_t packageIdx = m_ItemIndex % m_ThreadCount;
361  std::pair<uint64_t, uint32_t>& rPackageIdx(m_Pidx[packageIdx]);
362 
363  ++(rPackageIdx.second);
364  if (rPackageIdx.second > m_ItemLimit)
365  {
366  rPackageIdx = std::make_pair(m_rDb.NewPackage(), 1);
367  }
368 
369  return packageIdx;
370  }
371 
372  std::mutex m_DeleteLock;
373  std::deque< std::pair<ItemHandle*, uint64_t> > m_DeletedItems;
374 
375  database& m_rDb;
376  uint32_t m_ItemIndex;
377  uint32_t m_ItemLimit;
378  uint32_t m_ThreadCount;
379  std::pair<uint64_t, uint32_t>* m_Pidx;
380  };
381 
386  template<class T>
387  class ritem
388  {
389  public:
390  typedef T value_type;
391 
392  ritem() :
393  m_T(),
394  m_P(),
395  m_Pidx(),
396  m_Size()
397  {
398  m_Pidx = 0;
399  }
400 
401  ritem(const T& rVal_p, pool<T>& rPool_p) :
402  m_T(rPool_p.Alloc()),
403  m_P(&rPool_p),
404  m_Pidx(),
405  m_Size(size_<T>(rVal_p))
406  {
407  ++m_T->second;
408  const tostream<T> byteStream(rVal_p, m_Size);
409  m_Pidx = m_P->Store(byteStream.size(), byteStream.get(), &m_T->first).first;
410  }
411 
412  ritem(const ritem& rhs_p) :
413  m_T(rhs_p.m_T),
414  m_P(rhs_p.m_P),
415  m_Pidx(rhs_p.m_Pidx),
416  m_Size(rhs_p.m_Size)
417  {
418  ++m_T->second;
419  }
420 
421  void operator=(const ritem &rhs_p)
422  {
423  deref();
424 
425  m_T = rhs_p.m_T;
426  ++m_T->second;
427  m_P = rhs_p.m_P;
428  m_Pidx = rhs_p.m_Pidx;
429  }
430 
432  {
433  deref();
434  }
435 
436  void deref()
437  {
438  --m_T->second;
439  if (m_T->second == 0)
440  {
441  m_P->Delete(m_T, m_Pidx);
442  m_T = 0;
443  }
444  }
445 
446  std::unique_ptr<T> get() const
447  {
448  return fromstream<T>(m_Size).get(&m_P->Db(), &m_T->first, m_Pidx, m_Size);
449  }
450 
451  protected:
453 
454  private:
455  pool<T>* m_P;
456  uint64_t m_Pidx;
457  size_<T> m_Size;
458  };
459 
464  template<class T>
465  class sitem
466  {
467  public:
468  typedef T value_type;
469 
470  sitem() :
471  m_P(),
472  m_Pidx()
473  {
474  }
475 
476  sitem(const T& rVal_p, pool<T>& rPool_p) :
477  m_P(&rPool_p),
478  m_Pidx()
479  {
480  const tostream<T> byteStream(rVal_p);
481  m_Pidx = m_P->Store(byteStream.size(), byteStream.get());
482  }
483 
484  void operator=(const sitem &rhs_p)
485  {
486  m_P = rhs_p.m_P;
487  m_Pidx = rhs_p.m_Pidx;
488  }
489 
490  std::unique_ptr<T> get() const
491  {
492  if (!m_P) fromstream<T>().get();
493 
494  m_P->Db().Synchronize(m_Pidx.first);
495 
496  ReadInfo readInfo = m_P->Db().Open(m_Pidx.first);
497  for (uint32_t idx(1); idx < m_Pidx.second; ++idx)
498  {
499  fromstream<T> fromStream(&m_P->Db(), &readInfo);
500  const uint32_t bytesToRead = fromStream.size();
501  if (m_P->Db().Recover(bytesToRead, 0, readInfo) != bytesToRead)
502  throw std::runtime_error("Invalid read operation in item data recovery.");
503  }
504  return fromstream<T>(&m_P->Db(), &readInfo).get(&m_P->Db(), &readInfo);
505  }
506 
507  private:
508  pool<T>* m_P;
509  std::pair<uint64_t, uint32_t> m_Pidx;
510  };
511 
516  template<class T>
517  class key : public T
518  {
519  public:
520  key(const typename T::value_type& rVal_p, pool<typename T::value_type>& rPool_p) :
521  T(rVal_p, rPool_p)
522  {
523  }
524 
526  {
527  }
528 
529  bool operator<(const T& rhs_p) const
530  {
531  return (*this->get()) < (*rhs_p.get());
532  }
533 
534  bool operator>(const T& rhs_p) const
535  {
536  return ((*rhs_p.get() < *this->get()));
537  }
538  };
539 }
tdb::ritem::value_type
T value_type
Definition: threaddbSTLWrapper.h:390
tdb::sitem::get
std::unique_ptr< T > get() const
Definition: threaddbSTLWrapper.h:490
tdb::size_< std::string >::size_
size_(const std::string &rS_p)
Definition: threaddbSTLWrapper.h:55
tdb::fromstream< std::string >::get
std::unique_ptr< std::string > get(database *pDb_p, ItemInfo *pItemInfo_p, uint64_t Package_p) const
Definition: threaddbSTLWrapper.h:212
tdb::ReadInfo
C++ wrapper class to the threadDB_ReadInfo control structure.
Definition: threaddbCPP.h:70
tdb::fromstream::get
std::unique_ptr< T > get(database *pDb_p, ReadInfo *pReadInfo_p) const
Definition: threaddbSTLWrapper.h:124
tdb::fromstream< std::string >::get
std::unique_ptr< std::string > get(database *pDb_p, ItemInfo *pItemInfo_p, uint64_t Package_p, size_< std::string > Size_p) const
Definition: threaddbSTLWrapper.h:225
tdb::fromstream< std::string >::fromstream
fromstream()
Definition: threaddbSTLWrapper.h:196
tdb::ItemInfo
C++ wrapper class to the threadDB_ItemInfo control structure.
Definition: threaddbCPP.h:52
tdb::tostream< std::string >::tostream
tostream(const std::string &rVal_p, size_< std::string > size_p)
Definition: threaddbSTLWrapper.h:159
threaddbCPP.h
Interface C++ wrapper classes.
tdb::fromstream::fromstream
fromstream()
Definition: threaddbSTLWrapper.h:100
tdb::pool::pool
pool(database &rDb_p, uint32_t ItemLimit_p=std::numeric_limits< uint32_t >::max())
Definition: threaddbSTLWrapper.h:271
tdb::key::key
key(const typename T::value_type &rVal_p, pool< typename T::value_type > &rPool_p)
Definition: threaddbSTLWrapper.h:520
tdb::ritem::ritem
ritem(const T &rVal_p, pool< T > &rPool_p)
Definition: threaddbSTLWrapper.h:401
tdb::ritem::deref
void deref()
Definition: threaddbSTLWrapper.h:436
tdb::key
Wrapper class for storing key data items.
Definition: threaddbSTLWrapper.h:517
tdb::sitem::operator=
void operator=(const sitem &rhs_p)
Definition: threaddbSTLWrapper.h:484
tdb::tostream
Converter of fixed size data elements into a byte stream representation.
Definition: threaddbSTLWrapper.h:65
tdb::tostream< std::string >::tostream
tostream(const std::string &rVal_p)
Definition: threaddbSTLWrapper.h:150
tdb::database::Store
void Store(uint64_t Package_p, size_t Size_p, const char pData_p[], threadDB_ItemInfo *pItemHandle_p=0)
C++ wrapper to store a data item in the selected package see ThreadDB_Store.
Definition: threaddbCPP.h:234
tdb
Definition: threaddbCPP.h:28
tdb::sitem::value_type
T value_type
Definition: threaddbSTLWrapper.h:468
tdb::size_::size_
size_()
Definition: threaddbSTLWrapper.h:42
tdb::ritem::m_T
ItemHandle * m_T
Definition: threaddbSTLWrapper.h:452
tdb::tostream::tostream
tostream(const T &rVal_p, uint32_t)
Definition: threaddbSTLWrapper.h:73
tdb::database::Synchronize
void Synchronize(uint64_t Package_p)
C++ wrapper to synchronize the package buffers prior reading see ThreadDB_Synchronize.
Definition: threaddbCPP.h:243
tdb::sitem
Wrapper class for storing data items in a sequential fashion.
Definition: threaddbSTLWrapper.h:259
tdb::fromstream
Converter of byte stream into fixed data item representation.
Definition: threaddbSTLWrapper.h:97
tdb::tostream::tostream
tostream(const T &rVal_p)
Definition: threaddbSTLWrapper.h:68
tdb::fromstream< std::string >::size
uint32_t size() const
Definition: threaddbSTLWrapper.h:246
tdb::sitem::sitem
sitem()
Definition: threaddbSTLWrapper.h:470
tdb::size_::size_
size_(const T &)
Definition: threaddbSTLWrapper.h:43
tdb::tostream< std::string >::get
const char * get() const
Definition: threaddbSTLWrapper.h:172
tdb::fromstream::size
uint32_t size() const
Definition: threaddbSTLWrapper.h:136
tdb::sitem::sitem
sitem(const T &rVal_p, pool< T > &rPool_p)
Definition: threaddbSTLWrapper.h:476
tdb::fromstream::get
std::unique_ptr< std::string > get() const
Definition: threaddbSTLWrapper.h:131
tdb::ritem::get
std::unique_ptr< T > get() const
Definition: threaddbSTLWrapper.h:446
tdb::fromstream< std::string >::get
std::unique_ptr< std::string > get(database *pDb_p, ReadInfo *pReadInfo_p) const
Definition: threaddbSTLWrapper.h:233
tdb::size_< std::string >::size_
size_()
Definition: threaddbSTLWrapper.h:54
tdb::key::operator>
bool operator>(const T &rhs_p) const
Definition: threaddbSTLWrapper.h:534
tdb::fromstream::fromstream
fromstream(size_< std::string > size_p)
Definition: threaddbSTLWrapper.h:104
tdb::ritem::~ritem
~ritem()
Definition: threaddbSTLWrapper.h:431
tdb::size_< std::string >::m_size
uint32_t m_size
Definition: threaddbSTLWrapper.h:57
tdb::fromstream< std::string >::fromstream
fromstream(size_< std::string > size_p)
Definition: threaddbSTLWrapper.h:201
tdb::key::~key
~key()
Definition: threaddbSTLWrapper.h:525
tdb::size_::size
uint32_t size() const
Definition: threaddbSTLWrapper.h:44
tdb::ritem::ritem
ritem()
Definition: threaddbSTLWrapper.h:392
tdb::ritem::ritem
ritem(const ritem &rhs_p)
Definition: threaddbSTLWrapper.h:412
tdb::ItemHandle
std::pair< ItemInfo, std::atomic< uint8_t > > ItemHandle
Definition: threaddbSTLWrapper.h:259
tdb::database
C++ wrapper class of the threadDB file mapped memory container extension.
Definition: threaddbCPP.h:97
tdb::tostream< std::string >::size
uint32_t size() const
Definition: threaddbSTLWrapper.h:177
tdb::pool
Pool datastructure for keeping track and recycling open ItemHandles.
Definition: threaddbSTLWrapper.h:268
tdb::ritem::operator=
void operator=(const ritem &rhs_p)
Definition: threaddbSTLWrapper.h:421
tdb::fromstream::get
std::unique_ptr< T > get(database *pDb_p, ItemInfo *pItemInfo_p, uint64_t Package_p) const
Definition: threaddbSTLWrapper.h:112
tdb::database::Recover
size_t Recover(size_t Size_p, char pData_p[], tdb::ReadInfo &rReadInfo_p)
C++ wrapper for stream reading of data items see ThreadDB_RecoverContinous.
Definition: threaddbCPP.h:289
tdb::tostream::size
uint32_t size() const
Definition: threaddbSTLWrapper.h:83
tdb::fromstream::fromstream
fromstream(database *, ReadInfo *)
Definition: threaddbSTLWrapper.h:108
tdb::fromstream< std::string >::fromstream
fromstream(database *pDb_p, ReadInfo *pReadInfo_p)
Definition: threaddbSTLWrapper.h:206
tdb::database::NewPackage
uint64_t NewPackage()
C++ wrapper to create and register a threadDB data package see ThreadDB_NewPackage.
Definition: threaddbCPP.h:169
tdb::size_< std::string >
Trait class for size of variable length data items (std::string).
Definition: threaddbSTLWrapper.h:52
tdb::tostream< std::string >::~tostream
~tostream()
Definition: threaddbSTLWrapper.h:166
tdb::fromstream::get
std::unique_ptr< T > get(database *pDb_p, ItemInfo *pItemInfo_p, uint64_t Package_p, size_< T >) const
Definition: threaddbSTLWrapper.h:119
tdb::fromstream< std::string >::get
std::unique_ptr< std::string > get() const
Definition: threaddbSTLWrapper.h:241
tdb::tostream::get
const char * get() const
Definition: threaddbSTLWrapper.h:78
tdb::ritem
Wrapper class for storing data items in a random fashion.
Definition: threaddbSTLWrapper.h:256
tdb::pool::~pool
~pool()
Definition: threaddbSTLWrapper.h:286
tdb::key::operator<
bool operator<(const T &rhs_p) const
Definition: threaddbSTLWrapper.h:529
tdb::size_< std::string >::size
uint32_t size() const
Definition: threaddbSTLWrapper.h:56
tdb::size_
Trait class for size of fixed length data items.
Definition: threaddbSTLWrapper.h:40