Attachment 'sbuf-mempool-failed-integration.patch'

Download

   1 === modified file 'src/Mem.h'
   2 --- src/Mem.h	2009-01-21 03:47:47 +0000
   3 +++ src/Mem.h	2009-03-02 15:16:26 +0000
   4 @@ -56,4 +56,9 @@
   5      static void RegisterWithCacheManager(void);
   6  };
   7  
   8 +/**
   9 + * The system page size, as detected at runtime.
  10 + */
  11 +extern const size_t squidSystemPageSize;
  12 +
  13  #endif /* SQUID_MEM */
  14 
  15 === modified file 'src/MemBlob.cc'
  16 --- src/MemBlob.cc	2009-02-26 17:23:08 +0000
  17 +++ src/MemBlob.cc	2009-03-02 17:08:39 +0000
  18 @@ -61,6 +61,7 @@
  19   */
  20  MemBlob::MemBlob(MemBlob::size_type size)
  21  {
  22 +    debugs(MEMBLOB_DEBUGSECTION,9,HERE << "size=" << size);
  23      memAlloc(size);
  24  }
  25  
  26 @@ -68,31 +69,20 @@
  27  /**
  28   * Importer from the unmanaged world. Takes as argument a char* and a length,
  29   * and builds a managed MemBlob out of those.
  30 - * If isLiteral is true, then the imported block will not be freed by the
  31 - * destructor.
  32   */
  33  MemBlob::MemBlob(char * memBlock,
  34 -    MemBlob::size_type memSize, bool literal)
  35 -    :mem(memBlock), bufSize(memSize), bufUsed(memSize), isLiteral(literal),
  36 -     isImported(false)
  37 +    MemBlob::size_type memSize)
  38  {
  39 +    debugs(MEMBLOB_DEBUGSECTION,9,HERE << "importing" << memBlock);
  40 +    memAlloc(memSize);
  41 +    append(memBlock,memSize);
  42  }
  43  
  44  MemBlob::~MemBlob()
  45  {
  46      debugs(MEMBLOB_DEBUGSECTION,DBG_DATA,MYNAME << "this=" << (void*) this);
  47      --stats.live;
  48 -    if (!isLiteral) {
  49 -        if (isImported) {
  50 -            debugs(MEMBLOB_DEBUGSECTION,DBG_DATA,HERE << "isImported. Freeing");
  51 -            xfree(mem);
  52 -        } else {
  53 -            debugs(MEMBLOB_DEBUGSECTION,DBG_DATA,HERE << "mempool-freeing");
  54 -            memFreeString(bufSize,mem);
  55 -        }
  56 -    } else {
  57 -        debugs(MEMBLOB_DEBUGSECTION,DBG_DATA,HERE << "isLiteral. Not freeing");
  58 -    }
  59 +    memFreeString(bufSize,mem);
  60  }
  61  
  62  #ifndef _USE_INLINE_
  63 
  64 === modified file 'src/MemBlob.cci'
  65 --- src/MemBlob.cci	2009-02-25 15:46:15 +0000
  66 +++ src/MemBlob.cci	2009-03-02 16:48:13 +0000
  67 @@ -41,9 +41,9 @@
  68      Must(mem);
  69      bufSize=actualAlloc;
  70      bufUsed=0;
  71 -    isLiteral=false;
  72      debugs(MEMBLOB_DEBUGSECTION,8,
  73 -        MYNAME << "this=" <<(void *)this <<"requested="<<size);
  74 +        HERE << "this=" <<(void *)this <<", requested=" << size <<
  75 +        ", received=" << bufSize);
  76      ++stats.live;
  77      ++stats.alloc;
  78  }
  79 
  80 === modified file 'src/MemBlob.h'
  81 --- src/MemBlob.h	2009-02-26 17:23:08 +0000
  82 +++ src/MemBlob.h	2009-03-02 15:39:32 +0000
  83 @@ -34,6 +34,7 @@
  84  
  85  #define MEMBLOB_DEBUGSECTION 24
  86  
  87 +#define FORCEDEBUG 1
  88  #include "config.h"
  89  #include "RefCount.h"
  90  #include "protos.h"
  91 @@ -72,8 +73,6 @@
  92      char *mem;         /// The block of memory
  93      size_type bufSize; /// Size of the allocated memory
  94      size_type bufUsed; /// How much of the memory is actually used
  95 -    bool isLiteral;    /// a literal will not be freed upon destruction
  96 -    bool isImported;   /// an imported string, will not be freed via mempools
  97  
  98      //do not meddle with. Use getStats to obtain a copy for dumping
  99      static MemBlobStats stats;
 100 @@ -87,7 +86,7 @@
 101      public:
 102      MemBlob(size_type size);
 103  
 104 -    MemBlob(char * memBlock, size_type memSize, bool literal=false);
 105 +    MemBlob(char * memBlock, size_type memSize);
 106  
 107      ~MemBlob();
 108  
 109 
 110 === modified file 'src/SBuf.cc'
 111 --- src/SBuf.cc	2009-02-26 17:23:08 +0000
 112 +++ src/SBuf.cc	2009-03-02 17:12:12 +0000
 113 @@ -41,11 +41,6 @@
 114  MemBlob::Pointer SBuf::storePrototype=0;
 115  
 116  
 117 -#if 0
 118 -#undef debugs
 119 -#define debugs(X,Y,Z) do { std::cerr << Z << std::endl ;} while (0);
 120 -#endif
 121 -
 122  SBufStats::SBufStats()
 123      : alloc(0), live(0), qset(0), sset(0), qget(0), sget(0)
 124  {}
 125 @@ -167,9 +162,11 @@
 126          debugs(SBUF_DEBUGSECTION, 7, HERE << "not growing");
 127      }
 128      MemBlob::Pointer newbuf=new MemBlob(estimateCapacity(minsize));
 129 -    newbuf->append(buf(),length());
 130 +    if (length()>0)
 131 +        newbuf->append(buf(),length());
 132      store_=newbuf;
 133      off_=0;
 134 +    // len_ remains the same after the realloc
 135      ++MemBlob::stats.realloc;
 136      ++nreallocs;
 137      debugs(SBUF_DEBUGSECTION, 7, HERE << "new store capacity: " << store_->bufSize);
 138 @@ -186,7 +183,7 @@
 139      store_=getStorePrototype(); //uncomment to actually free storage upon clear()
 140  #else
 141      //enabling this code path, we try to release the store without deallocating it.
 142 -    if (store_->RefCountCount() == 1 && !store_->isLiteral)
 143 +    if (store_->RefCountCount() == 1)
 144          store_->bufUsed=0;
 145  #endif
 146      len_=0;
 147 @@ -342,8 +339,8 @@
 148  
 149          //we can assume that we'll need to append at least strlen(fmt) bytes,
 150          //times 1.2 for instance...
 151 -    if (!store_->canAppend(bufEnd(),strlen(fmt)*12/10)) {
 152 -        grow(length()+(strlen(fmt)*12/10));
 153 +    if (!store_->canAppend(bufEnd(),strlen(fmt)*2)) {
 154 +        grow(length()+(strlen(fmt)*2));
 155      }
 156  
 157      while (length() <= max_kbuf_size) {
 158 @@ -364,7 +361,7 @@
 159          /* snprintf on FreeBSD returns at least free_space on overflows */
 160  
 161          if (sz < 0 || sz >= (int)store_->freeSpace())
 162 -            grow(length()*12/10); ///TODO: understand how to best grow
 163 +            grow(length()+sz*2); /// \todo tune heuristics
 164          else
 165              break;
 166      }
 167 @@ -579,7 +576,6 @@
 168      if (len==npos)
 169          len=strlen(cstr);
 170  	store_=new MemBlob(cstr,len); //everything's used as far as we know
 171 -	store_->isImported=true;
 172  	off_=0;
 173  	len_=len;
 174  	nreallocs=0;
 175 @@ -588,24 +584,6 @@
 176  }
 177  
 178  /**
 179 - * Factory method: import a literal NULL-terminated C-string.
 180 - * The data ARE NOT COPIED and WILL NOT BE FREED when the SBuf is destroyed.
 181 - * The ONLY USE for this function is to construct a SBuf out of a C String literal.
 182 - * \example SBuf foo=SBuf::importLiteral("bar");
 183 - *
 184 - */
 185 -SBuf SBuf::importLiteral(const char *cstr)
 186 -{
 187 -    SBuf rv;
 188 -    rv.store_=new MemBlob((char*)cstr,strlen(cstr),true);
 189 -    rv.off_=0;
 190 -    rv.len_=strlen(cstr);
 191 -    rv.nreallocs=0;
 192 -    ++stats.qset;
 193 -    return rv;
 194 -}
 195 -
 196 -/**
 197   * exports the SBuf by copying the string. Freeing the returned
 198   * char* is up to the caller, which MUST free() it.
 199   * The returned string's contents are
 200 @@ -928,12 +906,6 @@
 201      return rv;
 202  }
 203  
 204 -/**
 205 - * System pagesize, detected at startup-time.
 206 - * Used for various memory management optimizations.
 207 - */
 208 -const size_t squidSystemPageSize=getpagesize();
 209 -
 210  #ifndef _USE_INLINE_
 211  #include "SBuf.cci"
 212  #endif
 213 
 214 === modified file 'src/SBuf.cci'
 215 --- src/SBuf.cci	2009-02-26 17:23:08 +0000
 216 +++ src/SBuf.cci	2009-03-02 15:15:49 +0000
 217 @@ -120,7 +120,7 @@
 218  bool SBuf::cow()
 219  {
 220      debugs(SBUF_DEBUGSECTION,DBG_DATA,MYNAME);
 221 -	if (store_->RefCountCount() == 1 && !store_->isLiteral) {
 222 +	if (store_->RefCountCount() == 1) {
 223  	    debugs(SBUF_DEBUGSECTION,DBG_DATA,"no cow needed");
 224  		return false;
 225  	}
 226 @@ -132,35 +132,14 @@
 227   * Try to guesstimate how big a MemBlob to allocate.
 228   * The result is guarranteed to be bigger than the desired
 229   * size.
 230 + * \todo improve heuristics
 231   */
 232  SBuf::size_type SBuf::estimateCapacity (SBuf::size_type desired)
 233  {
 234      size_type given;
 235 -    // grow to leave headroom, using the history of the membuf to
 236 -    // define how much headroom to have.
 237 -    if (nreallocs < 50)
 238 -        given = (desired*12)/10;
 239 -    else if (nreallocs < 100)
 240 -        given = desired * 2;
 241 -    else if (nreallocs < 200)
 242 -        given = desired * 4;
 243 -    else
 244 -        given = desired * 8;
 245 -        // block allocation strategy in steps: 512b, 1k, 2k, 4k, next 4k
 246 -        // multiple minus malloc_overhead
 247 -    given+=malloc_overhead;
 248 -    if (given <= 128)
 249 -        given=128;
 250 -    else if (given <= 512)
 251 -        given=512;
 252 -    else if (given <= 1024)
 253 -        given=1024;
 254 -    else if (given <= 2048)
 255 -        given=2048;
 256 -    else if (given <= 4096)
 257 -        given=4096;
 258 -    else
 259 -        given=roundto(given,squidSystemPageSize);
 260 +
 261 +    given=(desired*12)/10;
 262 +
 263      debugs(SBUF_DEBUGSECTION,5,
 264             MYNAME << "desired=" << desired << "returned=" << given);
 265      return given;
 266 @@ -210,7 +189,7 @@
 267  MemBlob::Pointer SBuf::getStorePrototype()
 268  {
 269      if (storePrototype==0)
 270 -        storePrototype = new MemBlob((char*)"",0,true);
 271 +        storePrototype = new MemBlob((char*)"",0);
 272      return storePrototype;
 273  }
 274  
 275 
 276 === modified file 'src/SBuf.h'
 277 --- src/SBuf.h	2009-02-26 17:23:08 +0000
 278 +++ src/SBuf.h	2009-03-02 15:39:19 +0000
 279 @@ -45,6 +45,7 @@
 280  
 281  #define SBUF_DEBUGSECTION 24
 282  
 283 +
 284  /* squid string placeholder (for printf) */
 285  #ifndef SQUIDSTRINGPH
 286  #define SQUIDSTRINGPH "%.*s"
 287 @@ -71,10 +72,6 @@
 288      SBufStats();
 289  };
 290  
 291 -/**
 292 - * The system page size, as detected at runtime.
 293 - */
 294 -extern const size_t squidSystemPageSize;
 295  
 296  /**
 297   * String-Buffer hybrid class (UniversalBuffer)
 298 @@ -145,7 +142,6 @@
 299      void terminate();
 300  
 301      SBuf& importCString(char * cstr, size_type len=npos);
 302 -    static SBuf importLiteral(const char *cstr);
 303  
 304      char* exportCopy() const;
 305      size_type copy(char *buf, size_type buflen) const;
 306 
 307 === modified file 'src/mem.cc'
 308 --- src/mem.cc	2009-01-21 03:47:47 +0000
 309 +++ src/mem.cc	2009-03-02 17:16:14 +0000
 310 @@ -58,24 +58,34 @@
 311  static double xm_deltat = 0;
 312  
 313  /* string pools */
 314 -#define mem_str_pool_count 3
 315 +#define mem_str_pool_count 6
 316  
 317  static const struct {
 318      const char *name;
 319      size_t obj_size;
 320  }
 321  
 322 +/* todo: eventually resolve the String/Buffer mempool dualism. */
 323  StrPoolsAttrs[mem_str_pool_count] = {
 324  
 325      {
 326          "Short Strings", MemAllocator::RoundedSize(36),
 327      },				/* to fit rfc1123 and similar */
 328      {
 329 -        "Medium Strings", MemAllocator::RoundedSize(128),
 330 +        "128b Strings", MemAllocator::RoundedSize(128),
 331      },				/* to fit most urls */
 332      {
 333 -        "Long Strings", MemAllocator::RoundedSize(512)
 334 -    }				/* other */
 335 +        "512b Strings", MemAllocator::RoundedSize(512)
 336 +    },
 337 +    {
 338 +        "1k Strings", MemAllocator::RoundedSize(1024)
 339 +    },
 340 +    {
 341 +        "2k Strings", MemAllocator::RoundedSize(2048)
 342 +    },
 343 +    {
 344 +        "4k Strings", MemAllocator::RoundedSize(4096)
 345 +    }
 346  };
 347  
 348  static struct {
 349 @@ -186,7 +196,18 @@
 350      MemPools[type]->free(p);
 351  }
 352  
 353 -/* allocate a variable size buffer using best-fit pool */
 354 +/**
 355 + * System pagesize, detected at startup-time.
 356 + * Used for various memory management optimizations.
 357 + */
 358 +const size_t squidSystemPageSize=getpagesize();
 359 +
 360 +/**
 361 + * allocate a variable size buffer using best-fit pool
 362 + * \return pointer to the allocated buffer
 363 + * \note the actually-allocated memory block size is returned by altering the
 364 + *      value pointed to by gross_size
 365 + */
 366  void *
 367  memAllocString(size_t net_size, size_t * gross_size)
 368  {
 369 @@ -194,18 +215,35 @@
 370      MemAllocator *pool = NULL;
 371      assert(gross_size);
 372  
 373 +    debugs(13,9,HERE << "got request for " << net_size);
 374 +
 375      for (i = 0; i < mem_str_pool_count; i++) {
 376          if (net_size <= StrPoolsAttrs[i].obj_size) {
 377              pool = StrPools[i].pool;
 378 +            *gross_size =  StrPoolsAttrs[i].obj_size;
 379 +            debugs(13,9,HERE << "selected pool @ " << static_cast<void*>(pool) <<
 380 +                " of size " << StrPoolsAttrs[i].obj_size);
 381              break;
 382          }
 383      }
 384  
 385 -    *gross_size = pool ? StrPoolsAttrs[i].obj_size : net_size;
 386 +    if (pool==NULL) {
 387 +        if (StrPools[0].pool) { //initialization has taken place.
 388 +            /// \todo use a boolean flag rather than this indirect detection?
 389 +            debugs(13,9,HERE << "no pool. Rounding to page size");
 390 +            *gross_size = roundto(net_size,squidSystemPageSize);
 391 +        } else {
 392 +            debugs(13,9,HERE << "MemPools have not initialized yet. Not rounding.");
 393 +            *gross_size=net_size;
 394 +        }
 395 +    }
 396 +
 397      assert(*gross_size >= net_size);
 398      memMeterInc(StrCountMeter);
 399      memMeterAdd(StrVolumeMeter, *gross_size);
 400 -    return pool ? pool->alloc() : xcalloc(1, net_size);
 401 +
 402 +    debugs(13,9,HERE << "returning allocated size " << *gross_size);
 403 +    return pool ? pool->alloc() : xcalloc(1, *gross_size);
 404  }
 405  
 406  extern size_t memStringCount();
 407 @@ -220,7 +258,10 @@
 408      return result;
 409  }
 410  
 411 -/* free buffer allocated with memAllocString() */
 412 +/**
 413 + *  free buffer allocated with memAllocString()
 414 + *  \note the size parameter MUST match a pool size.
 415 + */
 416  void
 417  memFreeString(size_t size, void *buf)
 418  {

Attached Files

To refer to attachments on a page, use attachment:filename, as shown below in the list of files. Do NOT use the URL of the [get] link, since this is subject to change and can break easily.
  • [get | view] (2008-09-03 07:44:41, 90.1 KB) [[attachment:KBuf-work-diagram.png]]
  • [get | view] (2009-03-03 15:05:54, 11.8 KB) [[attachment:sbuf-mempool-failed-integration.patch]]
 All files | Selected Files: delete move to page copy to page

You are not allowed to attach a file to this page.