[PATCH] Auto sized ringbuffers v3

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



This is still in a just-for-testing phase. Found a way to better stress test
this code, hence these changes:

- decreased the growth threshold from 1/2 to 1/3. IOW the goal is always for
   66% of the buffer to be free; this increases the working set, but the
   alternative (handling overflows by spilling over to the unused portion)
   makes thing more complex.
- made buffers grow by 1.5x, not 2x. Related to above.
- increased the initial bufsize from 128k to 192k.
- removed some now redundant Read() code.
- dropped the controversial cRecorder::Receive hunk (I'm keeping this locally
   as it prevents (very rare) data corruption while testing. Overflows will
   be logged even w/o this change, so it makes no difference otherwise).
- added verbose cRecorder::Action sleep logging. Shorter delays are a side
   effect, but do not help all that much (from the few times this triggered
   here so far less than half of the delays were <<100ms).

Patch attached (apparently this makes extracting it easier for gmail users).

artur
diff --git a/dvbdevice.c b/dvbdevice.c
index 955483e..20dcf29 100644
--- a/dvbdevice.c
+++ b/dvbdevice.c
@@ -1184,7 +1184,7 @@ bool cDvbDevice::OpenDvr(void)
   CloseDvr();
   fd_dvr = DvbOpen(DEV_DVB_DVR, CardIndex(), O_RDONLY | O_NONBLOCK, true);
   if (fd_dvr >= 0)
-     tsBuffer = new cTSBuffer(fd_dvr, MEGABYTE(2), CardIndex() + 1);
+     tsBuffer = new cTSBuffer(fd_dvr, MEGABYTE(8), CardIndex() + 1);
   return fd_dvr >= 0;
 }
 
diff --git a/recorder.c b/recorder.c
index 8bb1621..0b625d6 100644
--- a/recorder.c
+++ b/recorder.c
@@ -171,8 +171,22 @@ void cRecorder::Action(void)
            int Count = remux->Put(b, r);
            if (Count)
               ringBuffer->Del(Count);
-           else
-              cCondWait::SleepMs(100); // avoid busy loop when resultBuffer is full in cRemux::Put()
-           }
+           else { // avoid busy loop when resultBuffer is full in cRemux::Put()
+              int ms;
+              for (ms=5; Running() && ms<100; ms+=ms) {
+                 cCondWait::SleepMs(ms);
+                 if (!Running())
+                    return;
+                 Count = remux->Put(b, r);
+                 if (Count) {
+                    ringBuffer->Del(Count);
+                    dsyslog("cRecorder::Action() ring buffer consumer slept %d ms", ms);
+                    break;
+                    }
+                 }
+              if (ms>=100)
+                 dsyslog("cRecorder::Action() ring buffer consumer slept >100 ms");
+              }
+	   }
         }
 }
diff --git a/remux.c b/remux.c
index da805b7..bd29b2f 100644
--- a/remux.c
+++ b/remux.c
@@ -1851,7 +1851,7 @@ void cTS2PES::ts_to_pes(const uint8_t *Buf) // don't need count (=188)
 
 // --- cRemux ----------------------------------------------------------------
 
-#define RESULTBUFFERSIZE KILOBYTE(256)
+#define RESULTBUFFERSIZE MEGABYTE(2)
 
 cRemux::cRemux(int VPid, const int *APids, const int *DPids, const int *SPids, bool ExitOnFailure)
 {
diff --git a/ringbuffer.c b/ringbuffer.c
index 0633bd3..68ce081 100644
--- a/ringbuffer.c
+++ b/ringbuffer.c
@@ -151,13 +151,31 @@ void cRingBufferLinear::PrintDebugRBL(void)
   }
 #endif
 
+// cRingBufferLinear are dynamically sized, or at least we can pretend they are ;)
+// We treat 'Size' as the maximum size, but start with a small buffer, which can
+// grow later as it fills up. Memory is always allocated for the full buffer, but
+// the unused RAM portion remains untouched until (if at all) it is actually needed.
+// Note that we can not start with a larger than requested buffer because there are
+// ring buffer users that cause crashes then (eg softdevice mpegdecoder absolutely
+// needs 32/64k).
+
+// Startup size. 64k still causes overflows before buffer starts to grow, 128k doesn't.
+// The buffers grow to 200..500k anyway, so maybe increasing this a bit more would
+// make sense, but let's first see if it's really needed.
+// In fact 128k is on the low side, but let's try not going for 256k yet.
+#define DEFRBSIZE KILOBYTE(192)
+
 cRingBufferLinear::cRingBufferLinear(int Size, int Margin, bool Statistics, const char *Description)
-:cRingBuffer(Size, Statistics)
+:cRingBuffer(min(Size,DEFRBSIZE), Statistics)
 {
   description = Description ? strdup(Description) : NULL;
   tail = head = margin = Margin;
   gotten = 0;
   buffer = NULL;
+  maxsize = Size;
+  growthresh = size<maxsize ? size/3 : maxsize+1;
+  growbuf = 0;
+  dsyslog("New ring buffer \"%s\" size: %d margin: %d", description, Size, Margin );
   if (Size > 1) { // 'Size - 1' must not be 0!
      if (Margin <= Size / 2) {
         buffer = MALLOC(uchar, Size);
@@ -183,6 +201,8 @@ cRingBufferLinear::~cRingBufferLinear()
 #ifdef DEBUGRINGBUFFERS
   DelDebugRBL(this);
 #endif
+  dsyslog("Deleting ring buffer \"%s\" size: %d / %d (used %g%% of requested size)", description,
+                                               size, maxsize, size/(double)maxsize*100.0 );
   free(buffer);
   free(description);
 }
@@ -205,8 +225,20 @@ void cRingBufferLinear::Clear(void)
   EnablePut();
 }
 
+// Must only be called by the producer (ie Read()/Put()), not the consumer (Get()).
+void cRingBufferLinear::GrowBuffer(void)
+{
+  size = min(size+size/2, maxsize);
+  growthresh = size<maxsize ? size/3 : maxsize+1;
+  dsyslog("Enlarging ring buffer \"%s\": %d bytes (trigger %d, thresh %d)",
+                                      description, size, growbuf, growthresh);
+  growbuf = 0;
+}
+
 int cRingBufferLinear::Read(int FileHandle, int Max)
 {
+  if (growbuf && head>=tail && size<maxsize)
+     GrowBuffer();
   int Tail = tail;
   int diff = Tail - head;
   int free = (diff > 0) ? diff - 1 : Size() - head;
@@ -219,6 +251,10 @@ int cRingBufferLinear::Read(int FileHandle, int Max)
      Count = safe_read(FileHandle, buffer + head, free);
      if (Count > 0) {
         int Head = head + Count;
+        if (Available()+Count >= growthresh)
+           growbuf = 2;
+        if (growbuf && head>=tail && size<maxsize)
+           GrowBuffer();
         if (Head >= Size())
            Head = margin;
         head = Head;
@@ -245,6 +281,10 @@ int cRingBufferLinear::Read(int FileHandle, int Max)
 int cRingBufferLinear::Put(const uchar *Data, int Count)
 {
   if (Count > 0) {
+     if (Available()+Count >= growthresh)
+        growbuf = 3;
+     if (growbuf && head>=tail && size<maxsize)
+        GrowBuffer();
      int Tail = tail;
      int rest = Size() - head;
      int diff = Tail - head;
diff --git a/ringbuffer.h b/ringbuffer.h
index dba0e61..3545520 100644
--- a/ringbuffer.h
+++ b/ringbuffer.h
@@ -18,11 +18,11 @@ private:
   cCondWait readyForPut, readyForGet;
   int putTimeout;
   int getTimeout;
-  int size;
   time_t lastOverflowReport;
   int overflowCount;
   int overflowBytes;
 protected:
+  int size;
   tThreadId getThreadTid;
   int maxFill;//XXX
   int lastPercent;
@@ -59,7 +59,11 @@ private:
   int margin, head, tail;
   int gotten;
   uchar *buffer;
+  int growbuf;
+  int growthresh;
+  int maxsize;
   char *description;
+  void GrowBuffer(void);
 public:
   cRingBufferLinear(int Size, int Margin = 0, bool Statistics = false, const char *Description = NULL);
     ///< Creates a linear ring buffer.
_______________________________________________
vdr mailing list
vdr@xxxxxxxxxxx
http://www.linuxtv.org/cgi-bin/mailman/listinfo/vdr

[Index of Archives]     [Linux Media]     [Asterisk]     [DCCP]     [Netdev]     [Xorg]     [Util Linux NG]     [Xfree86]     [Big List of Linux Books]     [Fedora Users]     [Fedora Women]     [ALSA Devel]     [Linux USB]

  Powered by Linux