System.Net.Scat terGatherBuffer s.MemoryChuck allocates large byte[]

[lwickland]

Summary: System.Net.Scat terGatherBuffer s.MemoryChuck allocates inordinately
large byte[]s when sending large post data.

The following application consumes inordinate quantities of memory. My code
does not explicitly allocate memory in a loop nor does it explicitly allocate
large blocks of memory. Yet, the application’s memory footprint will grow as
large as 370 MB. Rarely will it run to completion; usually, it throws an out
of memory exception or locks up just before sending the packet that would
total 512 MBs.

SciTech .NET Memory Profiler, reports that
System.Net.Scat terGatherBuffer s.MemoryChuck is holding onto very large blocks
of memory, specifically byte[]s, which it appears to have allocated in a
doubling fashion. For example, the sizes of these blocks are 226MB, 113MB,
56.5MB, 28MB, 14MB, and so on down. It’s as if
System.Net.Scat terGatherBuffer s is allocating a byte of memory for every byte
I send on the socket, but is allocating increasingly large buffers to store
this.

If I send a smaller amount of data, the send completes, but the
System.Net.Scat terGatherBuffer s.MemoryChuck byte[]s are not released when the
socket stream is closed, all references to the WebRequest and its stream are
released, and a garbage collection is explicitly requested.

I’ve also tried performing the socket communication on the UI message pump
thread without seeing a different behavior.

I’m running Windows 2000 on a box with 512 MB of memory. I ran this test on
..NET Fx 1.1 SP1. I haven’t tried any other framework versions.

Am I doing something wrong? Is this a known issue? Do you have any
suggestions or explanations?

Thanks in advance for any help you can provide.
using System;
using System.Drawing;
using System.Collecti ons;
using System.Componen tModel;
using System.Windows. Forms;
using System.Data;
using System.Net;
using System.IO;
using System.Threadin g;

namespace SocketMemoryLea k {
public class Form1 : System.Windows. Forms.Form {
private const int KB = 1024;
private const int MB = KB * KB;
private System.Windows. Forms.Button btnSend;
private System.Componen tModel.Containe r components = null;
private System.Windows. Forms.Label lblMBsSent;
private System.Windows. Forms.Label lblGCed;
private System.Windows. Forms.Label lblMemoryAlloca ted;

public Form1() {
InitializeCompo nent();
}

protected override void Dispose(bool disposing) {
if (disposing) {
if (components != null) {
components.Disp ose();
}
}
base.Dispose(di sposing);
}

#region Windows Form Designer generated code
private void InitializeCompo nent() {
this.btnSend = new System.Windows. Forms.Button();
this.lblMBsSent = new System.Windows. Forms.Label();
this.lblGCed = new System.Windows. Forms.Label();
this.lblMemoryA llocated = new System.Windows. Forms.Label();
this.SuspendLay out();
//
// btnSend
//
this.btnSend.An chor =
((System.Window s.Forms.AnchorS tyles)(((System .Windows.Forms. AnchorStyles.To p
| System.Windows. Forms.AnchorSty les.Left)
| System.Windows. Forms.AnchorSty les.Right)));
this.btnSend.Lo cation = new System.Drawing. Point(16, 8);
this.btnSend.Na me = "btnSend";
this.btnSend.Si ze = new System.Drawing. Size(184, 23);
this.btnSend.Ta bIndex = 0;
this.btnSend.Te xt = "Start";
this.btnSend.Cl ick += new System.EventHan dler(this.btnSe nd_Click);
//
// lblMBsSent
//
this.lblMBsSent .Anchor =
((System.Window s.Forms.AnchorS tyles)(((System .Windows.Forms. AnchorStyles.To p
| System.Windows. Forms.AnchorSty les.Left)
| System.Windows. Forms.AnchorSty les.Right)));
this.lblMBsSent .Location = new System.Drawing. Point(16, 40);
this.lblMBsSent .Name = "lblMBsSent ";
this.lblMBsSent .Size = new System.Drawing. Size(184, 23);
this.lblMBsSent .TabIndex = 1;
this.lblMBsSent .Text = "Sent: 0 MBs";
this.lblMBsSent .TextAlign =
System.Drawing. ContentAlignmen t.MiddleLeft;
//
// lblGCed
//
this.lblGCed.An chor =
((System.Window s.Forms.AnchorS tyles)(((System .Windows.Forms. AnchorStyles.To p
| System.Windows. Forms.AnchorSty les.Left)
| System.Windows. Forms.AnchorSty les.Right)));
this.lblGCed.Lo cation = new System.Drawing. Point(16, 88);
this.lblGCed.Na me = "lblGCed";
this.lblGCed.Si ze = new System.Drawing. Size(184, 23);
this.lblGCed.Ta bIndex = 2;
this.lblGCed.Te xt = "No previous garbage collection";
this.lblGCed.Te xtAlign =
System.Drawing. ContentAlignmen t.MiddleLeft;
//
// lblMemoryAlloca ted
//
this.lblMemoryA llocated.Anchor =
((System.Window s.Forms.AnchorS tyles)(((System .Windows.Forms. AnchorStyles.To p
| System.Windows. Forms.AnchorSty les.Left)
| System.Windows. Forms.AnchorSty les.Right)));
this.lblMemoryA llocated.Locati on = new System.Drawing. Point(16,
64);
this.lblMemoryA llocated.Name = "lblMemoryAlloc ated";
this.lblMemoryA llocated.Size = new System.Drawing. Size(184, 23);
this.lblMemoryA llocated.TabInd ex = 3;
this.lblMemoryA llocated.Text = "Memory Allocated: 0 MB";
this.lblMemoryA llocated.TextAl ign =
System.Drawing. ContentAlignmen t.MiddleLeft;
//
// Form1
//
this.AutoScaleB aseSize = new System.Drawing. Size(5, 13);
this.ClientSize = new System.Drawing. Size(208, 117);
this.Controls.A dd(this.lblMemo ryAllocated);
this.Controls.A dd(this.lblGCed );
this.Controls.A dd(this.lblMBsS ent);
this.Controls.A dd(this.btnSend );
this.Name = "Form1";
this.Text = "Form1";
this.ResumeLayo ut(false);

}
#endregion

[STAThread]
static void Main() {
Application.Run (new Form1());
}

private void btnSend_Click(o bject sender, System.EventArg s e) {
this.btnSend.En abled = false;
Thread t = new Thread(new ThreadStart(thi s.Send));
t.IsBackground = true;
t.Start();
}

private void DoGC(int bytesSent) {
this.lblGCed.Te xt = "Starting GC...";
GC.Collect();
GC.WaitForPendi ngFinalizers();
GC.Collect();
this.lblGCed.Te xt = "GC'd at " + (bytesSent / MB) + " MB";
}

private void SetSent(int bytesSent) {
this.lblMBsSent .Text = "Sent: " + (bytesSent / MB) + " MB";
this.lblMemoryA llocated.Text = "Memory Allocated: " +
(Environment.Wo rkingSet / MB) + " MB";
}

private void Send() {
byte[] buffer = new byte[16 * KB];
int totalBytesToSen d = 514 * MB;

HttpWebRequest request = (HttpWebRequest )
WebRequest.Crea te("http://marias/cgi-bin/leif.cfg/php/enduser/fileUpload.php" );
request.Method = "POST";
request.Content Length = totalBytesToSen d;
Stream requestStream = request.GetRequ estStream();

int bytesSent = 0;
int gcInterval = 64 * MB;
int displayInterval = 1 * MB;
while (bytesSent < totalBytesToSen d) {
requestStream.W rite(buffer, 0, buffer.Length);
bytesSent += buffer.Length;
if (bytesSent % displayInterval == 0) {
this.SetSent(by tesSent);
}
if (bytesSent % gcInterval == 0) {
this.DoGC(bytes Sent);
}
}

requestStream.F lush();
requestStream.C lose();
requestStream = null;
request = null;

this.DoGC(bytes Sent);
MessageBox.Show ("Send thread ending.");
this.btnSend.En abled = true;
}
}
}

[Feroze [MSFT]]

this is because httpwebrequest buffers data, as it migt need to repost
it in so9me cases.

you can disable this by setting AllowWriteStrea mBuffering=fals e

-----
feroze

This posting is provided as-is

http://weblogs.asp.net/feroze_daud
lwickland <lw*******@disc ussions.microso ft.com> wrote in message news:<F5******* *************** ************@mi crosoft.com>...

Summary: System.Net.Scat terGatherBuffer s.MemoryChuck allocates inordinately
large byte[]s when sending large post data.

The following application consumes inordinate quantities of memory. My code
does not explicitly allocate memory in a loop nor does it explicitly allocate
large blocks of memory. Yet, the application’s memory footprint will grow as
large as 370 MB. Rarely will it run to completion; usually, it throws an out
of memory exception or locks up just before sending the packet that would
total 512 MBs.

SciTech .NET Memory Profiler, reports that
System.Net.Scat terGatherBuffer s.MemoryChuck is holding onto very large blocks
of memory, specifically byte[]s, which it appears to have allocated in a
doubling fashion. For example, the sizes of these blocks are 226MB, 113MB,
56.5MB, 28MB, 14MB, and so on down. It’s as if
System.Net.Scat terGatherBuffer s is allocating a byte of memory for every byte
I send on the socket, but is allocating increasingly large buffers to store
this.

If I send a smaller amount of data, the send completes, but the
System.Net.Scat terGatherBuffer s.MemoryChuck byte[]s are not released when the
socket stream is closed, all references to the WebRequest and its stream are
released, and a garbage collection is explicitly requested.

I’ve also tried performing the socket communication on the UI message pump
thread without seeing a different behavior.

I’m running Windows 2000 on a box with 512 MB of memory. I ran this test on
.NET Fx 1.1 SP1. I haven’t tried any other framework versions.

Am I doing something wrong? Is this a known issue? Do you have any
suggestions or explanations?

Thanks in advance for any help you can provide.
using System;
using System.Drawing;
using System.Collecti ons;
using System.Componen tModel;
using System.Windows. Forms;
using System.Data;
using System.Net;
using System.IO;
using System.Threadin g;

namespace SocketMemoryLea k {
public class Form1 : System.Windows. Forms.Form {
private const int KB = 1024;
private const int MB = KB * KB;
private System.Windows. Forms.Button btnSend;
private System.Componen tModel.Containe r components = null;
private System.Windows. Forms.Label lblMBsSent;
private System.Windows. Forms.Label lblGCed;
private System.Windows. Forms.Label lblMemoryAlloca ted;

public Form1() {
InitializeCompo nent();
}

protected override void Dispose(bool disposing) {
if (disposing) {
if (components != null) {
components.Disp ose();
}
}
base.Dispose(di sposing);
}

#region Windows Form Designer generated code
private void InitializeCompo nent() {
this.btnSend = new System.Windows. Forms.Button();
this.lblMBsSent = new System.Windows. Forms.Label();
this.lblGCed = new System.Windows. Forms.Label();
this.lblMemoryA llocated = new System.Windows. Forms.Label();
this.SuspendLay out();
//
// btnSend
//
this.btnSend.An chor =
((System.Window s.Forms.AnchorS tyles)(((System .Windows.Forms. AnchorStyles.To p
| System.Windows. Forms.AnchorSty les.Left)
| System.Windows. Forms.AnchorSty les.Right)));
this.btnSend.Lo cation = new System.Drawing. Point(16, 8);
this.btnSend.Na me = "btnSend";
this.btnSend.Si ze = new System.Drawing. Size(184, 23);
this.btnSend.Ta bIndex = 0;
this.btnSend.Te xt = "Start";
this.btnSend.Cl ick += new System.EventHan dler(this.btnSe nd_Click);
//
// lblMBsSent
//
this.lblMBsSent .Anchor =
((System.Window s.Forms.AnchorS tyles)(((System .Windows.Forms. AnchorStyles.To p
| System.Windows. Forms.AnchorSty les.Left)
| System.Windows. Forms.AnchorSty les.Right)));
this.lblMBsSent .Location = new System.Drawing. Point(16, 40);
this.lblMBsSent .Name = "lblMBsSent ";
this.lblMBsSent .Size = new System.Drawing. Size(184, 23);
this.lblMBsSent .TabIndex = 1;
this.lblMBsSent .Text = "Sent: 0 MBs";
this.lblMBsSent .TextAlign =
System.Drawing. ContentAlignmen t.MiddleLeft;
//
// lblGCed
//
this.lblGCed.An chor =
((System.Window s.Forms.AnchorS tyles)(((System .Windows.Forms. AnchorStyles.To p
| System.Windows. Forms.AnchorSty les.Left)
| System.Windows. Forms.AnchorSty les.Right)));
this.lblGCed.Lo cation = new System.Drawing. Point(16, 88);
this.lblGCed.Na me = "lblGCed";
this.lblGCed.Si ze = new System.Drawing. Size(184, 23);
this.lblGCed.Ta bIndex = 2;
this.lblGCed.Te xt = "No previous garbage collection";
this.lblGCed.Te xtAlign =
System.Drawing. ContentAlignmen t.MiddleLeft;
//
// lblMemoryAlloca ted
//
this.lblMemoryA llocated.Anchor =
((System.Window s.Forms.AnchorS tyles)(((System .Windows.Forms. AnchorStyles.To p
| System.Windows. Forms.AnchorSty les.Left)
| System.Windows. Forms.AnchorSty les.Right)));
this.lblMemoryA llocated.Locati on = new System.Drawing. Point(16,
64);
this.lblMemoryA llocated.Name = "lblMemoryAlloc ated";
this.lblMemoryA llocated.Size = new System.Drawing. Size(184, 23);
this.lblMemoryA llocated.TabInd ex = 3;
this.lblMemoryA llocated.Text = "Memory Allocated: 0 MB";
this.lblMemoryA llocated.TextAl ign =
System.Drawing. ContentAlignmen t.MiddleLeft;
//
// Form1
//
this.AutoScaleB aseSize = new System.Drawing. Size(5, 13);
this.ClientSize = new System.Drawing. Size(208, 117);
this.Controls.A dd(this.lblMemo ryAllocated);
this.Controls.A dd(this.lblGCed );
this.Controls.A dd(this.lblMBsS ent);
this.Controls.A dd(this.btnSend );
this.Name = "Form1";
this.Text = "Form1";
this.ResumeLayo ut(false);

}
#endregion

[STAThread]
static void Main() {
Application.Run (new Form1());
}

private void btnSend_Click(o bject sender, System.EventArg s e) {
this.btnSend.En abled = false;
Thread t = new Thread(new ThreadStart(thi s.Send));
t.IsBackground = true;
t.Start();
}

private void DoGC(int bytesSent) {
this.lblGCed.Te xt = "Starting GC...";
GC.Collect();
GC.WaitForPendi ngFinalizers();
GC.Collect();
this.lblGCed.Te xt = "GC'd at " + (bytesSent / MB) + " MB";
}

private void SetSent(int bytesSent) {
this.lblMBsSent .Text = "Sent: " + (bytesSent / MB) + " MB";
this.lblMemoryA llocated.Text = "Memory Allocated: " +
(Environment.Wo rkingSet / MB) + " MB";
}

private void Send() {
byte[] buffer = new byte[16 * KB];
int totalBytesToSen d = 514 * MB;

HttpWebRequest request = (HttpWebRequest )
WebRequest.Crea te("http://marias/cgi-bin/leif.cfg/php/enduser/fileUpload.php" );
request.Method = "POST";
request.Content Length = totalBytesToSen d;
Stream requestStream = request.GetRequ estStream();

int bytesSent = 0;
int gcInterval = 64 * MB;
int displayInterval = 1 * MB;
while (bytesSent < totalBytesToSen d) {
requestStream.W rite(buffer, 0, buffer.Length);
bytesSent += buffer.Length;
if (bytesSent % displayInterval == 0) {
this.SetSent(by tesSent);
}
if (bytesSent % gcInterval == 0) {
this.DoGC(bytes Sent);
}
}

requestStream.F lush();
requestStream.C lose();
requestStream = null;
request = null;

this.DoGC(bytes Sent);
MessageBox.Show ("Send thread ending.");
this.btnSend.En abled = true;
}
}
}