469,920 Members | 2,367 Online
Bytes | Developer Community
New Post

Home Posts Topics Members FAQ

Post your question to a community of 469,920 developers. It's quick & easy.

System.Net.ScatterGatherBuffers.MemoryChuck allocates large byte[]

Summary: System.Net.ScatterGatherBuffers.MemoryChuck allocates inordinately
large byte[]s when sending large post data.

The following application consumes inordinate quantities of memory. My code
does not explicitly allocate memory in a loop nor does it explicitly allocate
large blocks of memory. Yet, the application’s memory footprint will grow as
large as 370 MB. Rarely will it run to completion; usually, it throws an out
of memory exception or locks up just before sending the packet that would
total 512 MBs.

SciTech .NET Memory Profiler, reports that
System.Net.ScatterGatherBuffers.MemoryChuck is holding onto very large blocks
of memory, specifically byte[]s, which it appears to have allocated in a
doubling fashion. For example, the sizes of these blocks are 226MB, 113MB,
56.5MB, 28MB, 14MB, and so on down. It’s as if
System.Net.ScatterGatherBuffers is allocating a byte of memory for every byte
I send on the socket, but is allocating increasingly large buffers to store
this.

If I send a smaller amount of data, the send completes, but the
System.Net.ScatterGatherBuffers.MemoryChuck byte[]s are not released when the
socket stream is closed, all references to the WebRequest and its stream are
released, and a garbage collection is explicitly requested.

I’ve also tried performing the socket communication on the UI message pump
thread without seeing a different behavior.

I’m running Windows 2000 on a box with 512 MB of memory. I ran this test on
..NET Fx 1.1 SP1. I haven’t tried any other framework versions.

Am I doing something wrong? Is this a known issue? Do you have any
suggestions or explanations?

Thanks in advance for any help you can provide.
using System;
using System.Drawing;
using System.Collections;
using System.ComponentModel;
using System.Windows.Forms;
using System.Data;
using System.Net;
using System.IO;
using System.Threading;

namespace SocketMemoryLeak {
public class Form1 : System.Windows.Forms.Form {
private const int KB = 1024;
private const int MB = KB * KB;
private System.Windows.Forms.Button btnSend;
private System.ComponentModel.Container components = null;
private System.Windows.Forms.Label lblMBsSent;
private System.Windows.Forms.Label lblGCed;
private System.Windows.Forms.Label lblMemoryAllocated;

public Form1() {
InitializeComponent();
}

protected override void Dispose(bool disposing) {
if (disposing) {
if (components != null) {
components.Dispose();
}
}
base.Dispose(disposing);
}

#region Windows Form Designer generated code
private void InitializeComponent() {
this.btnSend = new System.Windows.Forms.Button();
this.lblMBsSent = new System.Windows.Forms.Label();
this.lblGCed = new System.Windows.Forms.Label();
this.lblMemoryAllocated = new System.Windows.Forms.Label();
this.SuspendLayout();
//
// btnSend
//
this.btnSend.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Wind ows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.btnSend.Location = new System.Drawing.Point(16, 8);
this.btnSend.Name = "btnSend";
this.btnSend.Size = new System.Drawing.Size(184, 23);
this.btnSend.TabIndex = 0;
this.btnSend.Text = "Start";
this.btnSend.Click += new System.EventHandler(this.btnSend_Click);
//
// lblMBsSent
//
this.lblMBsSent.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Wind ows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblMBsSent.Location = new System.Drawing.Point(16, 40);
this.lblMBsSent.Name = "lblMBsSent";
this.lblMBsSent.Size = new System.Drawing.Size(184, 23);
this.lblMBsSent.TabIndex = 1;
this.lblMBsSent.Text = "Sent: 0 MBs";
this.lblMBsSent.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// lblGCed
//
this.lblGCed.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Wind ows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblGCed.Location = new System.Drawing.Point(16, 88);
this.lblGCed.Name = "lblGCed";
this.lblGCed.Size = new System.Drawing.Size(184, 23);
this.lblGCed.TabIndex = 2;
this.lblGCed.Text = "No previous garbage collection";
this.lblGCed.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// lblMemoryAllocated
//
this.lblMemoryAllocated.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Wind ows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblMemoryAllocated.Location = new System.Drawing.Point(16,
64);
this.lblMemoryAllocated.Name = "lblMemoryAllocated";
this.lblMemoryAllocated.Size = new System.Drawing.Size(184, 23);
this.lblMemoryAllocated.TabIndex = 3;
this.lblMemoryAllocated.Text = "Memory Allocated: 0 MB";
this.lblMemoryAllocated.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// Form1
//
this.AutoScaleBaseSize = new System.Drawing.Size(5, 13);
this.ClientSize = new System.Drawing.Size(208, 117);
this.Controls.Add(this.lblMemoryAllocated);
this.Controls.Add(this.lblGCed);
this.Controls.Add(this.lblMBsSent);
this.Controls.Add(this.btnSend);
this.Name = "Form1";
this.Text = "Form1";
this.ResumeLayout(false);

}
#endregion

[STAThread]
static void Main() {
Application.Run(new Form1());
}

private void btnSend_Click(object sender, System.EventArgs e) {
this.btnSend.Enabled = false;
Thread t = new Thread(new ThreadStart(this.Send));
t.IsBackground = true;
t.Start();
}

private void DoGC(int bytesSent) {
this.lblGCed.Text = "Starting GC...";
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
this.lblGCed.Text = "GC'd at " + (bytesSent / MB) + " MB";
}

private void SetSent(int bytesSent) {
this.lblMBsSent.Text = "Sent: " + (bytesSent / MB) + " MB";
this.lblMemoryAllocated.Text = "Memory Allocated: " +
(Environment.WorkingSet / MB) + " MB";
}

private void Send() {
byte[] buffer = new byte[16 * KB];
int totalBytesToSend = 514 * MB;

HttpWebRequest request = (HttpWebRequest)
WebRequest.Create("http://marias/cgi-bin/leif.cfg/php/enduser/fileUpload.php");
request.Method = "POST";
request.ContentLength = totalBytesToSend;
Stream requestStream = request.GetRequestStream();

int bytesSent = 0;
int gcInterval = 64 * MB;
int displayInterval = 1 * MB;
while (bytesSent < totalBytesToSend) {
requestStream.Write(buffer, 0, buffer.Length);
bytesSent += buffer.Length;
if (bytesSent % displayInterval == 0) {
this.SetSent(bytesSent);
}
if (bytesSent % gcInterval == 0) {
this.DoGC(bytesSent);
}
}

requestStream.Flush();
requestStream.Close();
requestStream = null;
request = null;

this.DoGC(bytesSent);
MessageBox.Show("Send thread ending.");
this.btnSend.Enabled = true;
}
}
}
Nov 16 '05 #1
1 3732
this is because httpwebrequest buffers data, as it migt need to repost
it in so9me cases.

you can disable this by setting AllowWriteStreamBuffering=false

-----
feroze

This posting is provided as-is

http://weblogs.asp.net/feroze_daud
lwickland <lw*******@discussions.microsoft.com> wrote in message news:<F5**********************************@microso ft.com>...
Summary: System.Net.ScatterGatherBuffers.MemoryChuck allocates inordinately
large byte[]s when sending large post data.

The following application consumes inordinate quantities of memory. My code
does not explicitly allocate memory in a loop nor does it explicitly allocate
large blocks of memory. Yet, the application’s memory footprint will grow as
large as 370 MB. Rarely will it run to completion; usually, it throws an out
of memory exception or locks up just before sending the packet that would
total 512 MBs.

SciTech .NET Memory Profiler, reports that
System.Net.ScatterGatherBuffers.MemoryChuck is holding onto very large blocks
of memory, specifically byte[]s, which it appears to have allocated in a
doubling fashion. For example, the sizes of these blocks are 226MB, 113MB,
56.5MB, 28MB, 14MB, and so on down. It’s as if
System.Net.ScatterGatherBuffers is allocating a byte of memory for every byte
I send on the socket, but is allocating increasingly large buffers to store
this.

If I send a smaller amount of data, the send completes, but the
System.Net.ScatterGatherBuffers.MemoryChuck byte[]s are not released when the
socket stream is closed, all references to the WebRequest and its stream are
released, and a garbage collection is explicitly requested.

I’ve also tried performing the socket communication on the UI message pump
thread without seeing a different behavior.

I’m running Windows 2000 on a box with 512 MB of memory. I ran this test on
.NET Fx 1.1 SP1. I haven’t tried any other framework versions.

Am I doing something wrong? Is this a known issue? Do you have any
suggestions or explanations?

Thanks in advance for any help you can provide.
using System;
using System.Drawing;
using System.Collections;
using System.ComponentModel;
using System.Windows.Forms;
using System.Data;
using System.Net;
using System.IO;
using System.Threading;

namespace SocketMemoryLeak {
public class Form1 : System.Windows.Forms.Form {
private const int KB = 1024;
private const int MB = KB * KB;
private System.Windows.Forms.Button btnSend;
private System.ComponentModel.Container components = null;
private System.Windows.Forms.Label lblMBsSent;
private System.Windows.Forms.Label lblGCed;
private System.Windows.Forms.Label lblMemoryAllocated;

public Form1() {
InitializeComponent();
}

protected override void Dispose(bool disposing) {
if (disposing) {
if (components != null) {
components.Dispose();
}
}
base.Dispose(disposing);
}

#region Windows Form Designer generated code
private void InitializeComponent() {
this.btnSend = new System.Windows.Forms.Button();
this.lblMBsSent = new System.Windows.Forms.Label();
this.lblGCed = new System.Windows.Forms.Label();
this.lblMemoryAllocated = new System.Windows.Forms.Label();
this.SuspendLayout();
//
// btnSend
//
this.btnSend.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Wind ows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.btnSend.Location = new System.Drawing.Point(16, 8);
this.btnSend.Name = "btnSend";
this.btnSend.Size = new System.Drawing.Size(184, 23);
this.btnSend.TabIndex = 0;
this.btnSend.Text = "Start";
this.btnSend.Click += new System.EventHandler(this.btnSend_Click);
//
// lblMBsSent
//
this.lblMBsSent.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Wind ows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblMBsSent.Location = new System.Drawing.Point(16, 40);
this.lblMBsSent.Name = "lblMBsSent";
this.lblMBsSent.Size = new System.Drawing.Size(184, 23);
this.lblMBsSent.TabIndex = 1;
this.lblMBsSent.Text = "Sent: 0 MBs";
this.lblMBsSent.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// lblGCed
//
this.lblGCed.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Wind ows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblGCed.Location = new System.Drawing.Point(16, 88);
this.lblGCed.Name = "lblGCed";
this.lblGCed.Size = new System.Drawing.Size(184, 23);
this.lblGCed.TabIndex = 2;
this.lblGCed.Text = "No previous garbage collection";
this.lblGCed.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// lblMemoryAllocated
//
this.lblMemoryAllocated.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Wind ows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblMemoryAllocated.Location = new System.Drawing.Point(16,
64);
this.lblMemoryAllocated.Name = "lblMemoryAllocated";
this.lblMemoryAllocated.Size = new System.Drawing.Size(184, 23);
this.lblMemoryAllocated.TabIndex = 3;
this.lblMemoryAllocated.Text = "Memory Allocated: 0 MB";
this.lblMemoryAllocated.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// Form1
//
this.AutoScaleBaseSize = new System.Drawing.Size(5, 13);
this.ClientSize = new System.Drawing.Size(208, 117);
this.Controls.Add(this.lblMemoryAllocated);
this.Controls.Add(this.lblGCed);
this.Controls.Add(this.lblMBsSent);
this.Controls.Add(this.btnSend);
this.Name = "Form1";
this.Text = "Form1";
this.ResumeLayout(false);

}
#endregion

[STAThread]
static void Main() {
Application.Run(new Form1());
}

private void btnSend_Click(object sender, System.EventArgs e) {
this.btnSend.Enabled = false;
Thread t = new Thread(new ThreadStart(this.Send));
t.IsBackground = true;
t.Start();
}

private void DoGC(int bytesSent) {
this.lblGCed.Text = "Starting GC...";
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
this.lblGCed.Text = "GC'd at " + (bytesSent / MB) + " MB";
}

private void SetSent(int bytesSent) {
this.lblMBsSent.Text = "Sent: " + (bytesSent / MB) + " MB";
this.lblMemoryAllocated.Text = "Memory Allocated: " +
(Environment.WorkingSet / MB) + " MB";
}

private void Send() {
byte[] buffer = new byte[16 * KB];
int totalBytesToSend = 514 * MB;

HttpWebRequest request = (HttpWebRequest)
WebRequest.Create("http://marias/cgi-bin/leif.cfg/php/enduser/fileUpload.php");
request.Method = "POST";
request.ContentLength = totalBytesToSend;
Stream requestStream = request.GetRequestStream();

int bytesSent = 0;
int gcInterval = 64 * MB;
int displayInterval = 1 * MB;
while (bytesSent < totalBytesToSend) {
requestStream.Write(buffer, 0, buffer.Length);
bytesSent += buffer.Length;
if (bytesSent % displayInterval == 0) {
this.SetSent(bytesSent);
}
if (bytesSent % gcInterval == 0) {
this.DoGC(bytesSent);
}
}

requestStream.Flush();
requestStream.Close();
requestStream = null;
request = null;

this.DoGC(bytesSent);
MessageBox.Show("Send thread ending.");
this.btnSend.Enabled = true;
}
}
}

Nov 16 '05 #2

This discussion thread is closed

Replies have been disabled for this discussion.

Similar topics

reply views Thread by zhimin | last post: by
12 posts views Thread by Olaf Baeyens | last post: by
reply views Thread by NicK chlam via DotNetMonster.com | last post: by
14 posts views Thread by ThazKool | last post: by
7 posts views Thread by StupidScript | last post: by
4 posts views Thread by Jonathan Wood | last post: by
30 posts views Thread by nano2k | last post: by
reply views Thread by Waqarahmed | last post: by
By using this site, you agree to our Privacy Policy and Terms of Use.