How to get SHA1 or MD5 of a big file (+5MB - 20GB) as I can't read 20GB into
memory.
--
Chang.
6  4460 
May be a better way, but I would do it in block sizes of say 1KB or 4KB.
Get first 4KB, hash, and save. Get second hash by appending next 4K block
and prev hash bytes and hash SHA1(block+prev hash) and so on. It does not
matter what algo you use, as long as you reproduce it the same way each
time. Here is a CRC32 checksum I updated from crc32.cs in SharpZipLib by
Mike Kruger (by my read, it is pretty much open to mod and use).
using System;
using System.IO;
namespace NetFile
{
/// <summary>
/// CRC32 Modified from source by Mike Kruger.
/// Generate a table for a byte-wise 32-bit CRC calculation on the
polynomial:
/// x^32+x^26+x^23+ x^22+x^16+x^12+ x^11+x^10+x^8+x ^7+x^5+x^4+x^2+ x+1.
///
/// Polynomials over GF(2) are represented in binary, one bit per
coefficient,
/// with the lowest powers in the most significant bit. Then adding
polynomials
/// is just exclusive-or, and multiplying a polynomial by x is a right
shift by
/// one. If we call the above polynomial p, and represent a byte as the
/// polynomial q, also with the lowest power in the most significant bit
(so the
/// byte 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32) mod
p,
/// where a mod b means the remainder after dividing a by b.
///
/// This calculation is done using the shift-register method of multiplying
and
/// taking the remainder. The register is initialized to zero, and for
each
/// incoming bit, x^32 is added mod p to the register if the bit is a one
(where
/// x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod
p by
/// x (which is shifting right by one and adding x^32 mod p if the bit
shifted
/// out is a one). We start with the highest power (least significant bit)
of
/// q and repeat for all eight bits of q.
///
/// The table is simply the CRC of all possible eight bit values. This is
all
/// the information needed to generate CRC's on data a byte at a time for
all
/// combinations of CRC register values and incoming bytes.
/// </summary>
public sealed class Crc32
{
#region Fields
private readonly static uint CrcSeed = 0xFFFFFFFF;
private readonly static uint[] CrcTable = new uint[] {
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419,
0x706AF48F, 0xE963A535, 0x9E6495A3, 0x0EDB8832, 0x79DCB8A4,
0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07,
0x90BF1D91, 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE,
0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, 0x136C9856,
0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9,
0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4,
0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3,
0x45DF5C75, 0xDCD60DCF, 0xABD13D59, 0x26D930AC, 0x51DE003A,
0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599,
0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, 0x76DC4190,
0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F,
0x9FBFE4A5, 0xE8B8D433, 0x7807C9A2, 0x0F00F934, 0x9609A88E,
0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED,
0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950,
0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3,
0xFBD44C65, 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2,
0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A,
0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5,
0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA, 0xBE0B1010,
0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17,
0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6,
0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615,
0x73DC1683, 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,
0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, 0xF00F9344,
0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB,
0x196C3671, 0x6E6B06E7, 0xFED41B76, 0x89D32BE0, 0x10DA7A5A,
0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1,
0xA6BC5767, 0x3FB506DD, 0x48B2364B, 0xD80D2BDA, 0xAF0A1B4C,
0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF,
0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE,
0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31,
0x2CD99E8B, 0x5BDEAE1D, 0x9B64C2B0, 0xEC63F226, 0x756AA39C,
0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B,
0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242,
0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1,
0x18B74777, 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C,
0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, 0xA00AE278,
0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7,
0x4969474D, 0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66,
0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605,
0xCDD70693, 0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8,
0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B,
0x2D02EF8D
};
private uint crc = 0; // crc data checksum so far.
#endregion
#region Constructors
public Crc32()
{
}
#endregion
#region Properties
/// <summary>
/// Returns the CRC32 data checksum computed so far.
/// </summary>
public uint Value
{
get
{
return crc;
}
set
{
crc = value;
}
}
#endregion
#region Methods
public static uint GetStreamCRC32( Stream stream)
{
if ( stream == null )
throw new ArgumentNullExc eption("stream" );
if ( ! stream.CanRead )
throw new ArgumentExcepti on("stream is not readable.");
stream.Position = 0;
Crc32 crc32 = new Crc32();
byte[] buf = new byte[4096];
int len = 0;
while ( (len = stream.Read(buf , 0, buf.Length)) != 0 )
{
crc32.Update(bu f, 0, len);
}
stream.Position = 0;
return crc32.Value;
}
public static uint GetFileCRC32(st ring path)
{
if ( path == null )
throw new ArgumentNullExc eption("path");
Crc32 crc32 = new Crc32();
byte[] buf = new byte[4096];
int len = 0;
using(FileStrea m fs = new FileStream(path , FileMode.Open))
{
while ( (len = fs.Read(buf, 0, buf.Length)) != 0 )
{
crc32.Update(bu f, 0, len);
}
}
return crc32.Value;
}
/// <summary>
/// Resets the CRC32 data checksum as if no update was ever called.
/// </summary>
public void Reset()
{
crc = 0;
}
/// <summary>
/// Updates the checksum with the int bval.
/// </summary>
/// <param name = "bval">
/// the byte is taken as the lower 8 bits of bval
/// </param>
public void Update(int bval)
{
crc ^= CrcSeed;
crc = CrcTable[(crc ^ bval) & 0xFF] ^ (crc >> 8);
crc ^= CrcSeed;
}
/// <summary>
/// Updates the checksum with the bytes taken from the array.
/// </summary>
/// <param name="buffer">
/// buffer an array of bytes
/// </param>
public void Update(byte[] buffer)
{
Update(buffer, 0, buffer.Length);
}
/// <summary>
/// Adds the byte array to the data checksum.
/// </summary>
/// <param name = "buf">
/// the buffer which contains the data
/// </param>
/// <param name = "off">
/// the offset in the buffer where the data starts
/// </param>
/// <param name = "len">
/// the length of the data
/// </param>
public void Update(byte[] buf, int off, int len)
{
if (buf == null)
{
throw new ArgumentNullExc eption("buf");
}
if (off < 0 || len < 0 || off + len > buf.Length)
{
throw new ArgumentOutOfRa ngeException();
}
crc ^= CrcSeed;
while (--len >= 0)
{
crc = CrcTable[(crc ^ buf[off++]) & 0xFF] ^ (crc >> 8);
}
crc ^= CrcSeed;
}
#endregion
}
}
--
William Stacey, MVP http://mvp.support.microsoft.com
"Chang" <no*****@thisad dress.com> wrote in message
news:OH******** ******@TK2MSFTN GP14.phx.gbl... How to get SHA1 or MD5 of a big file (+5MB - 20GB) as I can't read 20GB
into memory.
--
Chang.
Here is my version using SHA1.
using System;
using System.IO;
using System.Security .Cryptography;
namespace MyNameSpace
{
public sealed class SHAFileHash
{
private byte[] hash = new byte[20];
private SHA1 sha = SHA1.Create();
public SHAFileHash()
{
}
/// <summary>
/// Returns the CRC32 data checksum computed so far.
/// </summary>
public byte[] Hash
{
get { return hash; }
set { hash = value; }
}
public static byte[] GetStreamSHAHas h(Stream stream)
{
if ( stream == null )
throw new ArgumentNullExc eption("stream" );
if ( ! stream.CanRead )
throw new ArgumentExcepti on("stream is not readable.");
stream.Position = 0;
SHAFileHash fh = new SHAFileHash();
byte[] buf = new byte[4096];
int len = 0;
while ( (len = stream.Read(buf , 0, buf.Length)) != 0 )
{
fh.Update(buf, 0, len);
}
stream.Position = 0;
return fh.Hash;
}
public static byte[] GetFileSHAHash( string path)
{
if ( path == null )
throw new ArgumentNullExc eption("path");
SHAFileHash fh = new SHAFileHash();
byte[] buf = new byte[4096];
int len = 0;
using(FileStrea m fs = new FileStream(path , FileMode.Open))
{
while ( (len = fs.Read(buf, 0, buf.Length)) != 0 )
{
fh.Update(buf, 0, len);
}
}
return fh.Hash;
}
/// <summary>
/// Resets the hash checksum.
/// </summary>
public void Reset()
{
hash = new byte[20];
}
/// <summary>
/// Updates the checksum with the bytes taken from the array.
/// </summary>
public void Update(byte[] buffer)
{
Update(buffer, 0, buffer.Length);
}
/// <summary>
/// Adds the byte array to the data checksum.
/// </summary>
/// <param name = "buf">
/// the buffer which contains the data
/// </param>
/// <param name = "off">
/// the offset in the buffer where the data starts
/// </param>
/// <param name = "len">
/// the length of the data
/// </param>
public void Update(byte[] buf, int off, int len)
{
if (buf == null)
throw new ArgumentNullExc eption("buf");
if (off < 0 || len < 0 || off + len > buf.Length)
throw new ArgumentOutOfRa ngeException();
byte[] data = JoinArrays(hash , buf);
hash = sha.ComputeHash (data);
}
private static byte[] JoinArrays(byte[] b1, byte[] b2)
{
byte[] ba = new byte[b1.Length + b2.Length];
Buffer.BlockCop y(b1, 0, ba, 0, b1.Length);
Buffer.BlockCop y(b2, 0, ba, b1.Length, b2.Length);
return ba;
}
}
}
--
William Stacey, MVP http://mvp.support.microsoft.com
"William Stacey [MVP]" <st***********@ mvps.org> wrote in message
news:#0******** *****@TK2MSFTNG P09.phx.gbl... May be a better way, but I would do it in block sizes of say 1KB or 4KB.
Get first 4KB, hash, and save. Get second hash by appending next 4K block
and prev hash bytes and hash SHA1(block+prev hash) and so on. It does not
matter what algo you use, as long as you reproduce it the same way each
time. Here is a CRC32 checksum I updated from crc32.cs in SharpZipLib by
Mike Kruger (by my read, it is pretty much open to mod and use).
using System;
using System.IO;
namespace NetFile
{
/// <summary>
/// CRC32 Modified from source by Mike Kruger.
/// Generate a table for a byte-wise 32-bit CRC calculation on the
polynomial:
/// x^32+x^26+x^23+ x^22+x^16+x^12+ x^11+x^10+x^8+x ^7+x^5+x^4+x^2+ x+1.
///
/// Polynomials over GF(2) are represented in binary, one bit per
coefficient,
/// with the lowest powers in the most significant bit. Then adding
polynomials
/// is just exclusive-or, and multiplying a polynomial by x is a right
shift by
/// one. If we call the above polynomial p, and represent a byte as the
/// polynomial q, also with the lowest power in the most significant bit
(so the
/// byte 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32)
mod p,
/// where a mod b means the remainder after dividing a by b.
///
/// This calculation is done using the shift-register method of
multiplying and
/// taking the remainder. The register is initialized to zero, and for
each
/// incoming bit, x^32 is added mod p to the register if the bit is a one
(where
/// x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied
mod p by
/// x (which is shifting right by one and adding x^32 mod p if the bit
shifted
/// out is a one). We start with the highest power (least significant
bit) of
/// q and repeat for all eight bits of q.
///
/// The table is simply the CRC of all possible eight bit values. This
is all
/// the information needed to generate CRC's on data a byte at a time for
all
/// combinations of CRC register values and incoming bytes.
/// </summary>
public sealed class Crc32
{
#region Fields
private readonly static uint CrcSeed = 0xFFFFFFFF;
private readonly static uint[] CrcTable = new uint[] {
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419,
0x706AF48F, 0xE963A535, 0x9E6495A3, 0x0EDB8832, 0x79DCB8A4,
0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07,
0x90BF1D91, 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE,
0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, 0x136C9856,
0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9,
0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4,
0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3,
0x45DF5C75, 0xDCD60DCF, 0xABD13D59, 0x26D930AC, 0x51DE003A,
0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599,
0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, 0x76DC4190,
0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F,
0x9FBFE4A5, 0xE8B8D433, 0x7807C9A2, 0x0F00F934, 0x9609A88E,
0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED,
0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950,
0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3,
0xFBD44C65, 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2,
0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A,
0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5,
0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA, 0xBE0B1010,
0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17,
0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6,
0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615,
0x73DC1683, 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,
0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, 0xF00F9344,
0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB,
0x196C3671, 0x6E6B06E7, 0xFED41B76, 0x89D32BE0, 0x10DA7A5A,
0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1,
0xA6BC5767, 0x3FB506DD, 0x48B2364B, 0xD80D2BDA, 0xAF0A1B4C,
0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF,
0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE,
0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31,
0x2CD99E8B, 0x5BDEAE1D, 0x9B64C2B0, 0xEC63F226, 0x756AA39C,
0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B,
0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242,
0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1,
0x18B74777, 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C,
0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, 0xA00AE278,
0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7,
0x4969474D, 0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66,
0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605,
0xCDD70693, 0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8,
0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B,
0x2D02EF8D
};
private uint crc = 0; // crc data checksum so far.
#endregion
#region Constructors
public Crc32()
{
}
#endregion
#region Properties
/// <summary>
/// Returns the CRC32 data checksum computed so far.
/// </summary>
public uint Value
{
get
{
return crc;
}
set
{
crc = value;
}
}
#endregion
#region Methods
public static uint GetStreamCRC32( Stream stream)
{
if ( stream == null )
throw new ArgumentNullExc eption("stream" );
if ( ! stream.CanRead )
throw new ArgumentExcepti on("stream is not readable.");
stream.Position = 0;
Crc32 crc32 = new Crc32();
byte[] buf = new byte[4096];
int len = 0;
while ( (len = stream.Read(buf , 0, buf.Length)) != 0 )
{
crc32.Update(bu f, 0, len);
}
stream.Position = 0;
return crc32.Value;
}
public static uint GetFileCRC32(st ring path)
{
if ( path == null )
throw new ArgumentNullExc eption("path");
Crc32 crc32 = new Crc32();
byte[] buf = new byte[4096];
int len = 0;
using(FileStrea m fs = new FileStream(path , FileMode.Open))
{
while ( (len = fs.Read(buf, 0, buf.Length)) != 0 )
{
crc32.Update(bu f, 0, len);
}
}
return crc32.Value;
}
/// <summary>
/// Resets the CRC32 data checksum as if no update was ever called.
/// </summary>
public void Reset()
{
crc = 0;
}
/// <summary>
/// Updates the checksum with the int bval.
/// </summary>
/// <param name = "bval">
/// the byte is taken as the lower 8 bits of bval
/// </param>
public void Update(int bval)
{
crc ^= CrcSeed;
crc = CrcTable[(crc ^ bval) & 0xFF] ^ (crc >> 8);
crc ^= CrcSeed;
}
/// <summary>
/// Updates the checksum with the bytes taken from the array.
/// </summary>
/// <param name="buffer">
/// buffer an array of bytes
/// </param>
public void Update(byte[] buffer)
{
Update(buffer, 0, buffer.Length);
}
/// <summary>
/// Adds the byte array to the data checksum.
/// </summary>
/// <param name = "buf">
/// the buffer which contains the data
/// </param>
/// <param name = "off">
/// the offset in the buffer where the data starts
/// </param>
/// <param name = "len">
/// the length of the data
/// </param>
public void Update(byte[] buf, int off, int len)
{
if (buf == null)
{
throw new ArgumentNullExc eption("buf");
}
if (off < 0 || len < 0 || off + len > buf.Length)
{
throw new ArgumentOutOfRa ngeException();
}
crc ^= CrcSeed;
while (--len >= 0)
{
crc = CrcTable[(crc ^ buf[off++]) & 0xFF] ^ (crc >> 8);
}
crc ^= CrcSeed;
}
#endregion
}
}
--
William Stacey, MVP
http://mvp.support.microsoft.com
"Chang" <no*****@thisad dress.com> wrote in message
news:OH******** ******@TK2MSFTN GP14.phx.gbl... How to get SHA1 or MD5 of a big file (+5MB - 20GB) as I can't read 20GB
into memory.
--
Chang.
On Thu, 3 Feb 2005 21:57:52 -0500, "William Stacey [MVP]"
<st***********@ mvps.org> wrote:
<snip>
Wouldn't
Stream stream=...
SHA1 sha=new SHA1CryptoServi ceProvider();
byte[] hash=sha.Comput eHash(stream);
be easier? (unless I'm mistaken)
Austin
That it would :-) Did not notice that overload. Thanks Austin.
--
William Stacey, MVP http://mvp.support.microsoft.com
"Austin Ehlers" <li************ *******@ku.edu> wrote in message
news:nt******** *************** *********@4ax.c om... On Thu, 3 Feb 2005 21:57:52 -0500, "William Stacey [MVP]"
<st***********@ mvps.org> wrote:
<snip>
Wouldn't
Stream stream=...
SHA1 sha=new SHA1CryptoServi ceProvider();
byte[] hash=sha.Comput eHash(stream);
be easier? (unless I'm mistaken)
Austin
Thank you William and Austin.
"Chang" <no*****@thisad dress.com> wrote in message
news:OH******** ******@TK2MSFTN GP14.phx.gbl... How to get SHA1 or MD5 of a big file (+5MB - 20GB) as I can't read 20GB
into memory.
--
Chang.
Thank you Chang for asking this question as I my self was not sure if stream
will pull the whole file before starting calculation.
"Chang" <no*****@thisad dress.com> wrote in message
news:OH******** ******@TK2MSFTN GP14.phx.gbl... How to get SHA1 or MD5 of a big file (+5MB - 20GB) as I can't read 20GB
into memory.
--
Chang. This thread has been closed and replies have been disabled. Please start a new discussion. Similar topics | |
by: Bryan Olson |
last post by:
The current Python standard library provides two cryptographic
hash functions: MD5 and SHA-1 . The authors of MD5 originally
stated:
It is conjectured that it is computationally infeasible to
produce two messages having the same message digest.
That conjecture is false, as demonstrated by Wang, Feng, Lai and
Yu in 2004 . Just recently, Wang, Yu, and Lin showed a short-
cut solution for finding collisions in SHA-1 . Their result
|
by: Joseph Lee |
last post by:
Dear All,
Is there a function in C# that will hash code a large byte array into an int
value?
I am searching for a hash function that will convert any size data(as for me
the input is a byte array of size 20) to a value range of my choosing.
If there is no such thing, would need some idea how to program this in C#.
|
by: Michael H |
last post by:
Hi all,
I guess I don't fully understand how a SHA1 hash value is calculated
in C# / .NET for a large file... I'm trying to calculate SHA1 values
for large files that are much larger than my physical main memory. It
seems the way to derive a SHA1 value involves opening a file stream to
the large file, passing it to a byte array, and passing the byte array
to the .NET hash method.
Does this load the entire file into main memory (within...
|
by: Dil via .NET 247 |
last post by:
Fresher to .NET
Aiming to produce a resulting hash of length 24 CHARACTERS, using MD5 or SHA1 Algorithms.
According to the Class Libraries, the hash size for the SHA1 algorithm is 160 bits, and 128 bits for MD5.
After generating the hash, I convert the byte result to a base64 String, and my resulting hash lengths are as below:
MD5 - 24 characters
SHA1 - 28 Characters
|
by: Adam Carpenter |
last post by:
Hello,
I have my users passwords stored to my DB hashs created using
SHA1CryptoServiceProvider, here is the function:
Public Shared Function EncryptPassword(ByVal password As String) As Byte()
Dim encoding As New UnicodeEncoding()
Dim hashBytes As Byte() = encoding.GetBytes(password)
' Compute the SHA-1 hash
Dim sha1 As New SHA1CryptoServiceProvider()
| | |
by: voipfc |
last post by:
I have a problem compiling the hash function from PECL into my PHP.
I get the error configure: error: C preprocessor "/lib/cpp" fails
sanity check
I would like to use a plain PHP implementation of these functions.
Is there a library of them around?
|
by: Wayne Deleersnyder |
last post by:
Hi All,
I was going to write and ask if someone could help me fix the
formatting of my output for hash values, but I believe I got it right
now. But, because I couldn't find any website or tutorial to help me
with this issue I figured I'd make a post just in case someone else
runs into the same issue.
....
|
by: Ashish Khandelwal |
last post by:
As MSDN is not giving us guarantee upon uniqueness of Hash Code, so
could any one suggest me that how to generate a unique Hash Code for
same string always, and generate different-2 Hash Code Different-2
string.
|
by: LaundroMat |
last post by:
Hi -
I'm trying to calculate unique hash values for binary files,
independent of their location and filename, and I was wondering
whether I'm going in the right direction.
Basically, the hash values are calculated thusly:
f = open('binaryfile.bin')
import hashlib
|
by: Hystou |
last post by:
Most computers default to English, but sometimes we require a different language, especially when relocating. Forgot to request a specific language before your computer shipped? No problem! You can effortlessly switch the default language on Windows 10 without reinstalling. I'll walk you through it.
First, let's disable language synchronization. With a Microsoft account, language settings sync across devices. To prevent any complications,...
|
by: Oralloy |
last post by:
Hello folks,
I am unable to find appropriate documentation on the type promotion of bit-fields when using the generalised comparison operator "<=>".
The problem is that using the GNU compilers, it seems that the internal comparison operator "<=>" tries to promote arguments from unsigned to signed.
This is as boiled down as I can make it.
Here is my compilation command:
g++-12 -std=c++20 -Wnarrowing bit_field.cpp
Here is the code in...
| | |
by: jinu1996 |
last post by:
In today's digital age, having a compelling online presence is paramount for businesses aiming to thrive in a competitive landscape. At the heart of this digital strategy lies an intricately woven tapestry of website design and digital marketing. It's not merely about having a website; it's about crafting an immersive digital experience that captivates audiences and drives business growth.
The Art of Business Website Design
Your website is...
|
by: agi2029 |
last post by:
Let's talk about the concept of autonomous AI software engineers and no-code agents. These AIs are designed to manage the entire lifecycle of a software development project—planning, coding, testing, and deployment—without human intervention. Imagine an AI that can take a project description, break it down, write the code, debug it, and then launch it, all on its own....
Now, this would greatly impact the work of software developers. The idea...
|
by: conductexam |
last post by:
I have .net C# application in which I am extracting data from word file and save it in database particularly. To store word all data as it is I am converting the whole word file firstly in HTML and then checking html paragraph one by one.
At the time of converting from word file to html my equations which are in the word document file was convert into image.
Globals.ThisAddIn.Application.ActiveDocument.Select();...
|
by: TSSRALBI |
last post by:
Hello
I'm a network technician in training and I need your help.
I am currently learning how to create and manage the different types of VPNs and I have a question about LAN-to-LAN VPNs.
The last exercise I practiced was to create a LAN-to-LAN VPN between two Pfsense firewalls, by using IPSEC protocols.
I succeeded, with both firewalls in the same network. But I'm wondering if it's possible to do the same thing, with 2 Pfsense firewalls...
|
by: adsilva |
last post by:
A Windows Forms form does not have the event Unload, like VB6. What one acts like?
|
by: 6302768590 |
last post by:
Hai team
i want code for transfer the data from one system to another through IP address by using C# our system has to for every 5mins then we have to update the data what the data is updated we have to send another system
| | |
by: bsmnconsultancy |
last post by:
In today's digital era, a well-designed website is crucial for businesses looking to succeed. Whether you're a small business owner or a large corporation in Toronto, having a strong online presence can significantly impact your brand's success. BSMN Consultancy, a leader in Website Development in Toronto offers valuable insights into creating effective websites that not only look great but also perform exceptionally well. In this comprehensive...
| |
