How to read and write to files larger than 150mb with MPI?

[Rob S]

Hi,
I am working on an MPI project using vb.net. If there is anyone in here that has experience using MPI.NET which is distributed by Indiana.edu please get back to me. I am having several unexplainable issues with this SDK and someone with experience migh have some tips that I could use.
I am particularly having issues working with large data files. The program that I am working on works fine with files upto 150MB but starts to fail with anything larger.
I am using Windows XP prof. and the system is a 1.8 Ghz with 1 GB of RAM.

The program i am writing simply reads a text file and then find the number of occurences of a "seaerchterm" in the file.

Thanks.
Rob

Here is the code:

Imports System.IO
Imports System.Threading.Tasks
Imports System.Diagnostics
Imports System.Text
Imports System.Text.RegularExpressions
 
 
Module Module1
    Dim FS As FileStream
    Dim comm As MPI.Intracommunicator
    Dim AllocationSize As Long
    Dim StartPos As Long
    Dim EndPos As Long
    Dim FileName As String
    Dim FileSize As Long
    Dim TextToRead As StringBuilder
    Dim mode As String
    Dim SearchTerm As String
    Dim matchCount As Long = 0
 
 
    Sub Main()
 
        Dim env As MPI.Environment = New MPI.Environment(Split(Command, " "))
        comm = MPI.Communicator.world
 
        Dim sw As New Stopwatch
        sw.Start()
 
        If Command() = "" Then
            Console.WriteLine("Please enter a valid filename to be processed.")
            Exit Sub
        Else
            Dim args As String() = Split(Command, ",")
            FileName = args(0)
            SearchTerm = args(1)
        End If
 
        'read in the file to be processed
        FS = New FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.Read)
        FileSize = FS.Length
 
        If comm.Size > FileSize Then
            Console.WriteLine("Quitting. Too many cooks spoil the broth.")
            Exit Sub
        End If
        AllocationSize = Fix(FileSize / comm.Size)
 
        If comm.Rank = comm.Size - 1 Then
 
            StartPos = (comm.Rank * AllocationSize) + 1
            EndPos = FileSize
        Else
            StartPos = (comm.Rank * AllocationSize) + 1
            EndPos = (comm.Rank + 1) * AllocationSize
        End If
 
        '______________________________________________________________
        Dim br As New StreamReader(FS)
        br.BaseStream.Seek(StartPos, SeekOrigin.Begin)
        Dim WordLen As Integer = SearchTerm.Length - 1
        Dim matches As Boolean = False
        Dim posCounter As Long = StartPos
        Dim i As Integer
        Dim c As Integer
        Console.WriteLine("TypeOf c is " & c.GetType.ToString)
        Console.WriteLine("TypeOf i is " & i.GetType.ToString)
        Console.WriteLine("TypeOf wordlen is " & WordLen.GetType.ToString)
        Console.WriteLine("TypeOf ascw searchterm.char is " & AscW(SearchTerm.Chars(0)).GetType.ToString)
 
        Do  'br.Peek <> -1
            c = br.Read
            posCounter = posCounter + 1
            If c = AscW(SearchTerm.Chars(0)) Then
                Console.Write("")
                matches = True
                For i = 1 To WordLen
                    c = br.Read
                    posCounter = posCounter + 1
                    If c = AscW(SearchTerm.Chars(i)) Then 'And posCounter <= EndPos Then
                        matches = True
                    Else
                        matches = False
                        Exit For
                    End If
                Next
            Else
                matches = False
            End If
 
            If matches = True Then
                matchCount += 1
                matches = False
            End If
 
        Loop While posCounter <= EndPos
 
        Console.WriteLine(comm.Rank & " found: " & matchCount & " between " & StartPos & "->" & posCounter)
        br.Close()
        '______________________________________________________________
 
 
 
        'DoWordSearch()
        'DoFileRead()
        'DoFileWrite(TextToRead.ToString)
        'Console.WriteLine("Worker {0} found {1} occurence.", comm.Rank, FindStringCount(TextToRead.ToString))
 
        Dim TotalMatches = comm.Reduce(Of Long)(matchCount, MPI.Operation(Of Long).Add, 0)
        If comm.Rank = 0 Then
            Console.WriteLine("Done Processing (MapReducing) and writing processed contents to \\{0}", MPI.Environment.ProcessorName & "\MPI\Output.txt")
            Console.WriteLine("Total matches found - " & TotalMatches)
        End If
 
        sw.Stop()
        Dim MaxTime As Long = sw.ElapsedMilliseconds
        Dim TimeToProcess As Long = comm.Reduce(Of Long)(MaxTime, MPI.Operation(Of Long).Max, 0)
        If comm.Rank = 0 Then
            Console.WriteLine()
            Console.WriteLine("The parallel read and write operation tooka max of:{0} milliseconds with a file of size {1}", TimeToProcess, FileSize)
            Console.WriteLine()
        End If
 
        FS.Close()
        comm = Nothing
        env.Dispose()
 
    End Sub