By using this site, you agree to our updated Privacy Policy and our Terms of Use. Manage your Cookies Settings.
459,696 Members | 1,607 Online
Bytes IT Community
+ Ask a Question
Need help? Post your question and get tips & solutions from a community of 459,696 IT Pros & Developers. It's quick & easy.

How to read and write to files larger than 150mb with MPI?

P: 14
Hi,
I am working on an MPI project using vb.net. If there is anyone in here that has experience using MPI.NET which is distributed by Indiana.edu please get back to me. I am having several unexplainable issues with this SDK and someone with experience migh have some tips that I could use.
I am particularly having issues working with large data files. The program that I am working on works fine with files upto 150MB but starts to fail with anything larger.
I am using Windows XP prof. and the system is a 1.8 Ghz with 1 GB of RAM.

The program i am writing simply reads a text file and then find the number of occurences of a "seaerchterm" in the file.

Thanks.
Rob

Here is the code:
Expand|Select|Wrap|Line Numbers
  1. Imports System.IO
  2. Imports System.Threading.Tasks
  3. Imports System.Diagnostics
  4. Imports System.Text
  5. Imports System.Text.RegularExpressions
  6.  
  7.  
  8. Module Module1
  9.     Dim FS As FileStream
  10.     Dim comm As MPI.Intracommunicator
  11.     Dim AllocationSize As Long
  12.     Dim StartPos As Long
  13.     Dim EndPos As Long
  14.     Dim FileName As String
  15.     Dim FileSize As Long
  16.     Dim TextToRead As StringBuilder
  17.     Dim mode As String
  18.     Dim SearchTerm As String
  19.     Dim matchCount As Long = 0
  20.  
  21.  
  22.     Sub Main()
  23.  
  24.         Dim env As MPI.Environment = New MPI.Environment(Split(Command, " "))
  25.         comm = MPI.Communicator.world
  26.  
  27.         Dim sw As New Stopwatch
  28.         sw.Start()
  29.  
  30.         If Command() = "" Then
  31.             Console.WriteLine("Please enter a valid filename to be processed.")
  32.             Exit Sub
  33.         Else
  34.             Dim args As String() = Split(Command, ",")
  35.             FileName = args(0)
  36.             SearchTerm = args(1)
  37.         End If
  38.  
  39.         'read in the file to be processed
  40.         FS = New FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.Read)
  41.         FileSize = FS.Length
  42.  
  43.         If comm.Size > FileSize Then
  44.             Console.WriteLine("Quitting. Too many cooks spoil the broth.")
  45.             Exit Sub
  46.         End If
  47.         AllocationSize = Fix(FileSize / comm.Size)
  48.  
  49.         If comm.Rank = comm.Size - 1 Then
  50.  
  51.             StartPos = (comm.Rank * AllocationSize) + 1
  52.             EndPos = FileSize
  53.         Else
  54.             StartPos = (comm.Rank * AllocationSize) + 1
  55.             EndPos = (comm.Rank + 1) * AllocationSize
  56.         End If
  57.  
  58.         '______________________________________________________________
  59.         Dim br As New StreamReader(FS)
  60.         br.BaseStream.Seek(StartPos, SeekOrigin.Begin)
  61.         Dim WordLen As Integer = SearchTerm.Length - 1
  62.         Dim matches As Boolean = False
  63.         Dim posCounter As Long = StartPos
  64.         Dim i As Integer
  65.         Dim c As Integer
  66.         Console.WriteLine("TypeOf c is " & c.GetType.ToString)
  67.         Console.WriteLine("TypeOf i is " & i.GetType.ToString)
  68.         Console.WriteLine("TypeOf wordlen is " & WordLen.GetType.ToString)
  69.         Console.WriteLine("TypeOf ascw searchterm.char is " & AscW(SearchTerm.Chars(0)).GetType.ToString)
  70.  
  71.         Do  'br.Peek <> -1
  72.             c = br.Read
  73.             posCounter = posCounter + 1
  74.             If c = AscW(SearchTerm.Chars(0)) Then
  75.                 Console.Write("")
  76.                 matches = True
  77.                 For i = 1 To WordLen
  78.                     c = br.Read
  79.                     posCounter = posCounter + 1
  80.                     If c = AscW(SearchTerm.Chars(i)) Then 'And posCounter <= EndPos Then
  81.                         matches = True
  82.                     Else
  83.                         matches = False
  84.                         Exit For
  85.                     End If
  86.                 Next
  87.             Else
  88.                 matches = False
  89.             End If
  90.  
  91.             If matches = True Then
  92.                 matchCount += 1
  93.                 matches = False
  94.             End If
  95.  
  96.         Loop While posCounter <= EndPos
  97.  
  98.         Console.WriteLine(comm.Rank & " found: " & matchCount & " between " & StartPos & "->" & posCounter)
  99.         br.Close()
  100.         '______________________________________________________________
  101.  
  102.  
  103.  
  104.         'DoWordSearch()
  105.         'DoFileRead()
  106.         'DoFileWrite(TextToRead.ToString)
  107.         'Console.WriteLine("Worker {0} found {1} occurence.", comm.Rank, FindStringCount(TextToRead.ToString))
  108.  
  109.         Dim TotalMatches = comm.Reduce(Of Long)(matchCount, MPI.Operation(Of Long).Add, 0)
  110.         If comm.Rank = 0 Then
  111.             Console.WriteLine("Done Processing (MapReducing) and writing processed contents to \\{0}", MPI.Environment.ProcessorName & "\MPI\Output.txt")
  112.             Console.WriteLine("Total matches found - " & TotalMatches)
  113.         End If
  114.  
  115.         sw.Stop()
  116.         Dim MaxTime As Long = sw.ElapsedMilliseconds
  117.         Dim TimeToProcess As Long = comm.Reduce(Of Long)(MaxTime, MPI.Operation(Of Long).Max, 0)
  118.         If comm.Rank = 0 Then
  119.             Console.WriteLine()
  120.             Console.WriteLine("The parallel read and write operation tooka max of:{0} milliseconds with a file of size {1}", TimeToProcess, FileSize)
  121.             Console.WriteLine()
  122.         End If
  123.  
  124.         FS.Close()
  125.         comm = Nothing
  126.         env.Dispose()
  127.  
  128.     End Sub
  129.  
Jan 5 '11 #1
Share this question for a faster answer!
Share on Google+

Post your reply

Sign in to post your reply or Sign up for a free account.