large datasets!!

[pilafi]

Hallo guys!!I have a problem and i would like your help.I have to make a reader for a platform,where reads csv files.So, i made this read and its possible to read now csv files.But the problem is that the size of some csv files is 300MB the largest one.In that case my program breaks!I include with my question the reading part of my code and a test data i create just to see how the data looks like!!

reading part:::::::::::::::::::::::::::::::::::::::::::::: :::::::::::::::::::::::::::::::::::::::::::::::::: ::

#ifdef _MSC_VER
#pragma warning ( disable : 4786 )
#endif
#include <iostream>
#include <stdio.h>
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include "itkExceptionObject.h"
#include "itkIOCommon.h"
#include "itkCSVImageIO.h"
#include "itksys/SystemTools.hxx"
#include "stdafx.h"
 
int get_next_double(char **str, double *d)
{
    char start=0;
    char isfloat=0;
    char tmp_num[10];
    int i=0,j=0;
    char *end;
 
    memset(tmp_num,0,10);
 
    end=(*str)+strlen(*str);
 
    while(((*str)++)<=end)
    { 
        if( isdigit(**str) )
        //if (((**str)>=0) && ((**str)<=9))
 
 
        {
            start=1;
            tmp_num[j] = **str;
            j=j+1;
        }
        //if( ispunct(**str) && start==1)
        if ((**str)=='.' && start==1)
 
        {
            isfloat=1;
            tmp_num[j] = **str;
            j=j+1;
        }
 
        if( (**str==' ') || (**str==0) )
        {
            if(isfloat==1 && start==1)
            {
                *d=atof(tmp_num);
                return 0;
            }
            if(*(*str+1)==0)
            {
                return 1;
            }
            start=0;
            isfloat=0;
            j=0;
            memset(tmp_num,0,10);
        }
 
    }    
 
    return 1;
}
 
int count_numbers(const char *str)
{
 
    char *tmp = 0;
    tmp=(char*)str;
    int counter=0;
 
 
    while(*tmp!='\n')
    {
    if (*tmp=='.')
    {counter=counter+1;}
    tmp=tmp+1;
    }
    return counter;
 
 
}
 
namespace itk
{
 
CSVImageIO::CSVImageIO()
{
  this->SetNumberOfDimensions(3); // CSV is 3D.
  this->SetNumberOfComponents(1); // CSV only has one component.
  m_ByteOrder = LittleEndian;
  m_FileType = ASCII;
 
} 
 
CSVImageIO::~CSVImageIO()
{
}
 
void CSVImageIO::PrintSelf(std::ostream& os, Indent indent) const
{
  Superclass::PrintSelf(os, indent);
}
 
bool CSVImageIO::CanReadFile( const char* filename ) 
{ 
  std::cout << "CSVImageIO::CanReadFile() " << std::endl;
  //
  // If the file exists, and have extension .csv, then we are good to read it.
  //
  if( !itksys::SystemTools::FileExists( filename ) )
    {
    std::cout << "File doesn't exist" << std::endl;
    return false;
    }
std::cout << itksys::SystemTools::GetFilenameLastExtension( filename ) << std::endl;
  if( itksys::SystemTools::GetFilenameLastExtension( filename ) != ".csv" )
    {
    std::cout << "Wrong extension" << std::endl;
    return false;
    }
 
  return true;
}
 
 
void CSVImageIO::ReadImageInformation()
{ 
    char onedataline[300000];
    char headerLine[100];
    _CRT_FLOAT ftemp;
    _CRT_FLOAT ScanningLength;
    _CRT_FLOAT ScanningResolution;
    _CRT_FLOAT IndexLength;
    _CRT_FLOAT IndexResolution;
    char str[10];
    int retval=0;
 
  // CSV only reads 8-bits unsigned short images.
  this->SetPixelType( SCALAR );
  this->SetComponentType( USHORT );
 
    // read our own information
    m_InputStream.open(this->m_FileName.c_str(), std::ios::in);
    m_InputStream.seekg(0, std::ios::beg);
    for (int i=0; i<33; i++)
    {
        m_InputStream.getline(headerLine, 100, '\n');
    }
    m_InputStream.getline(headerLine, 100, ':');
    m_InputStream.getline(headerLine, 100, '\n');
    strcpy(str,headerLine);
    retval=_atoflt( &ScanningLength,str);
 
    m_InputStream.getline(headerLine, 100, ':');
    m_InputStream.getline(headerLine, 100, '\n');
    strcpy(str,headerLine);
    retval=_atoflt( &ScanningResolution,str);
 
    m_InputStream.getline(headerLine, 100, '\n');
 
    m_InputStream.getline(headerLine, 100, ':');
    m_InputStream.getline(headerLine, 100, '\n');
    strcpy(str,headerLine);
    retval=_atoflt( &IndexLength,str);
 
    m_InputStream.getline(headerLine, 100, ':');
    m_InputStream.getline(headerLine, 100, '\n');
    strcpy(str,headerLine);
    retval=_atoflt( &IndexResolution,str);
 
    m_InputStream.getline(headerLine, 100, '\n');
 
    m_InputStream.getline(headerLine, 100, '\n');
 
    m_InputStream.getline(headerLine, 100, ':');
    m_InputStream.getline(headerLine, 100, '\n');
    strcpy(str,headerLine);
    retval=_atoflt( &ftemp,str);
 
    m_InputStream.getline(headerLine, 100, ':');
    m_InputStream.getline(headerLine, 100, '\n');
    strcpy(str,headerLine);
    retval=_atoflt( &ftemp,str);
 
    m_InputStream.getline(headerLine, 100, ':');
    m_InputStream.getline(headerLine, 100, '\n');
    strcpy(str,headerLine);
    retval=_atoflt( &ftemp,str);
 
    m_InputStream.getline(headerLine, 100, '\n');
    m_InputStream.getline(headerLine, 100, '\n');
    m_InputStream.getline(headerLine, 100, '\n');
 
    //data follows here
    data_position = m_InputStream.tellg();
    m_InputStream.getline(onedataline, 300000, '\n');
 
 
    this->SetOrigin( 0, 0.0 );
    this->SetOrigin( 1, 0.0 );
    this->SetOrigin( 2, 0.0 );
 
    this->SetDimensions( 0, count_numbers(onedataline)-1);
    this->SetDimensions( 1, ((ScanningLength.f)/(ScanningResolution.f)) );
    this->SetDimensions( 2, ((IndexLength.f)/(IndexResolution.f)) );
 
    this->SetSpacing( 0, ((0.0027)*100) );
    this->SetSpacing( 1, ((ScanningResolution.f)*100)  );
    this->SetSpacing( 2, ((IndexResolution.f)*100) );
 
  m_InputStream.close();
}
 
 
void CSVImageIO::Read( void * buffer)
{ 
    char onedataline[300000];
    char *tmp=0;
    double d=0;
    unsigned short data=0;
    unsigned short * inptr = static_cast< unsigned short * >( buffer );
    unsigned short * tmp_buf=0;
 
    //copy the pointer to the buffer, because we are going to change the pointer soon
    tmp_buf=inptr;
 
    //print the data for check
    std::ofstream     m_OutputStream("D:\\MIP\\outdata.csv");
 
    //open file
    this->m_InputStream.open(this->m_FileName.c_str(), std::ios::in);
 
    //seek to data position
    m_InputStream.seekg(data_position);
 
    //clear temporay buffer that hold a complete line
    //memset(onedataline,0,65000);
 
    //read in one data line
    //m_InputStream.getline(onedataline, 65000, '\n');
 
    //const unsigned int m = this->GetDimensions( 0 );
    //const unsigned int n = this->GetDimensions( 1 );
    //const unsigned int o = this->GetDimensions( 2 );
 
    int i=0,j=0,k=0;
 
    //memset(inptr,0,m*n*o);
 
    do 
    {    
        memset(onedataline,0,300000);
        m_InputStream.getline(onedataline, 300000, '\n');
        tmp=(char*)onedataline;
        while(get_next_double(&tmp,&d)==0)
        {
            data=(unsigned short)((double)fabs(d)*10000);
            *(tmp_buf++)=data;
            i++;
            m_OutputStream << data << " ";
        }
 
            m_OutputStream <<endl;
 
        //m_InputStream.getline(onedataline, 65000, '\n');
    }while(!m_InputStream.eof());
 
    std::cout << "data read: " << i << std::endl;
/*
  const unsigned int nx = this->GetDimensions( 0 );
  const unsigned int ny = this->GetDimensions( 1 );
  const unsigned int nz = this->GetDimensions( 2 );
 
  ImageIORegion regionToRead = this->GetIORegion();
 
  ImageIORegion::SizeType  size  = regionToRead.GetSize();
  ImageIORegion::IndexType start = regionToRead.GetIndex();
 
  const unsigned int mx = size[0];
  const unsigned int my = size[1];
  const unsigned int mz = size[2];
 
  const unsigned int sx = start[0];
  const unsigned int sy = start[1];
  const unsigned int sz = start[2];
 
  unsigned short * inptr = static_cast< unsigned short * >( buffer );
 
    //seek to data
 
    //read data
    //store into buffer
*/
  this->m_InputStream.close(); 
} 
 
 
bool CSVImageIO::CanWriteFile( const char * name )
{
  //
  // CSV is not affraid of writing either !!
  // 
  return true;
}
 
 
void 
CSVImageIO
::WriteImageInformation(void)
{
  // add writing here
}
 
 
/**
 *
 */
void 
CSVImageIO
::Write( const void* buffer) 
{
}
 
/** Given a requested region, determine what could be the region that we can
 * read from the file. This is called the streamable region, which will be
 * smaller than the LargestPossibleRegion and greater or equal to the 
RequestedRegion */
ImageIORegion 
CSVImageIO
::CalculateStreamableReadRegionFromRequestedRegion( const ImageIORegion & requested ) const
{
  std::cout << "CSVImageIO::CalculateStreamableReadRegionFromRequestedRegion()" << std::endl;
  std::cout << "Requested region = " << requested << std::endl;
  //
  // CSV is the ultimate streamer.
  //
  ImageIORegion streamableRegion = requested;
 
  std::cout << "StreamableRegion = " << streamableRegion << std::endl;
 
  return streamableRegion;
}
 
 
} // end namespace itk
 

and the test data:::::::::::::::::::::::::::::::::::::::::::::: :::::::::::::::::::::::::::::::::::::::::::::


_____________C-Scan Settings__________________________
Distance Unit: mm
Scanning Axis: 0
Scanning Length: 1.000
Scanning Resolution: 0.2500
Index Axis: 1
Index Length: 1.000
Index Resolution: 0.2500
_____________Output Waveform Settings____________________
Data Ch: 1
Waveform Start(us): 7.182
Waveform Length(#): 0.000
Waveform Sampling Rate(MHz): 4000.000

______Output Data (Format for each row: index Number, scanning number, Waveform)______

0 0 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
0 1 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
0 2 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
0 3 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
1 0 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
1 1 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
1 2 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
1 3 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
2 0 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
2 1 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
2 2 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
2 3 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
3 0 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
3 1 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
3 2 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
3 3 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000

I am looking forward for your answer!!
Thanx!!

[RedSon]

Please enclose your posted code in [code] tags (See How to Ask a Question).

This makes it easier for our Experts to read and understand it. Failing to do so creates extra work for the moderators, thus wasting resources, otherwise available to answer the members' questions.

Please use [code] tags in future.

MODERATOR

[pilafi]

Please enclose your posted code in [code] tags (See How to Ask a Question).

This makes it easier for our Experts to read and understand it. Failing to do so creates extra work for the moderators, thus wasting resources, otherwise available to answer the members' questions.

Please use [code] tags in future.

MODERATOR

So sorry,i couldnt understand it.I will post again my question.
Sorry!

[RedSon]

So sorry,i couldnt understand it.I will post again my question.
Sorry!

No don't post again. That is double posting and is a waste of time and energy and will get you one step closer to being banned from this site. Your original post is fine. Just use [code] tags in the future.

-MODERATOR

[pilafi]

Ok thnx!!Can anybody help me?I need an advice as soon as possible if its easy.
Thnx guys!!

[weaknessforcats]

My first suggestion is to get your arrays off the stack and onto the heap. I would never let the compiler manage the memory.

[pilafi]

My first suggestion is to get your arrays off the stack and onto the heap. I would never let the compiler manage the memory.

Can you tell practically how can i do this??
Thnx for answering to me!!

[weaknessforcats]

Just allocate your own memory:

char onedataline[300000];
 
//becomes:
 
char* onedataline = new char[300000];
 

This should require no code chnages anywhere since the heap array and the stack array use the same syntax. You just need to delete the array when you are finished with it.

I would do this will all the local variables.

[pilafi]

Just allocate your own memory:

char onedataline[300000];
 
//becomes:
 
char* onedataline = new char[300000];
 

This should require no code chnages anywhere since the heap array and the stack array use the same syntax. You just need to delete the array when you are finished with it.

I would do this will all the local variables.

Thank you very much!!I will try it and i will tell you what happened!!

[pilafi]

Thank you very much!!I will try it and i will tell you what happened!!

Ok i followed your advice and now its working for a large file 200MB.But
there is a strange artifact in every image a white slice.Do you have any idea
how this slice created?
Tnx for help!!
I am waiting your answer!!

[weaknessforcats]

But
there is a strange artifact in every image a white slice.Do you have any idea
how this slice created?

I'm not sure what you are talking about. As in "strange artifact" and "white slice".

If you allocate an array, it is required that a single contiguous allocation be made. The contents of the elements are indeterminate unless you have default constuctors.

[pilafi]

I'm not sure what you are talking about. As in "strange artifact" and "white slice".

If you allocate an array, it is required that a single contiguous allocation be made. The contents of the elements are indeterminate unless you have default constuctors.

Sorry again but i want you to make me a clarification.You told me to allocate my
own memory and i did only for the two spots in my code tha it refered:
code (cpp):
char onedataline[300000];

I have to do it somewhere else??

Sorry but i am new and i need your help!!

[RedSon]

Sorry again but i want you to make me a clarification.You told me to allocate my
own memory and i did only for the two spots in my code tha it refered:
code (cpp):
char onedataline[300000];

I have to do it somewhere else??

Sorry but i am new and i need your help!!

First I think you should explain what "strange artifact" and "white slice" mean.

[pilafi]

First I think you should explain what "strange artifact" and "white slice" mean.

Before i make this change you advice me my code could work for csv files until 80MB.The picture i had as a result in my screen was an image as i would like to be with 3 layers.Now after i did the change the image i get is totally different.I cant see the layers instead i can see a white layer and some strange spots.But the positive is that i can open a 200MB file, but its not correct.

I am looking forward for your answer!!
Thnx!!

[weaknessforcats]

Before i make this change you advice me my code could work for csv files until 80MB.The picture i had as a result in my screen was an image as i would like to be with 3 layers.Now after i did the change the image i get is totally different.I cant see the layers instead i can see a white layer and some strange spots.But the positive is that i can open a 200MB file, but its not correct.

You never answered RedSon's question, or mine: What is this "artifact" and what is this "white slice"??????

A csv file is just a file where each record is a line and the values in the line a separated by commas. That's all it is.

The actual meaning of the values differes by application. You cannot use a csv file for a picture unless you already know the structure of the file. Just opening any old csv file will get you garbage.

It's starting to look like an application bug and not a memory allocation problem.

Exactly what are you trying to do?

[pilafi]

You never answered RedSon's question, or mine: What is this "artifact" and what is this "white slice"??????

A csv file is just a file where each record is a line and the values in the line a separated by commas. That's all it is.

The actual meaning of the values differes by application. You cannot use a csv file for a picture unless you already know the structure of the file. Just opening any old csv file will get you garbage.

It's starting to look like an application bug and not a memory allocation problem.

Exactly what are you trying to do?

Hallo!!
An artifact is a structure or feature that is yisible only as a result of external action or experimental error.Also known as disturbance in biopotential signals.

The csv files i have is exctly like the example i sent you above.
Opening these files i expect to see an image with 3 layers.And i am trying to open all these files and seethe images.

I am looking forward for your answer.

[RedSon]

Hallo!!
An artifact is a structure or feature that is yisible only as a result of external action or experimental error.Also known as disturbance in biopotential signals.

The csv files i have is exctly like the example i sent you above.
Opening these files i expect to see an image with 3 layers.And i am trying to open all these files and seethe images.

I am looking forward for your answer.

CSV files are not images. If you open a csv file you will see some data separated by commas. If you are using a CSV file to represent an image, for example, where each entry represents an RGB value for a pixel then you are going to have to write a custom application viewer.

[pilafi]

CSV files are not images. If you open a csv file you will see some data separated by commas. If you are using a CSV file to represent an image, for example, where each entry represents an RGB value for a pixel then you are going to have to write a custom application viewer.

Ok then lets assume that we have another type of file.A file like this one i send you.These values give us a three layer image.Now it is possible to imagine the problem??

Sorry if i am not clear some times but i am new.
thank you for your help!

[arnaudk]

I suspect English is not your native language! Taking a wild guess, it looks like you wrote a program using some libraries which somehow converts your csv data (presumably representing tissue imaging scans or something) into actual images (bitmaps, etc) you can look at. If you see artefacts or "white regions" in your resulting images, then this is probably a problem with the way you're converting the csv data into images, do you actually understand how this is being done in your program? If the csv data represents electrical signals from your tissue scans, for example, perhaps you need to rescale your data by equating the maximum and minimum voltage to the maximum and minimum pixel brightness supported by the target image format?

[pilafi]

Good morning!!Its true that English its not my native language and also i am new
in the programming.All i had to do was to make a reader for a specific kind of files and integrate it in a visualization platform that other people make.My files has the following form:

_____________C-Scan Settings__________________________
Distance Unit: mm
Scanning Axis: 0
Scanning Length: 1.000
Scanning Resolution: 0.2500
Index Axis: 1
Index Length: 1.000
Index Resolution: 0.2500
_____________Output Waveform Settings____________________
Data Ch: 1
Waveform Start(us): 7.182
Waveform Length(#): 0.000
Waveform Sampling Rate(MHz): 4000.000

______Output Data (Format for each row: index Number, scanning number, Waveform)______

0 0 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
0 1 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
0 2 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
0 3 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
1 0 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
1 1 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
1 2 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
1 3 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
2 0 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
2 1 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
2 2 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
2 3 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
3 0 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
3 1 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
3 2 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000
3 3 0.0000 1.0000 0.0000 0.0000 2.0000 0.0000 0.0000 0.0000 3.0000 0.0000 0.0000

and when i run the program i get the following..:
[IMG]test[/IMG]

[pilafi]

Sorry how can i insert an image??

[arnaudk]

It sounds to me more like you don't understand what kind of input the visualization platform expects. Read its documentation to understand how it works.