Commit 9bf1a28a by Francois Gygi

implement parallel I/O using SharedFilePtr


git-svn-id: http://qboxcode.org/svn/qb/trunk@604 cba15fb0-1239-40c8-b417-11db7ca47a34
parent 89f62460
......@@ -3,7 +3,7 @@
// SampleWriter.C:
//
////////////////////////////////////////////////////////////////////////////////
// $Id: SampleWriter.C,v 1.6 2008-04-11 05:34:34 fgygi Exp $
// $Id: SampleWriter.C,v 1.7 2008-04-15 01:36:44 fgygi Exp $
#include "SampleWriter.h"
......@@ -11,6 +11,7 @@
#include "fstream"
#include "qbox_xmlns.h"
#include "Timer.h"
#include "SharedFilePtr.h"
#include <sstream>
#include <iomanip>
......@@ -73,6 +74,7 @@ void SampleWriter::writeSample(const Sample& s, const string filename,
MPI_Info info;
MPI_Info_create(&info);
MPI_Offset fsize;
SharedFilePtr sfp(ctxt_.comm(),fh);
int err;
err = MPI_File_open(ctxt_.comm(),(char*) filename_cstr,
......@@ -106,42 +108,39 @@ void SampleWriter::writeSample(const Sample& s, const string filename,
ss << s.atoms;
header += ss.str();
int len = header.size();
err = MPI_File_write(fh,(void*)header.c_str(),
len,MPI_CHAR,&status);
err = MPI_File_write_at(sfp.file(),sfp.mpi_offset(),(void*)header.c_str(),
len,MPI_CHAR,&status);
if ( err != 0 )
cout << ctxt_.mype() << ": error in MPI_File_write: header "
<< err << endl;
sfp.advance(len);
}
sfp.sync();
if ( !atomsonly )
{
s.wf.write(fh,encoding,"wavefunction");
s.wf.write(sfp,encoding,"wavefunction");
if ( s.wfv != 0 )
s.wfv->write(fh,encoding,"wavefunction_velocity");
s.wfv->write(sfp,encoding,"wavefunction_velocity");
}
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
MPI_File_seek(fh,0,MPI_SEEK_END);
sfp.sync();
if ( ctxt_.onpe0() )
{
char *trailer = "</fpmd:sample>\n";
int len = strlen(trailer);
err = MPI_File_write(fh,(void*)trailer,len,MPI_CHAR,&status);
err = MPI_File_write_at(sfp.file(),sfp.mpi_offset(),(void*)trailer,
len,MPI_CHAR,&status);
if ( err != 0 )
cout << ctxt_.mype() << ": error in MPI_File_write: trailer "
<< err << endl;
sfp.advance(len);
}
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
sfp.sync();
MPI_File_get_size(fh,&fsize);
file_size = fsize;
file_size = sfp.offset();
err = MPI_File_close(&fh);
if ( err != 0 )
......
......@@ -3,7 +3,7 @@
// SlaterDet.C
//
////////////////////////////////////////////////////////////////////////////////
// $Id: SlaterDet.C,v 1.47 2008-04-11 05:34:34 fgygi Exp $
// $Id: SlaterDet.C,v 1.48 2008-04-15 01:36:44 fgygi Exp $
#include "SlaterDet.h"
#include "FourierTransform.h"
......@@ -1326,7 +1326,7 @@ void SlaterDet::print(ostream& os, string encoding, double weight, int ispin,
}
////////////////////////////////////////////////////////////////////////////////
void SlaterDet::write(MPI_File& fh, string encoding, double weight, int ispin,
void SlaterDet::write(SharedFilePtr& sfp, string encoding, double weight, int ispin,
int nspin)
{
FourierTransform ft(*basis_,basis_->np(0),basis_->np(1),basis_->np(2));
......@@ -1632,25 +1632,23 @@ void SlaterDet::write(MPI_File& fh, string encoding, double weight, int ispin,
wbuf.append(rbuf,rbufsize);
delete [] rbuf;
seg[nloc].clear();
}
// wbuf now contains the data to be written in the correct order
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
MPI_File_seek(fh,0,MPI_SEEK_END);
// compute offsets
sfp.sync();
MPI_Offset off;
long long int local_offset,current_offset;
MPI_File_get_position(fh,&off);
current_offset = off;
current_offset = sfp.offset();
// compute local offset of next write
long long int local_size = wbuf.size();
MPI_Scan(&local_size, &local_offset, 1,
MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
MPI_LONG_LONG, MPI_SUM, ctxt_.comm());
// add base and correct for inclusive scan by subtracting local_size
local_offset += current_offset - local_size;
off = local_offset;
......@@ -1658,26 +1656,26 @@ void SlaterDet::write(MPI_File& fh, string encoding, double weight, int ispin,
MPI_Status status;
// write wbuf from all tasks using computed offset
int err = MPI_File_write_at_all(fh,off,(void*)wbuf.c_str(),wbuf.size(),
int len = wbuf.size();
int err = MPI_File_write_at_all(sfp.file(),off,(void*)wbuf.c_str(),len,
MPI_CHAR,&status);
if ( err != 0 )
cout << ctxt_.mype()
<< " error in MPI_File_write_at_all" << endl;
sfp.set_offset(local_offset+len);
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
MPI_File_seek(fh,0,MPI_SEEK_END);
sfp.sync();
if ( ctxt_.onpe0() )
{
string s("</slater_determinant>\n");
int err = MPI_File_write(fh,(void*) s.c_str(),
int err = MPI_File_write_at(sfp.file(),sfp.mpi_offset(),(void*) s.c_str(),
s.size(),MPI_CHAR,&status);
if ( err != 0 )
cout << ctxt_.mype()
<< " error in MPI_File_write, slater_determinant trailer"
<< endl;
sfp.advance(s.size());
}
}
......
......@@ -3,7 +3,7 @@
// SlaterDet.h
//
////////////////////////////////////////////////////////////////////////////////
// $Id: SlaterDet.h,v 1.22 2008-01-26 01:34:11 fgygi Exp $
// $Id: SlaterDet.h,v 1.23 2008-04-15 01:36:44 fgygi Exp $
#ifndef SLATERDET_H
#define SLATERDET_H
......@@ -19,7 +19,7 @@ class FourierTransform;
#include "Timer.h"
#include <string>
#include <map>
#include "mpi.h"
#include "SharedFilePtr.h"
typedef std::map<std::string,Timer> TimerMap;
......@@ -91,7 +91,7 @@ class SlaterDet
SlaterDet& operator=(SlaterDet& rhs);
void print(std::ostream& os, std::string encoding, double weight, int ispin,
int nspin);
void write(MPI_File& fh, std::string encoding, double weight, int ispin,
void write(SharedFilePtr& fh, std::string encoding, double weight, int ispin,
int nspin);
void info(std::ostream& os);
};
......
......@@ -3,11 +3,12 @@
// Wavefunction.C
//
////////////////////////////////////////////////////////////////////////////////
// $Id: Wavefunction.C,v 1.31 2008-04-11 05:34:34 fgygi Exp $
// $Id: Wavefunction.C,v 1.32 2008-04-15 01:36:44 fgygi Exp $
#include "Wavefunction.h"
#include "SlaterDet.h"
#include "jacobi.h"
#include "SharedFilePtr.h"
#include <vector>
#include <iomanip>
#include <sstream>
......@@ -658,12 +659,9 @@ void Wavefunction::print(ostream& os, string encoding, string tag) const
}
////////////////////////////////////////////////////////////////////////////////
void Wavefunction::write(MPI_File& fh, string encoding, string tag) const
void Wavefunction::write(SharedFilePtr& sfp, string encoding, string tag) const
{
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
MPI_File_seek(fh,0,MPI_SEEK_END);
sfp.sync();
if ( ctxt_.onpe0() )
{
......@@ -690,23 +688,22 @@ void Wavefunction::write(MPI_File& fh, string encoding, string tag) const
string str(os.str());
int len = str.size();
MPI_Status status;
int err = MPI_File_write(fh,(void*)str.c_str(),len,MPI_CHAR,&status);
int err = MPI_File_write_at(sfp.file(),sfp.mpi_offset(),(void*)str.c_str(),
len,MPI_CHAR,&status);
if ( err != 0 )
cout << " Wavefunction::write: error in MPI_File_write" << endl;
sfp.advance(len);
}
for ( int ispin = 0; ispin < nspin_; ispin++ )
{
for ( int ikp = 0; ikp < kpoint_.size(); ikp++ )
{
sd_[ispin][ikp]->write(fh,encoding,weight_[ikp],ispin,nspin_);
sd_[ispin][ikp]->write(sfp,encoding,weight_[ikp],ispin,nspin_);
}
}
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
MPI_File_seek(fh,0,MPI_SEEK_END);
sfp.sync();
if ( ctxt_.onpe0() )
{
......@@ -715,9 +712,11 @@ void Wavefunction::write(MPI_File& fh, string encoding, string tag) const
string str(os.str());
int len = str.size();
MPI_Status status;
int err = MPI_File_write(fh,(void*)str.c_str(),len,MPI_CHAR,&status);
int err = MPI_File_write_at(sfp.file(),sfp.mpi_offset(),(void*)str.c_str(),
len,MPI_CHAR,&status);
if ( err != 0 )
cout << " Wavefunction::write: error in MPI_File_write" << endl;
sfp.advance(len);
}
}
......
......@@ -3,16 +3,16 @@
// Wavefunction.h
//
////////////////////////////////////////////////////////////////////////////////
// $Id: Wavefunction.h,v 1.19 2008-01-26 01:34:11 fgygi Exp $
// $Id: Wavefunction.h,v 1.20 2008-04-15 01:36:44 fgygi Exp $
#ifndef WAVEFUNCTION_H
#define WAVEFUNCTION_H
#include "D3vector.h"
#include "UnitCell.h"
#include "SharedFilePtr.h"
#include <vector>
#include <complex>
#include "mpi.h"
class SlaterDet;
class Context;
......@@ -102,7 +102,7 @@ class Wavefunction
std::complex<double> dot(const Wavefunction& wf) const;
void print(std::ostream& os, std::string encoding, std::string tag) const;
void write(MPI_File& fh, std::string encoding, std::string tag) const;
void write(SharedFilePtr& fh, std::string encoding, std::string tag) const;
void info(std::ostream& os, std::string tag) const;
};
std::ostream& operator << ( std::ostream& os, const Wavefunction& wf );
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment