Commit 0cc5dccf by Francois Gygi

added parallel I/O using MPI-IO


git-svn-id: http://qboxcode.org/svn/qb/trunk@568 cba15fb0-1239-40c8-b417-11db7ca47a34
parent e1b8e902
......@@ -3,7 +3,7 @@
// SampleWriter.C:
//
////////////////////////////////////////////////////////////////////////////////
// $Id: SampleWriter.C,v 1.3 2008-01-13 23:04:46 fgygi Exp $
// $Id: SampleWriter.C,v 1.4 2008-01-26 01:34:11 fgygi Exp $
#include "SampleWriter.h"
......@@ -11,12 +11,9 @@
#include "fstream"
#include "qbox_xmlns.h"
#include "Timer.h"
#ifdef USE_CSTDIO_LFS
#include <cstdio>
#include <cstdlib>
#include <sstream>
#endif
#include <iomanip>
using namespace std;
////////////////////////////////////////////////////////////////////////////////
......@@ -25,7 +22,7 @@ SampleWriter::SampleWriter(const Context& ctxt) : ctxt_(ctxt) {}
////////////////////////////////////////////////////////////////////////////////
void SampleWriter::writeSample(const Sample& s, const string filename,
string description,
bool base64, bool atomsonly)
bool base64, bool atomsonly, bool serial)
{
Timer tm;
tm.start();
......@@ -33,90 +30,132 @@ void SampleWriter::writeSample(const Sample& s, const string filename,
string encoding = base64 ? "base64" : "text";
const char* filename_cstr = filename.c_str();
#if USE_CSTDIO_LFS
// This section for compilers with broken large file support
// As of 2003-09-30, this includes gcc-3.2, icc-7.0, pgCC-5.0
// IBM xlC has correct large file support
// Remove this ifdef when other compilers catch up...
FILE* outfile;
if ( ctxt_.onpe0() )
{
outfile = fopen(filename_cstr,"w");
string header("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<fpmd:sample xmlns:fpmd=\"");
header += qbox_xmlns();
header += string("\"\n");
header += string(
" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n");
header += string(" xsi:schemaLocation=\"");
header += qbox_xmlns();
header += string(" sample.xsd\">\n");
off_t len = header.size();
fwrite(header.c_str(),sizeof(char),len,outfile);
string desc = string("<description> ") +
description +
string(" </description>\n");
ostringstream ss("");
ss << desc;
ss << s.atoms;
string str = ss.str();
const char* buf = str.c_str();
len = str.length();
fwrite(buf,sizeof(char),len,outfile);
}
long long file_size;
if ( !atomsonly )
if ( serial )
{
s.wf.write(outfile,encoding,"wavefunction");
if ( s.wfv != 0 )
s.wfv->write(outfile,encoding,"wavefunction_velocity");
}
ofstream os;
if ( ctxt_.onpe0() )
{
os.open(filename_cstr);
cout << " <!-- SaveCmd: saving to file " << filename
<< ", encoding=" << encoding << " -->" << endl;
if ( ctxt_.onpe0() )
{
char *trailer = "</fpmd:sample>\n";
fwrite(trailer,sizeof(char),strlen(trailer),outfile);
fclose(outfile);
os <<"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
<<"<fpmd:sample xmlns:fpmd=\""
<< qbox_xmlns()
<< "\"\n"
<<" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n"
<<" xsi:schemaLocation=\""
<< qbox_xmlns() << " sample.xsd\">"
<< endl;
os << "<description> " << description
<< " </description>" << endl;
os << s.atoms;
}
if ( !atomsonly )
{
s.wf.print(os,encoding,"wavefunction");
if ( s.wfv != 0 )
s.wfv->print(os,encoding,"wavefunction_velocity");
}
if ( ctxt_.onpe0() )
os << "</fpmd:sample>" << endl;
os.close();
}
#else
ofstream os;
if ( ctxt_.onpe0() )
else
{
os.open(filename_cstr);
cout << " <!-- SaveCmd: saving to file " << filename
<< ", encoding=" << encoding << " -->" << endl;
os
<<"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
<<"<fpmd:sample xmlns:fpmd=\""
<< qbox_xmlns()
<< "\"\n"
<<" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n"
<<" xsi:schemaLocation=\""
<< qbox_xmlns() << " sample.xsd\">"
<< endl;
os << "<description> " << description
<< " </description>" << endl;
os << s.atoms;
}
MPI_File fh;
MPI_Info info;
MPI_Info_create(&info);
MPI_Offset fsize;
if ( !atomsonly )
{
s.wf.print(os,encoding,"wavefunction");
if ( s.wfv != 0 )
s.wfv->print(os,encoding,"wavefunction_velocity");
}
int err;
err = MPI_File_open(ctxt_.comm(),(char*) filename_cstr,
MPI_MODE_WRONLY|MPI_MODE_CREATE,info,&fh);
if ( err != 0 )
cout << s.ctxt_.mype() << ": error in MPI_File_open: " << err << endl;
if ( ctxt_.onpe0() )
os << "</fpmd:sample>" << endl;
MPI_File_set_size(fh,0);
MPI_Status status;
if ( ctxt_.onpe0() )
{
string header("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<fpmd:sample xmlns:fpmd=\"");
header += qbox_xmlns();
header += string("\"\n");
header += string(
" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n");
header += string(" xsi:schemaLocation=\"");
header += qbox_xmlns();
header += string(" sample.xsd\">\n");
string desc = string("<description> ") +
description +
string(" </description>\n");
header += desc;
ostringstream ss("");
ss << s.atoms;
header += ss.str();
err = MPI_File_write_shared(fh,(void*)header.c_str(),
header.size(),MPI_CHAR,&status);
if ( err != 0 )
cout << ctxt_.mype() << ": error in MPI_File_write_shared: header "
<< err << endl;
}
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
if ( !atomsonly )
{
s.wf.write(fh,encoding,"wavefunction");
if ( s.wfv != 0 )
s.wfv->write(fh,encoding,"wavefunction_velocity");
}
if ( ctxt_.onpe0() )
{
char *trailer = "</fpmd:sample>\n";
int len = strlen(trailer);
err = MPI_File_write_shared(fh,(void*)trailer,len,MPI_CHAR,&status);
if ( err != 0 )
cout << ctxt_.mype() << ": error in MPI_File_write_shared: trailer "
<< err << endl;
}
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
MPI_File_get_size(fh,&fsize);
file_size = fsize;
err = MPI_File_close(&fh);
if ( err != 0 )
cout << ctxt_.mype() << ": error in MPI_File_close: " << err << endl;
}
os.close();
#endif
tm.stop();
if ( ctxt_.onpe0() )
cout << " <!-- SampleWriter: write time: " << tm.real() << " s -->"
{
cout << "<!-- SampleWriter: write time: "
<< setprecision(3) << tm.real() << " s -->"
<< endl;
if ( !serial )
{
cout << "<!-- SampleWriter: aggregate write rate: "
<< setprecision(2) << file_size/(tm.real()*1024*1024)
<< " MB/s" << endl;
}
}
}
......@@ -3,7 +3,7 @@
// SampleWriter.h
//
////////////////////////////////////////////////////////////////////////////////
// $Id: SampleWriter.h,v 1.3 2007-10-19 16:24:05 fgygi Exp $
// $Id: SampleWriter.h,v 1.4 2008-01-26 01:34:11 fgygi Exp $
#ifndef SAMPLEWRITER_H
#define SAMPLEWRITER_H
......@@ -22,7 +22,7 @@ class SampleWriter
SampleWriter(const Context& ctxt);
void writeSample(const Sample& s, const std::string filename,
std::string description,
bool base64, bool atomsonly);
bool base64, bool atomsonly, bool serial);
};
class SampleWriterException
......
......@@ -3,7 +3,7 @@
// SaveCmd.C:
//
////////////////////////////////////////////////////////////////////////////////
// $Id: SaveCmd.C,v 1.12 2007-10-19 16:24:05 fgygi Exp $
// $Id: SaveCmd.C,v 1.13 2008-01-26 01:34:11 fgygi Exp $
#include "SaveCmd.h"
......@@ -16,20 +16,21 @@ using namespace std;
////////////////////////////////////////////////////////////////////////////////
int SaveCmd::action(int argc, char **argv)
{
string usage(" <!-- use: save [-text|-base64] [-atomsonly] [-serial] filename -->");
if ( !(argc>=2 && argc<=4 ) )
{
if ( ui->onpe0() )
cout << " <!-- use: save [-text|-base64] [-atomsonly] filename -->"
<< endl;
cout << usage << endl;
return 1;
}
// set default encoding
bool base64 = true;
bool atomsonly = false;
bool serial = false;
char* filename = 0;
// check for -text or -base64 or -atomsonly arguments
// check for -text or -base64 or -atomsonly or -serial arguments
for ( int i = 1; i < argc; i++ )
{
string arg(argv[i]);
......@@ -42,6 +43,10 @@ int SaveCmd::action(int argc, char **argv)
{
atomsonly = true;
}
else if ( arg=="-serial" )
{
serial = true;
}
else if ( arg[0] != '-' && i == argc-1 )
{
filename = argv[i];
......@@ -49,8 +54,7 @@ int SaveCmd::action(int argc, char **argv)
else
{
if ( ui->onpe0() )
cout << " <!-- use: save [-text|-base64] [-atomsonly] filename -->"
<< endl;
cout << usage << endl;
return 1;
}
}
......@@ -58,14 +62,13 @@ int SaveCmd::action(int argc, char **argv)
if ( filename == 0 )
{
if ( ui->onpe0() )
cout << " <!-- use: save [-text|-base64] [-atomsonly] filename -->"
<< endl;
cout << usage << endl;
return 1;
}
SampleWriter swriter(s->ctxt_);
string description = string(" Created ") + isodate() +
string(" by qbox-") + release() + string(" ");
swriter.writeSample(*s, filename, description, base64, atomsonly);
swriter.writeSample(*s, filename, description, base64, atomsonly, serial);
return 0;
}
......@@ -3,7 +3,7 @@
// SaveCmd.h:
//
////////////////////////////////////////////////////////////////////////////////
// $Id: SaveCmd.h,v 1.2 2007-10-19 16:24:05 fgygi Exp $
// $Id: SaveCmd.h,v 1.3 2008-01-26 01:34:11 fgygi Exp $
#ifndef SAVECMD_H
#define SAVECMD_H
......@@ -30,8 +30,10 @@ class SaveCmd : public Cmd
{
return
"\n save\n\n"
" syntax: save filename \n\n"
" The save command saves the sample to the file filename.\n\n";
" syntax: save [-serial] [-text] filename \n\n"
" The save command saves the sample to the file filename.\n\n"
" When using the -serial option, I/O is performed from the \n"
" head node only. \n\n";
}
int action(int argc, char **argv);
......
......@@ -3,7 +3,7 @@
// SlaterDet.h
//
////////////////////////////////////////////////////////////////////////////////
// $Id: SlaterDet.h,v 1.21 2007-10-19 17:37:06 fgygi Exp $
// $Id: SlaterDet.h,v 1.22 2008-01-26 01:34:11 fgygi Exp $
#ifndef SLATERDET_H
#define SLATERDET_H
......@@ -19,9 +19,7 @@ class FourierTransform;
#include "Timer.h"
#include <string>
#include <map>
#if USE_CSTDIO_LFS
#include <cstdio>
#endif
#include "mpi.h"
typedef std::map<std::string,Timer> TimerMap;
......@@ -93,10 +91,8 @@ class SlaterDet
SlaterDet& operator=(SlaterDet& rhs);
void print(std::ostream& os, std::string encoding, double weight, int ispin,
int nspin);
#if USE_CSTDIO_LFS
void write(FILE* outfile, std::string encoding, double weight, int ispin,
void write(MPI_File& fh, std::string encoding, double weight, int ispin,
int nspin);
#endif
void info(std::ostream& os);
};
std::ostream& operator << ( std::ostream& os, SlaterDet& sd );
......
......@@ -3,17 +3,14 @@
// Wavefunction.C
//
////////////////////////////////////////////////////////////////////////////////
// $Id: Wavefunction.C,v 1.27 2007-12-15 00:57:03 fgygi Exp $
// $Id: Wavefunction.C,v 1.28 2008-01-26 01:34:11 fgygi Exp $
#include "Wavefunction.h"
#include "SlaterDet.h"
#include "jacobi.h"
#include <vector>
#include <iomanip>
#if USE_CSTDIO_LFS
#include <sstream>
#include <cstdio>
#endif
using namespace std;
////////////////////////////////////////////////////////////////////////////////
......@@ -663,9 +660,8 @@ void Wavefunction::print(ostream& os, string encoding, string tag) const
os << "</" << tag << ">" << endl;
}
#if USE_CSTDIO_LFS
////////////////////////////////////////////////////////////////////////////////
void Wavefunction::write(FILE* outfile, string encoding, string tag) const
void Wavefunction::write(MPI_File& fh, string encoding, string tag) const
{
if ( ctxt_.onpe0() )
{
......@@ -690,14 +686,23 @@ void Wavefunction::write(FILE* outfile, string encoding, string tag) const
<< " ny=\"" << sd_[0][0]->basis().np(1) << "\""
<< " nz=\"" << sd_[0][0]->basis().np(2) << "\"/>" << endl;
string str(os.str());
off_t len = str.length();
fwrite(str.c_str(),sizeof(char),len,outfile);
int len = str.size();
MPI_Status status;
int err = MPI_File_write_shared(fh,(void*)str.c_str(),len,MPI_CHAR,&status);
if ( err != 0 )
cout << " Wavefunction::write: error in MPI_File_write_shared" << endl;
}
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
for ( int ispin = 0; ispin < nspin_; ispin++ )
{
for ( int ikp = 0; ikp < kpoint_.size(); ikp++ )
sd_[ispin][ikp]->write(outfile,encoding,weight_[ikp],ispin,nspin_);
{
sd_[ispin][ikp]->write(fh,encoding,weight_[ikp],ispin,nspin_);
}
}
if ( ctxt_.onpe0() )
......@@ -705,11 +710,18 @@ void Wavefunction::write(FILE* outfile, string encoding, string tag) const
ostringstream os;
os << "</" << tag << ">" << endl;
string str(os.str());
off_t len = str.length();
fwrite(str.c_str(),sizeof(char),len,outfile);
int len = str.size();
MPI_Status status;
int err = MPI_File_write_shared(fh,(void*)str.c_str(),len,MPI_CHAR,&status);
if ( err != 0 )
cout << " Wavefunction::write: error in MPI_File_write_shared" << endl;
}
MPI_File_sync(fh);
ctxt_.barrier();
MPI_File_sync(fh);
}
#endif
////////////////////////////////////////////////////////////////////////////////
void Wavefunction::info(ostream& os, string tag) const
......
......@@ -3,7 +3,7 @@
// Wavefunction.h
//
////////////////////////////////////////////////////////////////////////////////
// $Id: Wavefunction.h,v 1.18 2007-11-29 08:28:10 fgygi Exp $
// $Id: Wavefunction.h,v 1.19 2008-01-26 01:34:11 fgygi Exp $
#ifndef WAVEFUNCTION_H
#define WAVEFUNCTION_H
......@@ -12,9 +12,7 @@
#include "UnitCell.h"
#include <vector>
#include <complex>
#if USE_CSTDIO_LFS
#include <cstdio>
#endif
#include "mpi.h"
class SlaterDet;
class Context;
......@@ -104,9 +102,7 @@ class Wavefunction
std::complex<double> dot(const Wavefunction& wf) const;
void print(std::ostream& os, std::string encoding, std::string tag) const;
#if USE_CSTDIO_LFS
void write(FILE* outfile, std::string encoding, std::string tag) const;
#endif
void write(MPI_File& fh, std::string encoding, std::string tag) const;
void info(std::ostream& os, std::string tag) const;
};
std::ostream& operator << ( std::ostream& os, const Wavefunction& wf );
......
......@@ -3,7 +3,7 @@
# x8664_gcc.mk
#
#-------------------------------------------------------------------------------
# $Id: pavane.mk,v 1.5 2007-10-31 05:13:04 fgygi Exp $
# $Id: pavane.mk,v 1.6 2008-01-26 01:34:11 fgygi Exp $
#
PLT=Linux_x8664
#-------------------------------------------------------------------------------
......@@ -16,7 +16,7 @@
CXX=/usr/bin/g++
LD=$(CXX)
PLTFLAGS += -DIA32 -DUSE_FFTW -DUSE_CSTDIO_LFS -D_LARGEFILE_SOURCE \
PLTFLAGS += -DIA32 -DUSE_FFTW -D_LARGEFILE_SOURCE \
-D_FILE_OFFSET_BITS=64 -DUSE_MPI -DSCALAPACK -DADD_ \
-DAPP_NO_THREADS -DXML_USE_NO_THREADS -DUSE_XERCES
......
......@@ -3,10 +3,10 @@
// release.C
//
////////////////////////////////////////////////////////////////////////////////
// $Id: release.C,v 1.42 2008-01-13 23:05:47 fgygi Exp $
// $Id: release.C,v 1.43 2008-01-26 01:34:11 fgygi Exp $
#include "release.h"
std::string release(void)
{
return std::string("1.34.4");
return std::string("1.35.0");
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment