Hello,
I was experimenting with libgfapi a bit and found something which I
cannot explain.
I wanted to simulate a behavior of long running system, so I created a
simple program which reads a file from Gluster volume, saves it under a
new name, deletes the original file and prints out the memory
statistics. And again, reads the file saved in the last round, saves a
new one, … But it seems that such a program just consumes memory
indefinitely.
The full source code of the program is attached, or available on
http://pastebin.com/9emyDH5N. It is not a production ready code, just an
experiment, but Gluster usage should be correct there.
My volume consists of 2 bricks, no replication, default options, Gluster
server and client have both the same version 3.8.4-1.fc24. The file I am
using for experiments is ~70 bytes. Program started on 70MB RSS and the
last time I checked it was 250MB. Unfortunately it is not possible to
use valgrind properly, because libgfapi seems to leak just by
initializing and deinitializing (tested with different code).
Is there some call in the library to free the memory that I am not
using? Some settings of the volume?
Thanks for your suggestions, kind regards,
Pavel
// g++ --std=c++14 -ggdb3 -lgfapi glfs_new_files.cpp
#include <cstring>
#include <iostream>
#include <fstream>
#include <sstream>
#include <stdexcept>
#include <vector>
#include <glusterfs/api/glfs.h>
/// simple representation of the used memory
struct MemUsage
{
double vsz = 0.0;
double rss = 0.0;
};
/// Returns memory usage data.
MemUsage getMemUsage()
{
// dummy vars for leading entries in stat that we don't care about
std::string pid, comm, state, ppid, pgrp, session, tty_nr;
std::string tpgid, flags, minflt, cminflt, majflt, cmajflt;
std::string utime, stime, cutime, cstime, priority, nice;
std::string O, itrealvalue, starttime;
// the two fields we want
unsigned long vsize;
long rss;
std::ifstream statStream("/proc/self/stat", std::ios_base::in);
statStream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr
>> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt
>> utime >> stime >> cutime >> cstime >> priority >> nice
>> O >> itrealvalue >> starttime >> vsize >> rss; // don't care about the rest
statStream.close();
MemUsage result;
result.vsz = vsize / 1024.0;
long pageSizeKB = sysconf(_SC_PAGE_SIZE) / 1024; // in case x86-64 is configured to use 2MB pages
result.rss = rss * pageSizeKB;
return result;
}
/// Takes the file name of the original file and continuously replicates it with the "_n" where n is
/// increasing number.
/// Also prints memory statistics after each round.
void glusterReplicateFile(const std::string& initialFileName)
{
glfs_t* gluster = glfs_new("test_volume");
if (!gluster)
throw std::runtime_error("Unable to create virtual mount object");
int error = glfs_set_volfile_server(gluster, "tcp", "10.10.52.92", 24007);
if (error)
throw std::runtime_error("Unable to set volumefile server: " + std::string(strerror(errno)));
error = glfs_set_logging(gluster, "/tmp/gluster_test_logging", 127);
if (error)
throw std::runtime_error("Unable to set logging: " + std::string(strerror(errno)));
error = glfs_init(gluster);
if (error)
throw std::runtime_error("Error while initializing gluster: " + std::string(strerror(errno)));
bool initialFileRead = true;
uint64_t round = 0;
std::string lastFileName = initialFileName;
std::vector<unsigned char> fileData;
double lastRss = 0.0;
while (++round)
{
// read out the original file
glfs_fd_t* glusterFile = glfs_open(gluster, lastFileName.c_str(), O_RDONLY);
if (!glusterFile)
throw std::runtime_error("Error while opening the original file: " + std::string(strerror(errno)));
struct stat s;
error = glfs_fstat(glusterFile, &s);
if (error)
throw std::runtime_error("Error while stating the original file: " + std::string(strerror(errno)));
fileData.resize(s.st_size);
ssize_t bytesRead = glfs_pread(glusterFile, fileData.data(), fileData.size(), 0, 0);
if (bytesRead != static_cast<ssize_t>(fileData.size()))
throw std::runtime_error("Error while reading the original file: " + std::string(strerror(errno)));
uint64_t fileSum = 0;
for (unsigned char value: fileData)
fileSum += value;
error = glfs_close(glusterFile);
if (error)
throw std::runtime_error("Error while closing the original file: " + std::string(strerror(errno)));
// create the new file
std::ostringstream newFileName;
newFileName << initialFileName << "_" << round;
glusterFile = glfs_creat(gluster, newFileName.str().c_str(), O_WRONLY, 0644);
if (!glusterFile)
throw std::runtime_error("Error while opening the new file: " + std::string(strerror(errno)));
ssize_t bytesWritten = glfs_pwrite(glusterFile, fileData.data(), fileData.size(), 0, O_APPEND);
if (bytesWritten != static_cast<ssize_t>(fileData.size()))
throw std::runtime_error("Error while writing the new file: " + std::string(strerror(errno)));
error = glfs_close(glusterFile);
if (error)
throw std::runtime_error("Error while closing the new file: " + std::string(strerror(errno)));
// remove the original file
if (lastFileName != initialFileName)
{
error = glfs_unlink(gluster, lastFileName.c_str());
if (error)
throw std::runtime_error("Error while deleting the original file: " + std::string(strerror(errno)));
}
// print out memory usage and go for the next round
MemUsage memUsage = getMemUsage();
std::cout << "Round " << round << " OK with fileSum: " << fileSum << ", VSZ: " << memUsage.vsz
<< " kB, RSS: " << memUsage.rss << " kB, RSS difference: " << memUsage.rss - lastRss
<< " kB" << std::endl;
lastRss = memUsage.rss;
lastFileName = newFileName.str();
}
error = glfs_fini(gluster);
if (error)
throw std::runtime_error("Error while finalizing gluster.");
}
int main(int argc, const char** argv)
{
std::string fileName = "testDir/testFile.txt";
if (argc == 2)
fileName = argv[1];
try
{
glusterReplicateFile(fileName);
}
catch (const std::exception& ex)
{
std::cerr << ex.what() << std::endl;
}
return 0;
}
_______________________________________________
Gluster-users mailing list
Gluster-users@xxxxxxxxxxx
http://www.gluster.org/mailman/listinfo/gluster-users
