I have attached two source files: one for the reader (reader.cpp) and one for the writer (writer.cpp). I run those simultaneously on two different nodes.
The test relies on the existence of a small file (<1000 bytes) called "target" under /mnt/gv0. Alternatively you can modify the source files to point to another location.
I have four nodes: one primary server, one replicate secondary server, one for the reader (client node) and one for the writer (client node).
The gluster file system is physically located on a single disk local to the primary server.
Let me know if you need more details.
Thanks,
Louis
Le 06/08/14, Pranith Kumar Karampuri <pkarampu@xxxxxxxxxx> a écrit :
Louis,
Do you have a small test case or something we can try to re-create the issue?
Pranith
On 08/06/2014 08:33 PM, louisbenoit@xxxxxxxxxxxx wrote:
Hi,
Version: gluster 3.4.2
Use case: on one node a target file is updated while another node is reading it.
Issue: only about half of the read succeeds.
The strategy elected is to write to a temporary file then rename the temporary file to the target file.
During file renaming the gluster log from the reading node complains about stale file handle amongst other things:
[2014-08-06 13:54:24.653951] W [fuse-resolve.c:147:fuse_resolve_gfid_cbk] 0-fuse: f98aa46f-0e37-49bc-b578-8e70bec57a7f: failed to resolve (Invalid argument)
[2014-08-06 13:54:24.653983] E [fuse-bridge.c:1996:fuse_open_resume] 0-glusterfs-fuse: 428919: OPEN f98aa46f-0e37-49bc-b578-8e70bec57a7f resolution failed
[2014-08-06 13:54:24.754741] W [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.755263] W [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.755302] E [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null): failed to get the 'linkto' xattr No such file or directory
[2014-08-06 13:54:24.755347] W [fuse-bridge.c:2089:fuse_readv_cbk] 0-glusterfs-fuse: 428921: READ => -1 (No such file or directory)
[2014-08-06 13:54:24.755687] W [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.756193] W [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.756232] E [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null): failed to get the 'linkto' xattr No such file or directory
[2014-08-06 13:54:24.756267] W [fuse-bridge.c:2089:fuse_readv_cbk] 0-glusterfs-fuse: 428922: READ => -1 (No such file or directory)
[2014-08-06 13:54:24.756794] W [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.757292] W [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.757324] E [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null): failed to get the 'linkto' xattr No such file or directory
[2014-08-06 13:54:24.757356] W [fuse-bridge.c:2089:fuse_readv_cbk] 0-glusterfs-fuse: 428924: READ => -1 (No such file or directory)
[2014-08-06 13:54:24.958465] W [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.959020] W [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.959059] E [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null): failed to get the 'linkto' xattr No such file or directory
[2014-08-06 13:54:24.959100] W [fuse-bridge.c:2089:fuse_readv_cbk] 0-glusterfs-fuse: 428929: READ => -1 (No such file or directory)
[2014-08-06 13:54:24.959420] W [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.959885] W [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.959912] E [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null): failed to get the 'linkto' xattr No such file or directory
[2014-08-06 13:54:24.959949] W [fuse-bridge.c:2089:fuse_readv_cbk] 0-glusterfs-fuse: 428930: READ => -1 (No such file or directory)
[2014-08-06 13:54:24.960420] W [client-rpc-fops.c:2697:client3_3_readv_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.960940] W [client-rpc-fops.c:1179:client3_3_fgetxattr_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:24.960966] E [dht-helper.c:725:dht_migration_complete_check_task] 0-gv0-dht: (null): failed to get the 'linkto' xattr No such file or directory
[2014-08-06 13:54:24.960993] W [fuse-bridge.c:2089:fuse_readv_cbk] 0-glusterfs-fuse: 428932: READ => -1 (No such file or directory)
[2014-08-06 13:54:25.161921] W [client-rpc-fops.c:526:client3_3_stat_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:25.263287] W [client-rpc-fops.c:526:client3_3_stat_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:25.364792] W [client-rpc-fops.c:526:client3_3_stat_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:25.466796] W [client-rpc-fops.c:526:client3_3_stat_cbk] 0-gv0-client-0: remote operation failed: No such file or directory
[2014-08-06 13:54:25.568329] W [client-rpc-fops.c:2624:client3_3_lookup_cbk] 0-gv0-client-0: remote operation failed: Stale file handle. Path: /target_file (f98aa46f-0e37-49bc-b578-8e70bec57a7f)
[2014-08-06 13:54:25.568381] W [client-rpc-fops.c:2624:client3_3_lookup_cbk] 0-gv0-client-1: remote operation failed: Stale file handle. Path: /target_file (f98aa46f-0e37-49bc-b578-8e70bec57a7f)
I tried using RSYNC_FRIENDLY_NAME so that the file name hashing is the same for the temporary and target file (all my temporary file follows the rsync pattern ".<target filename>.tmp") but the issue remains.
Is there a way to make the "renaming" atomic? If not what is the best strategy to use when a node tries to read a file while another tries to update it?
Thanks,
Louis
_______________________________________________ Gluster-users mailing list Gluster-users@xxxxxxxxxxx http://supercolony.gluster.org/mailman/listinfo/gluster-users
#include <string.h>
#include <iostream>
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <time.h>
using namespace std;
const string gFileName ("/mnt/gv0/target");
int main()
{
int lErrno = 0;
unsigned long lSourceFileSize = 0;
char * lpSourceBuffer = NULL;
FILE* lpSourceFile = fopen(gFileName.c_str (), "rb");
if (lpSourceFile == NULL)
{
lErrno = errno;
cout << "Cannot open file " << gFileName << ": " << strerror (lErrno) << endl;
}
else
{
if (fseeko64 (lpSourceFile, 0L, SEEK_END) == 0)
{
lSourceFileSize = ftello64 (lpSourceFile);
rewind (lpSourceFile);
if (fseeko64 (lpSourceFile, 0, SEEK_SET) == 0)
{
lpSourceBuffer = new char [lSourceFileSize];
unsigned long lBytesRead = fread (lpSourceBuffer, 1, lSourceFileSize, lpSourceFile);
if (lBytesRead != lSourceFileSize)
{
lErrno = errno;
cout << "Cannot read all bytes from file " << gFileName << " " << lBytesRead << "/" << lSourceFileSize << ": feof " << feof (lpSourceFile) << ", ferror " << ferror (lpSourceFile) << ": " << strerror(lErrno) << endl;
}
}
else
{
lErrno = errno;
cout << "Cannot seek to beginning of file " << gFileName << ": " << strerror (lErrno) << endl;
}
}
else
{
lErrno = errno;
cout << "Cannot seek to end of file " << gFileName << ": " << strerror (lErrno) << endl;
}
fclose (lpSourceFile);
}
if (lpSourceBuffer == NULL)
{
return -1;
}
while (1)
{
std:string lFilePath = gFileName;
std::string lTmpFilePath = "/mnt/gv0/.target.tmp";
FILE* lpTmpFile = fopen (lTmpFilePath.c_str (), "wb");
if (lpTmpFile)
{
if (fwrite (lpSourceBuffer, 1, lSourceFileSize, lpTmpFile) == lSourceFileSize)
{
if (fclose (lpTmpFile) != 0)
{
int lError = errno;
cout << " Fail to flush temporary file " << lTmpFilePath << ": " << strerror (lError);
}
else if (rename (lTmpFilePath.c_str (), lFilePath.c_str ()) == -1)
{
int lError = errno;
cout << " Fail to write file " << lFilePath << ": " << strerror (lError);
}
}
else
{
fclose (lpTmpFile);
int lError = errno;
cout << " Fail to write temporary file " << lTmpFilePath << ": " << strerror (lError);
}
}
else
{
int lError = errno;
cout << " Fail to open temporary file " << lTmpFilePath << ": " << strerror (lError);
}
}
cout << "Hello World!" << endl;
return 0;
}
#include <string.h>
#include <iomanip>
#include <iostream>
#include <stdio.h>
#include <errno.h>
#include <time.h>
#include <sys/time.h>
using namespace std;
const string gFileName ("/mnt/gv0/target");
int main()
{
timespec lReadDelay;
timespec lRemainingDelay;
lReadDelay.tv_sec = 0;
lReadDelay.tv_nsec = 100000000;
while (1)
{
int lErrno = 0;
unsigned long lFileSize = 0;
FILE* lpFile = fopen(gFileName.c_str (), "rb");
if (lpFile == NULL)
{
lErrno = errno;
cout << "Cannot open file " << gFileName << ": " << strerror (lErrno) << endl;
}
else
{
if (fseeko64 (lpFile, 0L, SEEK_END) == 0)
{
lFileSize = ftello64 (lpFile);
rewind (lpFile);
unsigned long lBytesRead = 0;
if (fseeko64 (lpFile, 0, SEEK_SET) == 0)
{
char * lpBuffer = new char [lFileSize];
lBytesRead = fread (lpBuffer, 1, lFileSize, lpFile);
if (lBytesRead != lFileSize)
{
cout << "Cannot read all bytes from file " << gFileName << " " << lBytesRead << "/" << lFileSize << ": feof " << feof (lpFile) << ", ferror " << ferror (lpFile) << ": " << strerror(lErrno) << " read offset " << ftello64 (lpFile) << endl;
}
}
else
{
lErrno = errno;
cout << "Cannot seek to beginning of file " << gFileName << ": " << strerror (lErrno) << endl;
}
}
else
{
lErrno = errno;
cout << "Cannot seek to end of file " << gFileName << ": " << strerror (lErrno) << endl;
}
fclose (lpFile);
}
nanosleep (&lReadDelay, &lRemainingDelay);
}
return 0;
}
_______________________________________________ Gluster-users mailing list Gluster-users@xxxxxxxxxxx http://supercolony.gluster.org/mailman/listinfo/gluster-users
