src/Makefile.am | 1 src/fccache.c | 152 +-------------------------- src/fcdir.c | 2 src/fcint.h | 26 ++++ src/fcstat.c | 308 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 340 insertions(+), 149 deletions(-) New commits: commit 0ac6c98294d666762960824d39329459b22b48b7 Author: Mikhail Gusarov <dottedmag@xxxxxxxxxxxxx> Date: Mon May 28 14:52:21 2012 +0900 Fix cache aging for fonts on FAT filesystem under Linux Windows does not update mtime of directory on FAT filesystem when file is added to it or removed from it. Fontconfig uses mtime of directory to check cache file aging and hence fails to detect newly added or recently removed files. This changeset detects FAT filesystem (currently implemented for Linux) and adds generating checksum of directory entries instead of using mtime which guarantees proper cache rebuild. For non-FAT filesystems this patch adds single syscall per directory which is negligeable overhead. This fixes bug https://bugs.freedesktop.org/show_bug.cgi?id=25535 Signed-off-by: Mikhail Gusarov <dottedmag@xxxxxxxxxxxxx> diff --git a/src/fccache.c b/src/fccache.c index fddce94..9e582b9 100644 --- a/src/fccache.c +++ b/src/fccache.c @@ -181,7 +181,7 @@ FcDirCacheProcess (FcConfig *config, const FcChar8 *dir, struct stat file_stat, dir_stat; FcBool ret = FcFalse; - if (FcStat (dir, &dir_stat) < 0) + if (FcStatChecksum (dir, &dir_stat) < 0) return FcFalse; FcDirCacheBasename (dir, cache_base); @@ -508,14 +508,14 @@ FcCacheTimeValid (FcCache *cache, struct stat *dir_stat) if (!dir_stat) { - if (FcStat (FcCacheDir (cache), &dir_static) < 0) + if (FcStatChecksum (FcCacheDir (cache), &dir_static) < 0) return FcFalse; dir_stat = &dir_static; } if (FcDebug () & FC_DBG_CACHE) - printf ("FcCacheTimeValid dir \"%s\" cache time %d dir time %d\n", - FcCacheDir (cache), cache->mtime, (int) dir_stat->st_mtime); - return cache->mtime == (int) dir_stat->st_mtime; + printf ("FcCacheTimeValid dir \"%s\" cache checksum %d dir checksum %d\n", + FcCacheDir (cache), cache->checksum, (int) dir_stat->st_mtime); + return cache->checksum == (int) dir_stat->st_mtime; } /* @@ -679,7 +679,7 @@ FcDirCacheValidateHelper (int fd, struct stat *fd_stat, struct stat *dir_stat, v ret = FcFalse; else if (fd_stat->st_size != c.size) ret = FcFalse; - else if (c.mtime != (int) dir_stat->st_mtime) + else if (c.checksum != (int) dir_stat->st_mtime) ret = FcFalse; return ret; } @@ -754,7 +754,7 @@ FcDirCacheBuild (FcFontSet *set, const FcChar8 *dir, struct stat *dir_stat, FcSt cache->magic = FC_CACHE_MAGIC_ALLOC; cache->version = FC_CACHE_CONTENT_VERSION; cache->size = serialize->size; - cache->mtime = (int) dir_stat->st_mtime; + cache->checksum = (int) dir_stat->st_mtime; /* * Serialize directory name diff --git a/src/fcdir.c b/src/fcdir.c index 4399afc..2b476e8 100644 --- a/src/fcdir.c +++ b/src/fcdir.c @@ -245,7 +245,7 @@ FcDirCacheScan (const FcChar8 *dir, FcConfig *config) if (FcDebug () & FC_DBG_FONTSET) printf ("cache scan dir %s\n", dir); - if (FcStat (dir, &dir_stat) < 0) + if (FcStatChecksum (dir, &dir_stat) < 0) goto bail; set = FcFontSetCreate(); diff --git a/src/fcint.h b/src/fcint.h index 3d0e7bb..3d06fc6 100644 --- a/src/fcint.h +++ b/src/fcint.h @@ -358,7 +358,7 @@ struct _FcCache { intptr_t dirs; /* offset to subdirs */ int dirs_count; /* number of subdir strings */ intptr_t set; /* offset to font set */ - int mtime; /* low bits of directory mtime */ + int checksum; /* checksum of directory state */ }; #undef FcCacheDir @@ -1029,6 +1029,9 @@ FcMatrixFree (FcMatrix *mat); FcPrivate int FcStat (const FcChar8 *file, struct stat *statb); +FcPrivate int +FcStatChecksum (const FcChar8 *file, struct stat *statb); + FcPrivate FcBool FcIsFsMmapSafe (int fd); diff --git a/src/fcstat.c b/src/fcstat.c index 013e275..c2d9fe9 100644 --- a/src/fcstat.c +++ b/src/fcstat.c @@ -130,6 +130,90 @@ FcStat (const FcChar8 *file, struct stat *statb) #endif +/* Adler-32 checksum implementation */ +struct Adler32 { + int a; + int b; +}; + +static void +Adler32Init (struct Adler32 *ctx) +{ + ctx->a = 1; + ctx->b = 0; +} + +static void +Adler32Update (struct Adler32 *ctx, const char *data, int data_len) +{ + while (data_len--) + { + ctx->a = (ctx->a + *data++) % 65521; + ctx->b = (ctx->b + ctx->a) % 65521; + } +} + +static int +Adler32Finish (struct Adler32 *ctx) +{ + return ctx->a + (ctx->b << 16); +} + +/* dirent.d_type can be relied upon on FAT filesystem */ +static FcBool +FcDirChecksumScandirFilter(const struct dirent *entry) +{ + return entry->d_type != DT_DIR; +} + +static int +FcDirChecksumScandirSorter(const struct dirent **lhs, const struct dirent **rhs) +{ + return strcmp((*lhs)->d_name, (*rhs)->d_name); +} + +static int +FcDirChecksum (const FcChar8 *dir, time_t *checksum) +{ + struct Adler32 ctx; + struct dirent **files; + int n; + + Adler32Init (&ctx); + + n = scandir ((const char *)dir, &files, + &FcDirChecksumScandirFilter, + &FcDirChecksumScandirSorter); + if (n == -1) + return -1; + + while (n--) + { + Adler32Update (&ctx, files[n]->d_name, strlen(files[n]->d_name) + 1); + Adler32Update (&ctx, (char *)&files[n]->d_type, sizeof(files[n]->d_type)); + free(files[n]); + } + free(files); + + *checksum = Adler32Finish (&ctx); + return 0; +} + +int +FcStatChecksum (const FcChar8 *file, struct stat *statb) +{ + if (FcStat (file, statb) == -1) + return -1; + + if (FcIsFsMtimeBroken (file)) + { + if (FcDirChecksum (file, &statb->st_mtime) == -1) + return -1; + } + + return 0; +} + static int FcFStatFs (int fd, FcStatFS *statb) { commit dc2da23e69e6b3f6e6d0436d4777ee2c1d8ff1be Author: Akira TAGOH <akira@xxxxxxxxx> Date: Mon May 28 13:59:48 2012 +0900 Move statfs/statvfs wrapper to fcstat.c and add a test for the mtime broken fs just rework to share the efforts between FcIsFsMmapSafe() and FcIsFsMtimeBroken(). diff --git a/src/fccache.c b/src/fccache.c index dfe5b83..fddce94 100644 --- a/src/fccache.c +++ b/src/fccache.c @@ -37,21 +37,6 @@ # include <unistd.h> # include <sys/mman.h> #endif -#ifdef HAVE_SYS_VFS_H -#include <sys/vfs.h> -#endif -#ifdef HAVE_SYS_STATFS_H -#include <sys/statfs.h> -#endif -#ifdef HAVE_SYS_PARAM_H -#include <sys/param.h> -#endif -#ifdef HAVE_SYS_MOUNT_H -#include <sys/mount.h> -#endif -#ifdef HAVE_MNTENT_H -#include <mntent.h> -#endif #ifndef O_BINARY #define O_BINARY 0 @@ -74,7 +59,6 @@ static void MD5Transform(FcChar32 buf[4], FcChar32 in[16]); static FcBool FcCacheIsMmapSafe (int fd) { - FcBool retval = FcTrue; static FcBool is_initialized = FcFalse; static FcBool is_env_available = FcFalse; static FcBool use_mmap = FcFalse; @@ -93,40 +77,8 @@ FcCacheIsMmapSafe (int fd) } if (is_env_available) return use_mmap; -#if defined(HAVE_FSTATVFS) && (defined(HAVE_STRUCT_STATVFS_F_BASETYPE) || defined(HAVE_STRUCT_STATVFS_F_FSTYPENAME)) - struct statvfs buf; - - if (fstatvfs (fd, &buf) == 0) - { - const char *p; -#if defined(HAVE_STRUCT_STATVFS_F_BASETYPE) - p = buf.f_basetype; -#elif defined(HAVE_STRUCT_STATVFS_F_FSTYPENAME) - p = buf.f_fstypename; -#endif - - if (strcmp (p, "nfs") == 0) - retval = FcFalse; - } -#elif defined(HAVE_FSTATFS) && (defined(HAVE_STRUCT_STATFS_F_FLAGS) || defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) || defined(__linux__)) - struct statfs buf; - - if (fstatfs (fd, &buf) == 0) - { -# if defined(HAVE_STRUCT_STATFS_F_FLAGS) && defined(MNT_LOCAL) - if (!(buf.f_flags & MNT_LOCAL)) -# elif defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) - if (strcmp (buf.f_fstypename, "nfs") == 0) -# elif defined(__linux__) - if (buf.f_type == 0x6969) /* nfs */ -# else -# error "BUG: No way to figure out with fstatfs()" -# endif - retval = FcFalse; - } -#endif - return retval; + return FcIsFsMmapSafe (fd); } static const char bin2hex[] = { '0', '1', '2', '3', diff --git a/src/fcint.h b/src/fcint.h index d9ce801..3d0e7bb 100644 --- a/src/fcint.h +++ b/src/fcint.h @@ -539,6 +539,13 @@ struct _FcRange { FcChar32 end; }; +typedef struct _FcStatFS FcStatFS; + +struct _FcStatFS { + FcBool is_remote_fs; + FcBool is_mtime_broken; +}; + /* fcblanks.c */ /* fccache.c */ @@ -1022,6 +1029,12 @@ FcMatrixFree (FcMatrix *mat); FcPrivate int FcStat (const FcChar8 *file, struct stat *statb); +FcPrivate FcBool +FcIsFsMmapSafe (int fd); + +FcPrivate FcBool +FcIsFsMtimeBroken (const FcChar8 *dir); + /* fcstr.c */ FcPrivate void FcStrSetSort (FcStrSet * set); diff --git a/src/fcstat.c b/src/fcstat.c index 195878a..013e275 100644 --- a/src/fcstat.c +++ b/src/fcstat.c @@ -26,6 +26,21 @@ #include "fcint.h" #include "fcarch.h" #include <dirent.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#ifdef HAVE_SYS_VFS_H +#include <sys/vfs.h> +#endif +#ifdef HAVE_SYS_STATFS_H +#include <sys/statfs.h> +#endif +#ifdef HAVE_SYS_PARAM_H +#include <sys/param.h> +#endif +#ifdef HAVE_SYS_MOUNT_H +#include <sys/mount.h> +#endif #ifdef _WIN32 @@ -114,3 +129,96 @@ FcStat (const FcChar8 *file, struct stat *statb) } #endif + +static int +FcFStatFs (int fd, FcStatFS *statb) +{ + const char *p = NULL; + int ret; + FcBool flag = FcFalse; + + memset (statb, 0, sizeof (FcStatFS)); + +#if defined(HAVE_FSTATVFS) && (defined(HAVE_STRUCT_STATVFS_F_BASETYPE) || defined(HAVE_STRUCT_STATVFS_F_FSTYPENAME)) + struct statvfs buf; + + if ((ret = fstatvfs (fd, &buf)) == 0) + { +# if defined(HAVE_STRUCT_STATVFS_F_BASETYPE) + p = buf.f_basetype; +# elif defined(HAVE_STRUCT_STATVFS_F_FSTYPENAME) + p = buf.f_fstypename; +# endif + } +#elif defined(HAVE_FSTATFS) && (defined(HAVE_STRUCT_STATFS_F_FLAGS) || defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) || defined(__linux__)) + struct statfs buf; + + if ((ret = fstatfs (fd, &buf)) == 0) + { +# if defined(HAVE_STRUCT_STATFS_F_FLAGS) && defined(MNT_LOCAL) + statb->is_remote_fs = !(buf.f_flags & MNT_LOCAL); + flag = FcTrue; +# endif +# if defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) + p = buf.f_fstypename; +# elif defined(__linux__) + switch (buf.f_type) + { + case 0x6969: /* nfs */ + statb->is_remote_fs = FcTrue; + break; + case 0x4d44: /* fat */ + statb->is_mtime_broken = FcTrue; + break; + default: + break; + } + + return ret; +# else +# error "BUG: No way to figure out with fstatfs()" +# endif + } +#endif + if (p) + { + if (!flag && strcmp (p, "nfs") == 0) + statb->is_remote_fs = FcTrue; + if (strcmp (p, "msdosfs") == 0 || + strcmp (p, "pcfs") == 0) + statb->is_mtime_broken = FcTrue; + } + + return ret; +} + +FcBool +FcIsFsMmapSafe (int fd) +{ + FcStatFS statb; + + if (FcFStatFs (fd, &statb) < 0) + return FcTrue; + + return !statb.is_remote_fs; +} + +FcBool +FcIsFsMtimeBroken (const FcChar8 *dir) +{ + int fd = open ((const char *) dir, O_RDONLY); + + if (fd != -1) + { + FcStatFS statb; + int ret = FcFStatFs (fd, &statb); + + close (fd); + if (ret < 0) + return FcFalse; + + return statb.is_mtime_broken; + } + + return FcFalse; +} commit 6a83c1ad40594530994b826d928312e9eeb19c35 Author: Mikhail Gusarov <dottedmag@xxxxxxxxxxxxx> Date: Sun Apr 29 12:56:16 2012 +0200 Move FcStat to separate compilation unit FcStat() logic is quite complicated in presence of various semi-broken operating systems and filesystems, split it out in order to make it a bit easier. Signed-off-by: Mikhail Gusarov <dottedmag@xxxxxxxxxxxxx> diff --git a/src/Makefile.am b/src/Makefile.am index 0bd0e3d..65ec1e3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -119,6 +119,7 @@ libfontconfig_la_SOURCES = \ fcname.c \ fcpat.c \ fcserialize.c \ + fcstat.c \ fcstr.c \ fcxml.c \ ftglue.h \ diff --git a/src/fccache.c b/src/fccache.c index 882dd3e..dfe5b83 100644 --- a/src/fccache.c +++ b/src/fccache.c @@ -71,94 +71,6 @@ static void MD5Transform(FcChar32 buf[4], FcChar32 in[16]); #define CACHEBASE_LEN (1 + 32 + 1 + sizeof (FC_ARCHITECTURE) + sizeof (FC_CACHE_SUFFIX)) -#ifdef _WIN32 - -#include <windows.h> - -#ifdef __GNUC__ -typedef long long INT64; -#define EPOCH_OFFSET 11644473600ll -#else -#define EPOCH_OFFSET 11644473600i64 -typedef __int64 INT64; -#endif - -/* Workaround for problems in the stat() in the Microsoft C library: - * - * 1) stat() uses FindFirstFile() to get the file - * attributes. Unfortunately this API doesn't return correct values - * for modification time of a directory until some time after a file - * or subdirectory has been added to the directory. (This causes - * run-test.sh to fail, for instance.) GetFileAttributesEx() is - * better, it returns the updated timestamp right away. - * - * 2) stat() does some strange things related to backward - * compatibility with the local time timestamps on FAT volumes and - * daylight saving time. This causes problems after the switches - * to/from daylight saving time. See - * http://bugzilla.gnome.org/show_bug.cgi?id=154968 , especially - * comment #30, and http://www.codeproject.com/datetime/dstbugs.asp . - * We don't need any of that, FAT and Win9x are as good as dead. So - * just use the UTC timestamps from NTFS, converted to the Unix epoch. - */ - -int -FcStat (const FcChar8 *file, struct stat *statb) -{ - WIN32_FILE_ATTRIBUTE_DATA wfad; - char full_path_name[MAX_PATH]; - char *basename; - DWORD rc; - - if (!GetFileAttributesEx (file, GetFileExInfoStandard, &wfad)) - return -1; - - statb->st_dev = 0; - - /* Calculate a pseudo inode number as a hash of the full path name. - * Call GetLongPathName() to get the spelling of the path name as it - * is on disk. - */ - rc = GetFullPathName (file, sizeof (full_path_name), full_path_name, &basename); - if (rc == 0 || rc > sizeof (full_path_name)) - return -1; - - rc = GetLongPathName (full_path_name, full_path_name, sizeof (full_path_name)); - statb->st_ino = FcStringHash (full_path_name); - - statb->st_mode = _S_IREAD | _S_IWRITE; - statb->st_mode |= (statb->st_mode >> 3) | (statb->st_mode >> 6); - - if (wfad.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - statb->st_mode |= _S_IFDIR; - else - statb->st_mode |= _S_IFREG; - - statb->st_nlink = 1; - statb->st_uid = statb->st_gid = 0; - statb->st_rdev = 0; - - if (wfad.nFileSizeHigh > 0) - return -1; - statb->st_size = wfad.nFileSizeLow; - - statb->st_atime = (*(INT64 *)&wfad.ftLastAccessTime)/10000000 - EPOCH_OFFSET; - statb->st_mtime = (*(INT64 *)&wfad.ftLastWriteTime)/10000000 - EPOCH_OFFSET; - statb->st_ctime = statb->st_mtime; - - return 0; -} - -#else - -int -FcStat (const FcChar8 *file, struct stat *statb) -{ - return stat ((char *) file, statb); -} - -#endif - static FcBool FcCacheIsMmapSafe (int fd) { diff --git a/src/fcint.h b/src/fcint.h index ad9db8a..d9ce801 100644 --- a/src/fcint.h +++ b/src/fcint.h @@ -567,9 +567,6 @@ FcCacheFini (void); FcPrivate void FcDirCacheReference (FcCache *cache, int nref); -FcPrivate int -FcStat (const FcChar8 *file, struct stat *statb); - /* fccfg.c */ FcPrivate FcChar8 * @@ -1020,6 +1017,11 @@ extern FcPrivate const FcMatrix FcIdentityMatrix; FcPrivate void FcMatrixFree (FcMatrix *mat); +/* fcstat.c */ + +FcPrivate int +FcStat (const FcChar8 *file, struct stat *statb); + /* fcstr.c */ FcPrivate void FcStrSetSort (FcStrSet * set); diff --git a/src/fcstat.c b/src/fcstat.c new file mode 100644 index 0000000..195878a --- /dev/null +++ b/src/fcstat.c @@ -0,0 +1,116 @@ +/* + * Copyright © 2000 Keith Packard + * Copyright © 2005 Patrick Lam + * + * Permission to use, copy, modify, distribute, and sell this software and its + * documentation for any purpose is hereby granted without fee, provided that + * the above copyright notice appear in all copies and that both that + * copyright notice and this permission notice appear in supporting + * documentation, and that the name of the author(s) not be used in + * advertising or publicity pertaining to distribution of the software without + * specific, written prior permission. The authors make no + * representations about the suitability of this software for any purpose. It + * is provided "as is" without express or implied warranty. + * + * THE AUTHOR(S) DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO + * EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY SPECIAL, INDIRECT OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, + * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER + * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "fcint.h" +#include "fcarch.h" +#include <dirent.h> + +#ifdef _WIN32 + +#include <windows.h> + +#ifdef __GNUC__ +typedef long long INT64; +#define EPOCH_OFFSET 11644473600ll +#else +#define EPOCH_OFFSET 11644473600i64 +typedef __int64 INT64; +#endif + +/* Workaround for problems in the stat() in the Microsoft C library: + * + * 1) stat() uses FindFirstFile() to get the file + * attributes. Unfortunately this API doesn't return correct values + * for modification time of a directory until some time after a file + * or subdirectory has been added to the directory. (This causes + * run-test.sh to fail, for instance.) GetFileAttributesEx() is + * better, it returns the updated timestamp right away. + * + * 2) stat() does some strange things related to backward + * compatibility with the local time timestamps on FAT volumes and + * daylight saving time. This causes problems after the switches + * to/from daylight saving time. See + * http://bugzilla.gnome.org/show_bug.cgi?id=154968 , especially + * comment #30, and http://www.codeproject.com/datetime/dstbugs.asp . + * We don't need any of that, FAT and Win9x are as good as dead. So + * just use the UTC timestamps from NTFS, converted to the Unix epoch. + */ + +int +FcStat (const FcChar8 *file, struct stat *statb) +{ + WIN32_FILE_ATTRIBUTE_DATA wfad; + char full_path_name[MAX_PATH]; + char *basename; + DWORD rc; + + if (!GetFileAttributesEx (file, GetFileExInfoStandard, &wfad)) + return -1; + + statb->st_dev = 0; + + /* Calculate a pseudo inode number as a hash of the full path name. + * Call GetLongPathName() to get the spelling of the path name as it + * is on disk. + */ + rc = GetFullPathName (file, sizeof (full_path_name), full_path_name, &basename); + if (rc == 0 || rc > sizeof (full_path_name)) + return -1; + + rc = GetLongPathName (full_path_name, full_path_name, sizeof (full_path_name)); + statb->st_ino = FcStringHash (full_path_name); + + statb->st_mode = _S_IREAD | _S_IWRITE; + statb->st_mode |= (statb->st_mode >> 3) | (statb->st_mode >> 6); + + if (wfad.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + statb->st_mode |= _S_IFDIR; + else + statb->st_mode |= _S_IFREG; + + statb->st_nlink = 1; + statb->st_uid = statb->st_gid = 0; + statb->st_rdev = 0; + + if (wfad.nFileSizeHigh > 0) + return -1; + statb->st_size = wfad.nFileSizeLow; + + statb->st_atime = (*(INT64 *)&wfad.ftLastAccessTime)/10000000 - EPOCH_OFFSET; + statb->st_mtime = (*(INT64 *)&wfad.ftLastWriteTime)/10000000 - EPOCH_OFFSET; + statb->st_ctime = statb->st_mtime; + + return 0; +} + +#else + +int +FcStat (const FcChar8 *file, struct stat *statb) +{ + return stat ((char *) file, statb); +} + +#endif
_______________________________________________ Fontconfig mailing list Fontconfig@xxxxxxxxxxxxxxxxxxxxx http://lists.freedesktop.org/mailman/listinfo/fontconfig