Allow git on Mac OS to store file names in the index in precomposed unicode,
while the file system uses decomposed unicode.
The problem:
When a file called "LATIN CAPITAL LETTER A WITH DIAERESIS"
(in utf-8 encoded as 0xc3 0x84) is created, the Mac OS filesystem
converts "precomposed unicode" into "decomposed unicode".
This means that readdir() will return 0x41 0xcc 0x88.
Git under Mac OS reverts the unicode decomposition of filenames.
This is useful when pulling/pushing from repositories containing utf-8
encoded filenames using precomposed utf-8 like Linux or Windows (*).
It allows sharing git repositories stored on a VFAT file system
(e.g. a USB stick), and mounted network share using samba.
* (Not all Windows versions support UTF-8 yet:
Msysgit needs the unicode branch, cygwin supports UTF-8 since 1.7)
A new confguration variable is added: "core.precomposedunicode"
If set to false, git behaves exactly as older versions of git.
When a new git version is installed and there is a repository
where the configuration "core.precomposedunicode" is not present,
the new git is backward compatible.
The code in compat/precomposed_utf8.c implements basically 4 new functions:
precomposed_utf8_opendir(), precomposed_utf8_readdir(),
precomposed_utf8_closedir() precompose_argv()
In order to prevent that ever a file name in decomposed unicode is entering
the index, a "brute force" attempt is taken:
all arguments into git (argv[1]..argv[n]) are converted into
precomposed unicode.
This is done in git.c by calling precompose_argv().
This function is actually a #define, and it is only defined under Mac OS.
Nothing is converted on any other OS.
Auto sensing:
When creating a new git repository with "git init" or "git clone",
"core.precomposedunicode" will be set "false".
The user needs to activate this feature manually.
She typically sets core.precomposedunicode to "true" on HFS and VFAT,
or file systems mounted via SAMBA onto a Linux box.
Signed-off-by: Torsten Bögershausen <tboegi@xxxxxx>
---
Documentation/config.txt | 9 ++
Makefile | 3 +
builtin/init-db.c | 2 +
compat/precomposed_utf8.c | 208 ++++++++++++++++++++++++++++++++++++++++++
compat/precomposed_utf8.h | 30 ++++++
git-compat-util.h | 9 ++
git.c | 1 +
t/t3910-mac-os-precompose.sh | 117 +++++++++++++++++++++++
8 files changed, 379 insertions(+), 0 deletions(-)
create mode 100644 compat/precomposed_utf8.c
create mode 100644 compat/precomposed_utf8.h
create mode 100755 t/t3910-mac-os-precompose.sh
diff --git a/Documentation/config.txt b/Documentation/config.txt
index 2959390..29ba4b0 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -175,6 +175,15 @@ The default is false, except linkgit:git-clone[1] or linkgit:git-init[1]
will probe and set core.ignorecase true if appropriate when the repository
is created.
+core.precomposedunicode::
+ This option is only used by Mac OS implementation of git.
+ When core.precomposedunicode=true,
+ git reverts the unicode decomposition of filenames done by Mac OS.
+ This is useful when pulling/pushing from repositories containing utf-8
+ encoded filenames using precomposed unicode (like Linux).
+ When false, file names are handled fully transparent by git.
+ If in doubt, keep core.precomposedunicode=false.
+
core.trustctime::
If false, the ctime differences between the index and the
working tree are ignored; useful when the inode change time
diff --git a/Makefile b/Makefile
index b21d2f1..a912b45 100644
--- a/Makefile
+++ b/Makefile
@@ -519,6 +519,7 @@ LIB_H += compat/bswap.h
LIB_H += compat/cygwin.h
LIB_H += compat/mingw.h
LIB_H += compat/obstack.h
+LIB_H += compat/precomposed_utf8.h
LIB_H += compat/win32/pthread.h
LIB_H += compat/win32/syslog.h
LIB_H += compat/win32/poll.h
@@ -884,6 +885,8 @@ ifeq ($(uname_S),Darwin)
endif
NO_MEMMEM = YesPlease
USE_ST_TIMESPEC = YesPlease
+ COMPAT_OBJS += compat/precomposed_utf8.o
+ BASIC_CFLAGS += -DPRECOMPOSED_UNICODE
endif
ifeq ($(uname_S),SunOS)
NEEDS_SOCKET = YesPlease
diff --git a/builtin/init-db.c b/builtin/init-db.c
index 0dacb8b..06953df 100644
--- a/builtin/init-db.c
+++ b/builtin/init-db.c
@@ -290,6 +290,8 @@ static int create_default_files(const char *template_path)
strcpy(path + len, "CoNfIg");
if (!access(path, F_OK))
git_config_set("core.ignorecase", "true");
+
+ probe_utf8_pathname_composition(path, len);
}
return reinit;
diff --git a/compat/precomposed_utf8.c b/compat/precomposed_utf8.c
new file mode 100644
index 0000000..285fb45
--- /dev/null
+++ b/compat/precomposed_utf8.c
@@ -0,0 +1,208 @@
+#define __PRECOMPOSED_UNICODE_C__
+
+#include "../cache.h"
+#include "../utf8.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdint.h>
+#include "precomposed_utf8.h"
+
+static int mac_os_precomposed_unicode;
+const static char *repo_encoding = "UTF-8";
+const static char *path_encoding = "UTF-8-MAC";
+
+
+/* Code borrowed from utf8.c */
+#if defined(OLD_ICONV) || (defined(__sun__) && !defined(_XPG6))
+ typedef const char * iconv_ibp;
+#else
+ typedef char * iconv_ibp;
+#endif
+
+static char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv)
+{
+ size_t outsz, outalloc;
+ char *out, *outpos;
+ iconv_ibp cp;
+
+ outsz = insz;
+ outalloc = outsz + 1; /* for terminating NUL */
+ out = xmalloc(outalloc);
+ outpos = out;
+ cp = (iconv_ibp)in;
+
+ while (1) {
+ size_t cnt = iconv(conv, &cp, &insz, &outpos, &outsz);
+
+ if (cnt == -1) {
+ size_t sofar;
+ if (errno != E2BIG) {
+ free(out);
+ return NULL;
+ }
+ /* insz has remaining number of bytes.
+ * since we started outsz the same as insz,
+ * it is likely that insz is not enough for
+ * converting the rest.
+ */
+ sofar = outpos - out;
+ outalloc = sofar + insz * 2 + 32;
+ out = xrealloc(out, outalloc);
+ outpos = out + sofar;
+ outsz = outalloc - sofar - 1;
+ }
+ else {
+ *outpos = '\0';
+ break;
+ }
+ }
+ return out;
+}
+
+static size_t has_utf8(const char *s, size_t maxlen, size_t *strlen_c)
+{
+ const uint8_t *utf8p = (const uint8_t*) s;
+ size_t strlen_chars = 0;
+ size_t ret = 0;
+
+ if ((!utf8p) || (!*utf8p))
+ return 0;
+
+ while((*utf8p) && maxlen) {
+ if (*utf8p & 0x80)
+ ret++;
+ strlen_chars++;
+ utf8p++;
+ maxlen--;
+ }
+ if (strlen_c)
+ *strlen_c = strlen_chars;
+
+ return ret;
+}
+
+
+void probe_utf8_pathname_composition(char *path, int len)
+{
+ const static char *auml_nfc = "\xc3\xa4";
+ const static char *auml_nfd = "\x61\xcc\x88";
+ int output_fd;
+ path[len] = 0;
+ strcpy(path + len, auml_nfc);
+ output_fd = open(path, O_CREAT|O_EXCL|O_RDWR, 0600);
+ if (output_fd >=0) {
+ close(output_fd);
+ path[len] = 0;
+ strcpy(path + len, auml_nfd);
+ if (0 == access(path, R_OK))
+ git_config_set("core.precomposedunicode", "false");
+ path[len] = 0;
+ strcpy(path + len, auml_nfc);
+ unlink(path);
+ }
+}
+
+
+static int precomposed_unicode_config(const char *var, const char *value, void *cb)
+{
+ if (!strcasecmp(var, "core.precomposedunicode")) {
+ mac_os_precomposed_unicode = git_config_bool(var, value);
+ return 0;
+ }
+ return 1;
+}
+
+void precompose_argv(int argc, const char **argv)
+{
+ int i = 0;
+ const char *oldarg;
+ char *newarg;
+ iconv_t ic_precompose;
+
+ git_config(precomposed_unicode_config, NULL);
+ if (!mac_os_precomposed_unicode)
+ return;
+
+ ic_precompose = iconv_open(repo_encoding, path_encoding);
+ if (ic_precompose == (iconv_t) -1)
+ return;
+
+ while (i < argc) {
+ size_t namelen;
+ oldarg = argv[i];
+ if (has_utf8(oldarg, (size_t)-1, &namelen)) {
+ newarg = reencode_string_iconv(oldarg, namelen, ic_precompose);
+ if (newarg)
+ argv[i] = newarg;
+ }
+ i++;
+ }
+ iconv_close(ic_precompose);
+}
+
+
+PRECOMPOSED_UTF_DIR * precomposed_utf8_opendir(const char *dirname)
+{
+ PRECOMPOSED_UTF_DIR *precomposed_utf8_dir;
+ precomposed_utf8_dir = xmalloc(sizeof(PRECOMPOSED_UTF_DIR));
+
+ precomposed_utf8_dir->dirp = opendir(dirname);
+ if (!precomposed_utf8_dir->dirp) {
+ free(precomposed_utf8_dir);
+ return NULL;
+ }
+ precomposed_utf8_dir->ic_precompose = iconv_open(repo_encoding, path_encoding);
+ if (precomposed_utf8_dir->ic_precompose == (iconv_t) -1) {
+ closedir(precomposed_utf8_dir->dirp);
+ free(precomposed_utf8_dir);
+ return NULL;
+ }
+
+ return precomposed_utf8_dir;
+}
+
+struct dirent * precomposed_utf8_readdir(PRECOMPOSED_UTF_DIR *precomposed_utf8_dirp)
+{
+ struct dirent *res;
+ size_t namelen = 0;
+
+ res = readdir(precomposed_utf8_dirp->dirp);
+ if (res && mac_os_precomposed_unicode && has_utf8(res->d_name, (size_t)-1, &namelen)) {
+ int ret_errno = errno;
+ size_t outsz = sizeof(precomposed_utf8_dirp->dirent_nfc.d_name) - 1; /* one for \0 */
+ char *outpos = precomposed_utf8_dirp->dirent_nfc.d_name;
+ iconv_ibp cp;
+ size_t cnt;
+ size_t insz = namelen;
+ cp = (iconv_ibp)res->d_name;
+
+ /* Copy all data except the name */
+ memcpy(&precomposed_utf8_dirp->dirent_nfc, res,
+ sizeof(precomposed_utf8_dirp->dirent_nfc)-sizeof(precomposed_utf8_dirp->dirent_nfc.d_name));
+ errno = 0;
+
+ cnt = iconv(precomposed_utf8_dirp->ic_precompose, &cp, &insz, &outpos, &outsz);
+ if (cnt < sizeof(precomposed_utf8_dirp->dirent_nfc.d_name) -1) {
+ *outpos = 0;
+ errno = ret_errno;
+ return &precomposed_utf8_dirp->dirent_nfc;
+ }
+ errno = ret_errno;
+ }
+ return res;
+}
+
+
+int precomposed_utf8_closedir(PRECOMPOSED_UTF_DIR *precomposed_utf8_dirp)
+{
+ int ret_value;
+ int ret_errno;
+ ret_value = closedir(precomposed_utf8_dirp->dirp);
+ ret_errno = errno;
+ if (precomposed_utf8_dirp->ic_precompose != (iconv_t)-1)
+ iconv_close(precomposed_utf8_dirp->ic_precompose);
+ free(precomposed_utf8_dirp);
+ errno = ret_errno;
+ return ret_value;
+}
diff --git a/compat/precomposed_utf8.h b/compat/precomposed_utf8.h
new file mode 100644
index 0000000..79e65e7
--- /dev/null
+++ b/compat/precomposed_utf8.h
@@ -0,0 +1,30 @@
+#ifndef __PRECOMPOSED_UNICODE_H__
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <iconv.h>
+
+
+typedef struct {
+ iconv_t ic_precompose;
+ DIR *dirp;
+ struct dirent dirent_nfc;
+} PRECOMPOSED_UTF_DIR;
+
+char *precompose_str(const char *in, iconv_t ic_precompose);
+void precompose_argv(int argc, const char **argv);
+void probe_utf8_pathname_composition(char *, int);
+
+PRECOMPOSED_UTF_DIR *precomposed_utf8_opendir(const char *dirname);
+struct dirent *precomposed_utf8_readdir(PRECOMPOSED_UTF_DIR *dirp);
+int precomposed_utf8_closedir(PRECOMPOSED_UTF_DIR *dirp);
+
+#ifndef __PRECOMPOSED_UNICODE_C__
+#define opendir(n) precomposed_utf8_opendir(n)
+#define readdir(d) precomposed_utf8_readdir(d)
+#define closedir(d) precomposed_utf8_closedir(d)
+#define DIR PRECOMPOSED_UTF_DIR
+#endif /* __PRECOMPOSED_UNICODE_C__ */
+
+#define __PRECOMPOSED_UNICODE_H__
+#endif /* __PRECOMPOSED_UNICODE_H__ */
diff --git a/git-compat-util.h b/git-compat-util.h
index 230e198..8911743 100644
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -153,6 +153,15 @@
#include "compat/msvc.h"
#endif
+/* used on Mac OS X */
+#ifdef PRECOMPOSED_UNICODE
+#include "compat/precomposed_utf8.h"
+#else
+#define precompose_str(in,i_nfd2nfc)
+#define precompose_argv(c,v)
+#define probe_utf8_pathname_composition(a,b)
+#endif
+
#ifndef NO_LIBGEN_H
#include <libgen.h>
#else
diff --git a/git.c b/git.c
index 8e34903..265db96 100644
--- a/git.c
+++ b/git.c
@@ -298,6 +298,7 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv)
startup_info->have_repository) /* get_git_dir() may set up repo, avoid that */
trace_repo_setup(prefix);
}
+ precompose_argv(argc, argv);
commit_pager_choice();
if (!help && p->option & NEED_WORK_TREE)
diff --git a/t/t3910-mac-os-precompose.sh b/t/t3910-mac-os-precompose.sh
new file mode 100755
index 0000000..ba3d83c
--- /dev/null
+++ b/t/t3910-mac-os-precompose.sh
@@ -0,0 +1,117 @@
+#!/bin/sh
+#
+# Copyright (c) 2012 Torsten Bögershausen
+#
+
+test_description='utf-8 decomposed (nfd) converted to precomposed (nfc)'
+
+. ./test-lib.sh
+
+Adiarnfc=`printf '\303\204'`
+Odiarnfc=`printf '\303\226'`
+Adiarnfd=`printf 'A\314\210'`
+Odiarnfd=`printf 'O\314\210'`
+
+mkdir junk &&
+>junk/"$Adiarnfc" &&
+case "$(cd junk && echo *)" in
+ "$Adiarnfd")
+ test_nfd=1
+ ;;
+ *) ;;
+esac
+rm -rf junk
+
+if test "$test_nfd"
+then
+ test_expect_success "detect if nfd needed" '
+ precomposedunicode=`git config core.precomposedunicode` &&
+ test "$precomposedunicode" = false &&
+ git config core.precomposedunicode true
+ '
+ test_expect_success "setup" '
+ >x &&
+ git add x &&
+ git commit -m "1st commit" &&
+ git rm x &&
+ git commit -m "rm x"
+ '
+ test_expect_success "setup case mac" '
+ git checkout -b mac_os
+ '
+ # This will test nfd2nfc in readdir()
+ test_expect_success "add file Adiarnfc" '
+ echo f.Adiarnfc >f.$Adiarnfc &&
+ git add f.$Adiarnfc &&
+ git commit -m "add f.$Adiarnfc"
+ '
+ # This will test nfd2nfc in git stage()
+ test_expect_success "stage file d.Adiarnfd/f.Adiarnfd" '
+ mkdir d.$Adiarnfd &&
+ echo d.$Adiarnfd/f.$Adiarnfd >d.$Adiarnfd/f.$Adiarnfd &&
+ git stage d.$Adiarnfd/f.$Adiarnfd &&
+ git commit -m "add d.$Adiarnfd/f.$Adiarnfd"
+ '
+ test_expect_success "add link Adiarnfc" '
+ ln -s d.$Adiarnfd/f.$Adiarnfd l.$Adiarnfc &&
+ git add l.$Adiarnfc &&
+ git commit -m "add l.Adiarnfc"
+ '
+ # This will test git log
+ test_expect_success "git log f.Adiar" '
+ git log f.$Adiarnfc > f.Adiarnfc.log &&
+ git log f.$Adiarnfd > f.Adiarnfd.log &&
+ test -s f.Adiarnfc.log &&
+ test -s f.Adiarnfd.log &&
+ test_cmp f.Adiarnfc.log f.Adiarnfd.log &&
+ rm f.Adiarnfc.log f.Adiarnfd.log
+ '
+ # This will test git ls-files
+ test_expect_success "git lsfiles f.Adiar" '
+ git ls-files f.$Adiarnfc > f.Adiarnfc.log &&
+ git ls-files f.$Adiarnfd > f.Adiarnfd.log &&
+ test -s f.Adiarnfc.log &&
+ test -s f.Adiarnfd.log &&
+ test_cmp f.Adiarnfc.log f.Adiarnfd.log &&
+ rm f.Adiarnfc.log f.Adiarnfd.log
+ '
+ # This will test git mv
+ test_expect_success "git mv" '
+ git mv f.$Adiarnfd f.$Odiarnfc &&
+ git mv d.$Adiarnfd d.$Odiarnfc &&
+ git mv l.$Adiarnfd l.$Odiarnfc &&
+ git commit -m "mv Adiarnfd Odiarnfc"
+ '
+ # Files can be checked out as nfc
+ # And the link has been corrected from nfd to nfc
+ test_expect_success "git checkout nfc" '
+ rm f.$Odiarnfc &&
+ git checkout f.$Odiarnfc
+ '
+ # Make it possible to checkout files with their NFD names
+ test_expect_success "git checkout file nfd" '
+ rm -f f.* &&
+ git checkout f.$Odiarnfd
+ '
+ # Make it possible to checkout links with their NFD names
+ test_expect_success "git checkout link nfd" '
+ rm l.* &&
+ git checkout l.$Odiarnfd
+ '
+ test_expect_success "setup case mac2" '
+ git checkout master &&
+ git reset --hard &&
+ git checkout -b mac_os_2
+ '
+ # This will test nfd2nfc in git commit
+ test_expect_success "commit file d2.Adiarnfd/f.Adiarnfd" '
+ mkdir d2.$Adiarnfd &&
+ echo d2.$Adiarnfd/f.$Adiarnfd >d2.$Adiarnfd/f.$Adiarnfd &&
+ git add d2.$Adiarnfd/f.$Adiarnfd &&
+ git commit -m "add d2.$Adiarnfd/f.$Adiarnfd" -- d2.$Adiarnfd/f.$Adiarnfd
+ '
+else
+ say "Skipping nfc/nfd tests"
+fi
+
+test_done
--
1.7.8.rc0.43.gb49a8
--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html