Hi,
I was trying to enable quota on a volume thats usign SSL based auth as below :
[root@scratchpad-vm ssl]# gluster v i gv1
Volume Name: gv1
Type: Distribute
Volume ID: 7ad201b6-6e20-4065-b1c1-5b459f41604e
Status: Started
Number of Bricks: 1
Transport-type: tcp
Bricks:
Brick1: scratchpad-vm:/bricks/gv1-brick0
Options Reconfigured:
auth.ssl-allow: TheServer
features.quota: on
server.ssl: on
client.ssl: on
snap-max-hard-limit: 256
snap-max-soft-limit: 90
auto-delete: disable
[root@scratchpad-vm ssl]# gluster volume quota gv1 limit-usage / 2GB
quota command failed : Failed to find the directory /var/run/gluster/gv1/. Reason : Transport endpoint is not connected
[root@scratchpad-vm ssl]#
It looks like this doesn't work as quota tries to create a temp mount which fails hence the above error. quota acts as a local client for glusterd (IIUC) and since we have the gluster volume enabled for SSL it fails the mount hence limit-usage fails.
In general when i tried to do a local mount with ssl enabled, it failed for me too.. not sure if this is supposed to work as both client and server look at the same /etc/ssl/ keys/certs
[root@scratchpad-vm bricks]# mount -t glusterfs localhost:/gv1 /mnt
WARNING: getfattr not found, certain checks will be skipped..
Mount failed. Please check the log file for more details.
[2014-09-19 12:22:31.260095] I [MSGID: 100030] [glusterfsd.c:2021:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.7dev (args: /usr/sbin/glusterfs --volfile-server=localhost --volfile-id=/gv1 /mnt)
[2014-09-19 12:22:31.281189] I [dht-shared.c:334:dht_init_regex] 0-gv1-dht: using regex rsync-hash-regex = ^\.(.+)\.[^.]+$
[2014-09-19 12:22:31.281694] I [socket.c:3698:socket_init] 0-gv1-client-0: SSL support on the I/O path is ENABLED
[2014-09-19 12:22:31.281707] I [socket.c:3718:socket_init] 0-gv1-client-0: using private polling thread
[2014-09-19 12:22:31.283522] I [client.c:2280:notify] 0-gv1-client-0: parent translators are ready, attempting connect on transport
Final graph:
+------------------------------------------------------------------------------+
1: volume gv1-client-0
2: type protocol/client
3: option ping-timeout 42
4: option remote-host scratchpad-vm
5: option remote-subvolume /bricks/gv1-brick0
6: option transport-type socket
7: option username f14d0690-30cd-4be6-a4b3-eee5359a7180
8: option password 5486f372-be3f-415a-8b46-0b952c427bbf
9: option transport.socket.ssl-enabled on
10: option send-gids true
11: end-volume
12:
13: volume gv1-dht
14: type cluster/distribute
15: subvolumes gv1-client-0
16: end-volume
17:
18: volume gv1-write-behind
19: type performance/write-behind
20: subvolumes gv1-dht
21: end-volume
22:
23: volume gv1-read-ahead
24: type performance/read-ahead
25: subvolumes gv1-write-behind
26: end-volume
27:
28: volume gv1-io-cache
29: type performance/io-cache
30: subvolumes gv1-read-ahead
31: end-volume
32:
33: volume gv1-quick-read
34: type performance/quick-read
35: subvolumes gv1-io-cache
36: end-volume
37:
38: volume gv1-open-behind
39: type performance/open-behind
40: subvolumes gv1-quick-read
41: end-volume
42:
43: volume gv1-md-cache
44: type performance/md-cache
45: subvolumes gv1-open-behind
46: end-volume
47:
48: volume gv1
49: type debug/io-stats
50: option latency-measurement off
51: option count-fop-hits off
52: subvolumes gv1-md-cache
53: end-volume
54:
55: volume meta-autoload
56: type meta
57: subvolumes gv1
58: end-volume
59:
+------------------------------------------------------------------------------+
[2014-09-19 12:22:31.294407] I [rpc-clnt.c:1765:rpc_clnt_reconfig] 0-gv1-client-0: changing port to 49153 (from 0)
[2014-09-19 12:22:31.583691] E [socket.c:304:ssl_setup_connection] 0-gv1-client-0: SSL connect error
[2014-09-19 12:22:31.583728] E [socket.c:2282:socket_poller] 0-gv1-client-0: client setup failed
[2014-09-19 12:22:31.587481] I [fuse-bridge.c:5041:fuse_graph_setup] 0-fuse: switched to graph 0
[2014-09-19 12:22:31.587673] I [fuse-bridge.c:3970:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.22 kernel 7.22
[2014-09-19 12:22:31.587860] W [fuse-bridge.c:759:fuse_attr_cbk] 0-glusterfs-fuse: 2: LOOKUP() / => -1 (Transport endpoint is not connected)
[2014-09-19 12:22:31.591950] I [fuse-bridge.c:4882:fuse_thread_proc] 0-fuse: unmounting /mnt
[2014-09-19 12:22:31.592133] W [glusterfsd.c:1197:cleanup_and_exit] (--> 0-: received signum (15), shutting down
[2014-09-19 12:22:31.592146] I [fuse-bridge.c:5560:fini] 0-fuse: Unmounting '/mnt'.
[root@scratchpad-vm ssl]# pwd
/etc/ssl
[root@scratchpad-vm ssl]# ls -l
total 28
lrwxrwxrwx. 1 root root 16 Dec 12 2013 certs -> ../pki/tls/certs
-rw-r--r--. 1 root root 550 Sep 9 14:18 client2.csr
-rw-r--r--. 1 root root 623 Sep 9 14:19 client2.pem
-rw-r--r--. 1 root root 891 Sep 9 14:18 client.key
-rw-r--r--. 1 root root 1482 Sep 19 12:22 everyone.ca
lrwxrwxrwx. 1 root root 13 Jul 25 09:38 glusterfs.ca -> ./everyone.ca
lrwxrwxrwx. 1 root root 12 Jul 25 09:38 glusterfs.key -> ./server.key
lrwxrwxrwx. 1 root root 12 Jul 25 09:38 glusterfs.pem -> ./server.pem
-rw-r--r--. 1 root root 887 Sep 9 14:17 server.key
-rw-r--r--. 1 root root 741 Sep 9 14:17 server.pem
-rw-r--r--. 1 root root 3 Sep 9 14:19 server.srl
[root@scratchpad-vm ssl]# openssl x509 -text -in ./glusterfs.pem -noout
Certificate:
Data:
Version: 3 (0x2)
Serial Number: 12206605644881992844 (0xa966932920781c8c)
Signature Algorithm: sha1WithRSAEncryption
Issuer: CN=TheServer
Validity
Not Before: Sep 9 14:17:45 2014 GMT
Not After : Oct 9 14:17:45 2014 GMT
Subject: CN=TheServer
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
Public-Key: (1024 bit)
Modulus:
00:d7:07:7b:10:83:6a:70:af:7b:05:33:60:94:40:
80:07:ae:4d:a5:90:8e:fa:93:36:55:dc:95:03:5d:
47:a3:bb:fc:52:1c:a1:24:77:ac:56:89:29:1f:bd:
06:a8:80:06:ff:05:9e:ce:d6:c6:52:1b:0d:a8:fc:
73:20:36:31:2d:4c:f5:48:58:40:a4:ff:61:93:f0:
a5:90:06:8a:b9:f5:3c:dd:dc:d3:72:e8:45:dc:3f:
d9:0b:19:e6:a5:12:58:0c:e2:80:be:f1:e8:5e:a5:
90:66:33:aa:84:0e:20:28:21:06:f4:6e:ce:f3:99:
5b:0a:48:fd:2a:79:8e:82:cb
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Subject Key Identifier:
6D:2A:61:77:DE:21:A0:26:C5:F4:5F:7E:A7:B8:23:39:34:D5:76:11
X509v3 Authority Key Identifier:
keyid:6D:2A:61:77:DE:21:A0:26:C5:F4:5F:7E:A7:B8:23:39:34:D5:76:11
X509v3 Basic Constraints:
CA:TRUE
Signature Algorithm: sha1WithRSAEncryption
0b:86:fc:a6:04:33:00:3e:e7:0d:4e:60:dd:11:b0:b6:e1:43:
3a:59:a7:9e:9f:c0:99:88:6e:35:4b:82:74:d5:12:5a:80:e0:
9b:f1:db:da:44:b9:a2:ff:c5:bc:86:0f:96:af:4c:98:25:79:
aa:5f:a5:a5:ab:e2:74:87:e7:be:17:c5:1a:f7:e0:e6:39:ea:
d3:1f:6f:dc:cd:df:0f:b5:cd:ea:71:ee:f2:71:93:69:a7:8d:
52:5b:4b:9f:b1:9c:87:d3:5d:e5:ee:2c:e7:36:80:83:6b:59:
94:07:12:14:6c:ef:fb:c4:ca:7e:56:11:9a:52:9d:53:dc:c2:
d7:8d
I copied glusterfs.pem into everyone.ca so both client and server should use the same certs for mutual authI was trying to enable quota on a volume thats usign SSL based auth as below :
[root@scratchpad-vm ssl]# gluster v i gv1
Volume Name: gv1
Type: Distribute
Volume ID: 7ad201b6-6e20-4065-b1c1-5b459f41604e
Status: Started
Number of Bricks: 1
Transport-type: tcp
Bricks:
Brick1: scratchpad-vm:/bricks/gv1-brick0
Options Reconfigured:
auth.ssl-allow: TheServer
features.quota: on
server.ssl: on
client.ssl: on
snap-max-hard-limit: 256
snap-max-soft-limit: 90
auto-delete: disable
[root@scratchpad-vm ssl]# gluster volume quota gv1 limit-usage / 2GB
quota command failed : Failed to find the directory /var/run/gluster/gv1/. Reason : Transport endpoint is not connected
[root@scratchpad-vm ssl]#
It looks like this doesn't work as quota tries to create a temp mount which fails hence the above error. quota acts as a local client for glusterd (IIUC) and since we have the gluster volume enabled for SSL it fails the mount hence limit-usage fails.
In general when i tried to do a local mount with ssl enabled, it failed for me too.. not sure if this is supposed to work as both client and server look at the same /etc/ssl/ keys/certs
[root@scratchpad-vm bricks]# mount -t glusterfs localhost:/gv1 /mnt
WARNING: getfattr not found, certain checks will be skipped..
Mount failed. Please check the log file for more details.
[2014-09-19 12:22:31.260095] I [MSGID: 100030] [glusterfsd.c:2021:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.7dev (args: /usr/sbin/glusterfs --volfile-server=localhost --volfile-id=/gv1 /mnt)
[2014-09-19 12:22:31.281189] I [dht-shared.c:334:dht_init_regex] 0-gv1-dht: using regex rsync-hash-regex = ^\.(.+)\.[^.]+$
[2014-09-19 12:22:31.281694] I [socket.c:3698:socket_init] 0-gv1-client-0: SSL support on the I/O path is ENABLED
[2014-09-19 12:22:31.281707] I [socket.c:3718:socket_init] 0-gv1-client-0: using private polling thread
[2014-09-19 12:22:31.283522] I [client.c:2280:notify] 0-gv1-client-0: parent translators are ready, attempting connect on transport
Final graph:
+------------------------------------------------------------------------------+
1: volume gv1-client-0
2: type protocol/client
3: option ping-timeout 42
4: option remote-host scratchpad-vm
5: option remote-subvolume /bricks/gv1-brick0
6: option transport-type socket
7: option username f14d0690-30cd-4be6-a4b3-eee5359a7180
8: option password 5486f372-be3f-415a-8b46-0b952c427bbf
9: option transport.socket.ssl-enabled on
10: option send-gids true
11: end-volume
12:
13: volume gv1-dht
14: type cluster/distribute
15: subvolumes gv1-client-0
16: end-volume
17:
18: volume gv1-write-behind
19: type performance/write-behind
20: subvolumes gv1-dht
21: end-volume
22:
23: volume gv1-read-ahead
24: type performance/read-ahead
25: subvolumes gv1-write-behind
26: end-volume
27:
28: volume gv1-io-cache
29: type performance/io-cache
30: subvolumes gv1-read-ahead
31: end-volume
32:
33: volume gv1-quick-read
34: type performance/quick-read
35: subvolumes gv1-io-cache
36: end-volume
37:
38: volume gv1-open-behind
39: type performance/open-behind
40: subvolumes gv1-quick-read
41: end-volume
42:
43: volume gv1-md-cache
44: type performance/md-cache
45: subvolumes gv1-open-behind
46: end-volume
47:
48: volume gv1
49: type debug/io-stats
50: option latency-measurement off
51: option count-fop-hits off
52: subvolumes gv1-md-cache
53: end-volume
54:
55: volume meta-autoload
56: type meta
57: subvolumes gv1
58: end-volume
59:
+------------------------------------------------------------------------------+
[2014-09-19 12:22:31.294407] I [rpc-clnt.c:1765:rpc_clnt_reconfig] 0-gv1-client-0: changing port to 49153 (from 0)
[2014-09-19 12:22:31.583691] E [socket.c:304:ssl_setup_connection] 0-gv1-client-0: SSL connect error
[2014-09-19 12:22:31.583728] E [socket.c:2282:socket_poller] 0-gv1-client-0: client setup failed
[2014-09-19 12:22:31.587481] I [fuse-bridge.c:5041:fuse_graph_setup] 0-fuse: switched to graph 0
[2014-09-19 12:22:31.587673] I [fuse-bridge.c:3970:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.22 kernel 7.22
[2014-09-19 12:22:31.587860] W [fuse-bridge.c:759:fuse_attr_cbk] 0-glusterfs-fuse: 2: LOOKUP() / => -1 (Transport endpoint is not connected)
[2014-09-19 12:22:31.591950] I [fuse-bridge.c:4882:fuse_thread_proc] 0-fuse: unmounting /mnt
[2014-09-19 12:22:31.592133] W [glusterfsd.c:1197:cleanup_and_exit] (--> 0-: received signum (15), shutting down
[2014-09-19 12:22:31.592146] I [fuse-bridge.c:5560:fini] 0-fuse: Unmounting '/mnt'.
[root@scratchpad-vm ssl]# pwd
/etc/ssl
[root@scratchpad-vm ssl]# ls -l
total 28
lrwxrwxrwx. 1 root root 16 Dec 12 2013 certs -> ../pki/tls/certs
-rw-r--r--. 1 root root 550 Sep 9 14:18 client2.csr
-rw-r--r--. 1 root root 623 Sep 9 14:19 client2.pem
-rw-r--r--. 1 root root 891 Sep 9 14:18 client.key
-rw-r--r--. 1 root root 1482 Sep 19 12:22 everyone.ca
lrwxrwxrwx. 1 root root 13 Jul 25 09:38 glusterfs.ca -> ./everyone.ca
lrwxrwxrwx. 1 root root 12 Jul 25 09:38 glusterfs.key -> ./server.key
lrwxrwxrwx. 1 root root 12 Jul 25 09:38 glusterfs.pem -> ./server.pem
-rw-r--r--. 1 root root 887 Sep 9 14:17 server.key
-rw-r--r--. 1 root root 741 Sep 9 14:17 server.pem
-rw-r--r--. 1 root root 3 Sep 9 14:19 server.srl
[root@scratchpad-vm ssl]# openssl x509 -text -in ./glusterfs.pem -noout
Certificate:
Data:
Version: 3 (0x2)
Serial Number: 12206605644881992844 (0xa966932920781c8c)
Signature Algorithm: sha1WithRSAEncryption
Issuer: CN=TheServer
Validity
Not Before: Sep 9 14:17:45 2014 GMT
Not After : Oct 9 14:17:45 2014 GMT
Subject: CN=TheServer
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
Public-Key: (1024 bit)
Modulus:
00:d7:07:7b:10:83:6a:70:af:7b:05:33:60:94:40:
80:07:ae:4d:a5:90:8e:fa:93:36:55:dc:95:03:5d:
47:a3:bb:fc:52:1c:a1:24:77:ac:56:89:29:1f:bd:
06:a8:80:06:ff:05:9e:ce:d6:c6:52:1b:0d:a8:fc:
73:20:36:31:2d:4c:f5:48:58:40:a4:ff:61:93:f0:
a5:90:06:8a:b9:f5:3c:dd:dc:d3:72:e8:45:dc:3f:
d9:0b:19:e6:a5:12:58:0c:e2:80:be:f1:e8:5e:a5:
90:66:33:aa:84:0e:20:28:21:06:f4:6e:ce:f3:99:
5b:0a:48:fd:2a:79:8e:82:cb
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Subject Key Identifier:
6D:2A:61:77:DE:21:A0:26:C5:F4:5F:7E:A7:B8:23:39:34:D5:76:11
X509v3 Authority Key Identifier:
keyid:6D:2A:61:77:DE:21:A0:26:C5:F4:5F:7E:A7:B8:23:39:34:D5:76:11
X509v3 Basic Constraints:
CA:TRUE
Signature Algorithm: sha1WithRSAEncryption
0b:86:fc:a6:04:33:00:3e:e7:0d:4e:60:dd:11:b0:b6:e1:43:
3a:59:a7:9e:9f:c0:99:88:6e:35:4b:82:74:d5:12:5a:80:e0:
9b:f1:db:da:44:b9:a2:ff:c5:bc:86:0f:96:af:4c:98:25:79:
aa:5f:a5:a5:ab:e2:74:87:e7:be:17:c5:1a:f7:e0:e6:39:ea:
d3:1f:6f:dc:cd:df:0f:b5:cd:ea:71:ee:f2:71:93:69:a7:8d:
52:5b:4b:9f:b1:9c:87:d3:5d:e5:ee:2c:e7:36:80:83:6b:59:
94:07:12:14:6c:ef:fb:c4:ca:7e:56:11:9a:52:9d:53:dc:c2:
d7:8d
but the local mount still fails as above. The rationale being if local mount works, quota also should work!
in the trusted storage pool, it should be able to do a mount irrespective of ssl being set or not.
Thoughts ?
Let me know if anythign more is needed for debug
thanx,
deepak
_______________________________________________ Gluster-devel mailing list Gluster-devel@xxxxxxxxxxx http://supercolony.gluster.org/mailman/listinfo/gluster-devel
