Re: [ceph-calamari] Does anyone understand Calamari??

Steffen W Sørensen <stefws@xxxxxx> · Wed, 13 May 2015 11:23:40 +0200

On 13/05/2015, at 04.08, Gregory Meno <gmeno@xxxxxxxxxx> wrote:

Ideally I would like everything in /var/log/calmari

be sure to set calamari.conf like so:
[shadow_man@vpm107 ~]$ grep DEBUG /etc/calamari/calamari.conf 
log_level = DEBUG
db_log_level = DEBUG
log_level = DEBUG

then restart cthulhu and apache

visit http://essperf3/api/v2/cluster
and http://essperf3

and then share the logs here. Hopefully something obvious will be off in either calamari or cthulhu log
Since I got similar issue, I sneak my log data in here as well, no offence…

-rw-r--r-- 1 www-data www-data   1554 May 13 11:14 info.log
-rw-r--r-- 1 root     root      29311 May 13 11:14 httpd_error.log
-rw-r--r-- 1 root     root       4599 May 13 11:14 httpd_access.log
-rw-r--r-- 1 www-data www-data    739 May 13 11:14 calamari.log
-rw-r--r-- 1 root     root     238047 May 13 11:14 cthulhu.log

root@node1:/var/log/calamari# cat calamari.log
2015-05-13 04:05:40,787 - metric_access - django.request Not Found: /favicon.ico
2015-05-13 04:14:02,263 - DEBUG - django.request.profile [17.5249576569ms] /api/v2/cluster
2015-05-13 04:14:02,263 - DEBUG - django.request.profile RPC timing for 'list_clusters': 3.89504432678/3.89504432678/3.89504432678 avg/min/max ms
2015-05-13 04:14:02,263 - DEBUG - django.request.profile Total time in RPC: 3.89504432678ms
2015-05-13 04:14:06,172 - DEBUG - django.request.profile [15.8069133759ms] /api/v2/cluster
2015-05-13 04:14:06,173 - DEBUG - django.request.profile RPC timing for 'list_clusters': 2.44808197021/2.44808197021/2.44808197021 avg/min/max ms
2015-05-13 04:14:06,173 - DEBUG - django.request.profile Total time in RPC: 2.44808197021ms

root@node1:/var/log/calamari# tail cthulhu.log
2015-05-13 11:14:46,694 - DEBUG - cthulhu nivcsw: 102
2015-05-13 11:14:46,709 - DEBUG - cthulhu Eventer.on_tick
2015-05-13 11:14:46,710 - INFO - cthulhu Eventer._emit: 2015-05-13 09:14:46.710030+00:00/WARNING/Cluster 'ceph' is late reporting in
2015-05-13 11:14:46,710 - INFO - sqlalchemy.engine.base.Engine BEGIN (implicit)
2015-05-13 11:14:46,711 - INFO - sqlalchemy.engine.base.Engine INSERT INTO cthulhu_event ("when", severity, message, fsid, fqdn, service_type, service_id) VALUES (%(when)s, %(severity)s, %(message)s, %(fsid)s, %(fqdn)s, %(service_type)s, %(service_id)s) RETURNING cthulhu_event.id
2015-05-13 11:14:46,711 - INFO - sqlalchemy.engine.base.Engine {'severity': 3, 'when': datetime.datetime(2015, 5, 13, 9, 14, 46, 710030, tzinfo=tzutc()), 'fqdn': None, 'service_type': None, 'service_id': None, 'message': "Cluster 'ceph' is late reporting in", 'fsid': u'16fe2dcf-2629-422f-a649-871deba78bcd'}
2015-05-13 11:14:46,713 - DEBUG - sqlalchemy.engine.base.Engine Col ('id',)
2015-05-13 11:14:46,714 - DEBUG - sqlalchemy.engine.base.Engine Row (54,)
2015-05-13 11:14:46,714 - INFO - sqlalchemy.engine.base.Engine COMMIT
2015-05-13 11:14:56,710 - DEBUG - cthulhu Eventer.on_tick

root@node1:/var/log/calamari# tail httpd_error.log
[Wed May 13 04:14:05 2015] [warn]   File "/usr/lib/python2.7/dist-packages/git/__init__.py", line 20, in _init_externals
[Wed May 13 04:14:05 2015] [warn]     import gitdb
[Wed May 13 04:14:05 2015] [warn]   File "/usr/lib/python2.7/dist-packages/gitdb/__init__.py", line 25, in <module>
[Wed May 13 04:14:05 2015] [warn]     _init_externals()
[Wed May 13 04:14:05 2015] [warn]   File "/usr/lib/python2.7/dist-packages/gitdb/__init__.py", line 17, in _init_externals
[Wed May 13 04:14:05 2015] [warn]     __import__(module)
[Wed May 13 04:14:05 2015] [warn]   File "/usr/lib/python2.7/dist-packages/async/__init__.py", line 36, in <module>
[Wed May 13 04:14:05 2015] [warn]     _init_signals()
[Wed May 13 04:14:05 2015] [warn]   File "/usr/lib/python2.7/dist-packages/async/__init__.py", line 26, in _init_signals
[Wed May 13 04:14:05 2015] [warn]     signal.signal(signal.SIGINT, thread_interrupt_handler)

root@node1:/var/log/calamari# tail httpd_access.log
<ip> - - [13/May/2015:11:14:02 +0200] "GET /static/rest_framework/js/default.js HTTP/1.1" 304 209
<ip> - - [13/May/2015:11:14:04 +0200] "GET /api/v2/cluster HTTP/1.1" 200 2258
<ip> - - [13/May/2015:11:14:06 +0200] "GET /static/rest_framework/css/bootstrap.min.css HTTP/1.1" 304 211
<ip> - - [13/May/2015:11:14:06 +0200] "GET /static/rest_framework/css/bootstrap-tweaks.css HTTP/1.1" 304 209
<ip> - - [13/May/2015:11:14:06 +0200] "GET /static/rest_framework/css/prettify.css HTTP/1.1" 304 209
<ip> - - [13/May/2015:11:14:06 +0200] "GET /static/rest_framework/js/jquery-1.8.1-min.js HTTP/1.1" 304 211
<ip> - - [13/May/2015:11:14:06 +0200] "GET /static/rest_framework/js/bootstrap.min.js HTTP/1.1" 304 210
<ip> - - [13/May/2015:11:14:06 +0200] "GET /static/rest_framework/css/default.css HTTP/1.1" 304 209
<ip> - - [13/May/2015:11:14:06 +0200] "GET /static/rest_framework/js/prettify-min.js HTTP/1.1" 304 210
<ip> - - [13/May/2015:11:14:06 +0200] "GET /static/rest_framework/js/default.js HTTP/1.1" 304 209

wget /api/v2/cluster returns:

GET /api/v2/cluster
HTTP 200 OK
Vary: Accept
Content-Type: text/html; charset=utf-8
Allow: GET, HEAD, OPTIONS

[
    {
        "update_time": "2015-05-13T09:13:46.607104+00:00", 
        "id": "16fe2dcf-2629-422f-a649-871deba78bcd", 
        "name": "ceph"
    }
]

_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com