4556565 [rkeene@sledge /home/rkeene/tmp]$ cat openstack-nova-compute-is-hung.txt
2013-12-10 16:48:00.281 29212 INFO nova.openstack.common.rpc.common [req-fb48c72a-980a-46f7-9188-a9ff679e1539 None None] Connected to AMQP server on 10.0.2.0:5672
*** terminated and restarted "nova-compute" ***
2013-12-10 16:55:04.206 29291 DEBUG nova.servicegroup.api [-] ServiceGroup driver defined as an instance of db __new__ /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:62
2013-12-10 16:55:04.293 29291 INFO nova.openstack.common.periodic_task [-] Skipping periodic task _periodic_update_dns because its interval is negative
2013-12-10 16:55:04.328 29291 DEBUG stevedore.extension [-] found extension EntryPoint.parse('file = nova.image.download.file') _load_plugins /usr/lib/python2.7/dist-packages/stevedore/extension.py:84
2013-12-10 16:55:04.340 29291 DEBUG stevedore.extension [-] found extension EntryPoint.parse('file = nova.image.download.file') _load_plugins /usr/lib/python2.7/dist-packages/stevedore/extension.py:84
2013-12-10 16:55:04.342 29291 INFO nova.virt.driver [-] Loading compute driver 'libvirt.LibvirtDriver'
2013-12-10 16:55:04.404 29291 DEBUG nova.openstack.common.rpc.amqp [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] Making synchronous call on conductor ... multicall /usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/amqp.py:553
2013-12-10 16:55:04.405 29291 DEBUG nova.openstack.common.rpc.amqp [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] MSG_ID is 8d9d7656f99e444cbf6034697cd9990e multicall /usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/amqp.py:556
2013-12-10 16:55:04.405 29291 DEBUG nova.openstack.common.rpc.amqp [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] UNIQUE_ID is 5d03fadb2430432d9872cb1670e719c7. _add_unique_id /usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/amqp.py:341
2013-12-10 16:55:04.425 29291 DEBUG amqp [-] Start from server, version: 0.9, properties: {u'information': u'Licensed under the MPL.  See http://www.rabbitmq.com/', u'product': u'RabbitMQ', u'copyright': u'Copyright (C) 2007-2012 VMware, Inc.', u'capabilities': {u'exchange_exchange_bindings': True, u'consumer_cancel_notify': True, u'publisher_confirms': True, u'basic.nack': True}, u'platform': u'Erlang/OTP', u'version': u'2.8.4'}, mechanisms: [u'PLAIN', u'AMQPLAIN'], locales: [u'en_US'] _start /usr/lib/python2.7/dist-packages/amqp/connection.py:706
2013-12-10 16:55:04.427 29291 DEBUG amqp [-] Open OK! _open_ok /usr/lib/python2.7/dist-packages/amqp/connection.py:592
2013-12-10 16:55:04.427 29291 DEBUG amqp [-] using channel_id: 1 __init__ /usr/lib/python2.7/dist-packages/amqp/channel.py:70
2013-12-10 16:55:04.428 29291 DEBUG amqp [-] Channel open _open_ok /usr/lib/python2.7/dist-packages/amqp/channel.py:420
2013-12-10 16:55:04.428 29291 INFO nova.openstack.common.rpc.common [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] Connected to AMQP server on 10.0.2.0:5672
2013-12-10 16:55:04.433 29291 DEBUG nova.openstack.common.rpc.amqp [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] Pool creating new connection create /usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/amqp.py:75
2013-12-10 16:55:04.440 29291 DEBUG amqp [-] Start from server, version: 0.9, properties: {u'information': u'Licensed under the MPL.  See http://www.rabbitmq.com/', u'product': u'RabbitMQ', u'copyright': u'Copyright (C) 2007-2012 VMware, Inc.', u'capabilities': {u'exchange_exchange_bindings': True, u'consumer_cancel_notify': True, u'publisher_confirms': True, u'basic.nack': True}, u'platform': u'Erlang/OTP', u'version': u'2.8.4'}, mechanisms: [u'PLAIN', u'AMQPLAIN'], locales: [u'en_US'] _start /usr/lib/python2.7/dist-packages/amqp/connection.py:706
2013-12-10 16:55:04.442 29291 DEBUG amqp [-] Open OK! _open_ok /usr/lib/python2.7/dist-packages/amqp/connection.py:592
2013-12-10 16:55:04.442 29291 DEBUG amqp [-] using channel_id: 1 __init__ /usr/lib/python2.7/dist-packages/amqp/channel.py:70
2013-12-10 16:55:04.444 29291 DEBUG amqp [-] Channel open _open_ok /usr/lib/python2.7/dist-packages/amqp/channel.py:420
2013-12-10 16:55:04.444 29291 INFO nova.openstack.common.rpc.common [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] Connected to AMQP server on 10.0.2.0:5672



root@cloudseed-1:/var/log/nova# ps -eaf|grep nova-compute
nova     29291     1  0 16:55 ?        00:00:00 /usr/bin/python /usr/bin/nova-compute --config-file=/etc/nova/nova.conf --log-file=/var/log/nova/nova-compute.log --config-file=/etc/nova/nova-compute.conf
root     29452 27493  0 17:21 pts/1    00:00:00 grep nova-compute
root@cloudseed-1:/var/log/nova# strace -f -p 29291
Process 29291 attached - interrupt to quit
epoll_wait(8, ^C <unfinished ...>
Process 29291 detached
root@cloudseed-1:/var/log/nova# cd /proc/29291/fd
root@cloudseed-1:/proc/29291/fd# ls -l
total 0
lrwx------ 1 nova nova 64 Dec 10 17:01 0 -> /dev/null
lrwx------ 1 nova nova 64 Dec 10 17:01 1 -> /dev/null
lrwx------ 1 nova nova 64 Dec 10 17:01 2 -> /dev/null
lrwx------ 1 nova nova 64 Dec 10 17:01 3 -> socket:[9163150]
l-wx------ 1 nova nova 64 Dec 10 17:01 4 -> /var/log/nova/nova-compute.log
lr-x------ 1 nova nova 64 Dec 10 17:01 5 -> /dev/urandom
lrwx------ 1 nova nova 64 Dec 10 17:01 6 -> socket:[9163155]
lr-x------ 1 nova nova 64 Dec 10 17:01 7 -> /dev/urandom
lrwx------ 1 nova nova 64 Dec 10 17:01 8 -> anon_inode:[eventpoll]
lrwx------ 1 nova nova 64 Dec 10 17:01 9 -> socket:[9163157]
root@cloudseed-1:/proc/29291/fd# netstat -tapen | grep 29291
tcp        0      0 10.0.0.1:48701          10.0.2.0:5672           ESTABLISHED 107        9163157     29291/python    
tcp        0      0 10.0.0.1:48700          10.0.2.0:5672           ESTABLISHED 107        9163155     29291/python    
root@cloudseed-1:/proc/29291/fd# 




root@cloudseed-1:/proc/29291/fd# nova-manage service list
2013-12-10 17:22:10.660 29465 DEBUG nova.servicegroup.api [-] ServiceGroup driver defined as an instance of db __new__ /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:62
2013-12-10 17:22:10.713 29465 DEBUG stevedore.extension [-] found extension EntryPoint.parse('file = nova.image.download.file') _load_plugins /usr/lib/python2.7/dist-packages/stevedore/extension.py:84
2013-12-10 17:22:10.725 29465 DEBUG stevedore.extension [-] found extension EntryPoint.parse('file = nova.image.download.file') _load_plugins /usr/lib/python2.7/dist-packages/stevedore/extension.py:84
2013-12-10 17:22:10.726 29465 DEBUG nova.openstack.common.lockutils [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Got semaphore "dbapi_backend" lock /usr/lib/python2.7/dist-packages/nova/openstack/common/lockutils.py:166
2013-12-10 17:22:10.726 29465 DEBUG nova.openstack.common.lockutils [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Got semaphore / lock "__get_backend" inner /usr/lib/python2.7/dist-packages/nova/openstack/common/lockutils.py:245
Binary           Host                                 Zone             Status     State Updated_At
2013-12-10 17:22:10.956 29465 DEBUG nova.servicegroup.api [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Check if the given member [{'binary': u'nova-scheduler', 'availability_zone': 'internal', 'deleted': 0L, 'created_at': datetime.datetime(2013, 11, 26, 16, 3, 16), 'updated_at': datetime.datetime(2013, 12, 10, 17, 22, 3), 'report_count': 121382L, 'topic': u'scheduler', 'host': u'cloudseed-dashboard', 'disabled': False, 'deleted_at': None, 'disabled_reason': None, 'id': 1L}] is part of the ServiceGroup, is up service_is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:94
2013-12-10 17:22:10.957 29465 DEBUG nova.servicegroup.drivers.db [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] DB_Driver.is_up last_heartbeat = 2013-12-10 17:22:03 elapsed = 7.957 is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/drivers/db.py:71
nova-scheduler   cloudseed-dashboard                  internal         enabled    :-)   2013-12-10 17:22:03
2013-12-10 17:22:10.957 29465 DEBUG nova.servicegroup.api [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Check if the given member [{'binary': u'nova-console', 'availability_zone': 'internal', 'deleted': 0L, 'created_at': datetime.datetime(2013, 11, 26, 16, 3, 16), 'updated_at': datetime.datetime(2013, 12, 10, 17, 22, 3), 'report_count': 121370L, 'topic': u'console', 'host': u'cloudseed-dashboard', 'disabled': False, 'deleted_at': None, 'disabled_reason': None, 'id': 2L}] is part of the ServiceGroup, is up service_is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:94
2013-12-10 17:22:10.958 29465 DEBUG nova.servicegroup.drivers.db [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] DB_Driver.is_up last_heartbeat = 2013-12-10 17:22:03 elapsed = 7.958185 is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/drivers/db.py:71
nova-console     cloudseed-dashboard                  internal         enabled    :-)   2013-12-10 17:22:03
2013-12-10 17:22:10.958 29465 DEBUG nova.servicegroup.api [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Check if the given member [{'binary': u'nova-conductor', 'availability_zone': 'internal', 'deleted': 0L, 'created_at': datetime.datetime(2013, 11, 26, 18, 0, 49), 'updated_at': datetime.datetime(2013, 12, 10, 17, 22, 4), 'report_count': 120691L, 'topic': u'conductor', 'host': u'cloudseed-dashboard', 'disabled': False, 'deleted_at': None, 'disabled_reason': None, 'id': 3L}] is part of the ServiceGroup, is up service_is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:94
2013-12-10 17:22:10.959 29465 DEBUG nova.servicegroup.drivers.db [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] DB_Driver.is_up last_heartbeat = 2013-12-10 17:22:04 elapsed = 6.959261 is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/drivers/db.py:71
nova-conductor   cloudseed-dashboard                  internal         enabled    :-)   2013-12-10 17:22:04
2013-12-10 17:22:10.959 29465 DEBUG nova.servicegroup.api [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Check if the given member [{'binary': u'nova-compute', 'availability_zone': 'nova', 'deleted': 0L, 'created_at': datetime.datetime(2013, 11, 26, 18, 1, 22), 'updated_at': datetime.datetime(2013, 12, 10, 16, 6, 15), 'report_count': 119833L, 'topic': u'compute', 'host': u'cloudseed-1', 'disabled': False, 'deleted_at': None, 'disabled_reason': None, 'id': 4L}] is part of the ServiceGroup, is up service_is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:94
2013-12-10 17:22:10.960 29465 DEBUG nova.servicegroup.drivers.db [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] DB_Driver.is_up last_heartbeat = 2013-12-10 16:06:15 elapsed = 4555.960334 is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/drivers/db.py:71
nova-compute     cloudseed-1                          nova             enabled    XXX   2013-12-10 16:06:15
root@cloudseed-1:/proc/29291/fd# 



----
root@cloudseed-dashboard:/var/log/nova# rabbitmqctl list_consumers
Listing consumers ...
network.cloudseed-0025b501a001  <'rabbit@cloudseed-dashboard'.3.384.0>  2       true
compute_fanout_c276b64fa980438e83bb6a0b08757659 <'rabbit@cloudseed-dashboard'.3.293.0>  3       true
compute <'rabbit@cloudseed-dashboard'.3.392.0>  1       true
compute <'rabbit@cloudseed-dashboard'.3.356.0>  1       true
compute <'rabbit@cloudseed-dashboard'.3.324.0>  1       true
compute <'rabbit@cloudseed-dashboard'.3.293.0>  1       true
conductor.cloudseed-dashboard   <'rabbit@cloudseed-dashboard'.3.29820.103>      2       true
scheduler_fanout_d1bd43c6d7ee426cb2d36e9bf58dddaf       <'rabbit@cloudseed-dashboard'.3.538.0>  3       true
network_fanout_7084e29745c7428cb0999e3696967f4b <'rabbit@cloudseed-dashboard'.3.384.0>  3       true
compute_fanout_e49a53e237d44905820a5a9bc18eb518 <'rabbit@cloudseed-dashboard'.3.324.0>  3       true
network.cloudseed-0025b501a000  <'rabbit@cloudseed-dashboard'.3.284.0>  2       true
compute.cloudseed-0025b501a001  <'rabbit@cloudseed-dashboard'.3.392.0>  2       true
q-plugin        <'rabbit@cloudseed-dashboard'.3.11666.0>        2       true
q-l3-plugin     <'rabbit@cloudseed-dashboard'.3.11666.0>        1       true
compute.cloudseed-0025b501a000  <'rabbit@cloudseed-dashboard'.3.293.0>  2       true
ceilometer.agent.compute        <'rabbit@cloudseed-dashboard'.3.11804.0>        1       true
conductor_fanout_eac080cfb745445faa54b35c9b262739       <'rabbit@cloudseed-dashboard'.3.29820.103>      3       true
compute_fanout_122c101f3b00495c9f52ddf80c9494e9 <'rabbit@cloudseed-dashboard'.3.356.0>  3       true
network_fanout_70d1ef577e1341b49eccbe1863de9175 <'rabbit@cloudseed-dashboard'.3.284.0>  3       true
network_fanout_63868efcc3454d7b884b668cf015459d <'rabbit@cloudseed-dashboard'.3.317.0>  3       true
reply_dfb751ccb8054d28a215d80cade34f50  <'rabbit@cloudseed-dashboard'.3.29404.103>      1       true
scheduler       <'rabbit@cloudseed-dashboard'.3.538.0>  1       true
console.cloudseed-dashboard     <'rabbit@cloudseed-dashboard'.3.549.0>  2       true
compute.cloudseed-0025b501a005  <'rabbit@cloudseed-dashboard'.3.356.0>  2       true
network.cloudseed-0025b501a005  <'rabbit@cloudseed-dashboard'.3.352.0>  2       true
scheduler.cloudseed-dashboard   <'rabbit@cloudseed-dashboard'.3.538.0>  2       true
ceilometer.agent.compute_fanout_67b1a234f0d14f2fb954b82818b9c2f0        <'rabbit@cloudseed-dashboard'.3.11804.0>        3       true
console <'rabbit@cloudseed-dashboard'.3.549.0>  1       true
reply_1b5207e85ed344a9818a9fe67dfd101a  <'rabbit@cloudseed-dashboard'.3.11962.103>      1       true
network <'rabbit@cloudseed-dashboard'.3.384.0>  1       true
network <'rabbit@cloudseed-dashboard'.3.352.0>  1       true
network <'rabbit@cloudseed-dashboard'.3.317.0>  1       true
network <'rabbit@cloudseed-dashboard'.3.284.0>  1       true
compute_fanout_818c19dd593b4fc098e3b7a35e9a6a9f <'rabbit@cloudseed-dashboard'.3.392.0>  3       true
ceilometer.agent.compute.cloudseed-1    <'rabbit@cloudseed-dashboard'.3.11804.0>        2       true
compute.cloudseed-0025b501a003  <'rabbit@cloudseed-dashboard'.3.324.0>  2       true
console_fanout_d3969082d6884ea5aec3a1967932c7ef <'rabbit@cloudseed-dashboard'.3.549.0>  3       true
network_fanout_9ff8449c0e0547d1b84dff8f609b00b9 <'rabbit@cloudseed-dashboard'.3.352.0>  3       true
network.cloudseed-0025b501a003  <'rabbit@cloudseed-dashboard'.3.317.0>  2       true
conductor       <'rabbit@cloudseed-dashboard'.3.29820.103>      1       true
...done.
root@cloudseed-dashboard:/var/log/nova# rabbitmqctl list_consumers|grep compute.cloudseed-1
ceilometer.agent.compute.cloudseed-1    <'rabbit@cloudseed-dashboard'.3.11804.0>        2       true
root@cloudseed-dashboard:/var/log/nova# 

4556566 [rkeene@sledge /home/rkeene/tmp]$

Click here to go back to the directory listing.
Click here to download this file.
last modified: 2013-12-10 17:57:24