2013-12-10 16:48:00.281 29212 INFO nova.openstack.common.rpc.common [req-fb48c72a-980a-46f7-9188-a9ff679e1539 None None] Connected to AMQP server on 10.0.2.0:5672 *** terminated and restarted "nova-compute" *** 2013-12-10 16:55:04.206 29291 DEBUG nova.servicegroup.api [-] ServiceGroup driver defined as an instance of db __new__ /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:62 2013-12-10 16:55:04.293 29291 INFO nova.openstack.common.periodic_task [-] Skipping periodic task _periodic_update_dns because its interval is negative 2013-12-10 16:55:04.328 29291 DEBUG stevedore.extension [-] found extension EntryPoint.parse('file = nova.image.download.file') _load_plugins /usr/lib/python2.7/dist-packages/stevedore/extension.py:84 2013-12-10 16:55:04.340 29291 DEBUG stevedore.extension [-] found extension EntryPoint.parse('file = nova.image.download.file') _load_plugins /usr/lib/python2.7/dist-packages/stevedore/extension.py:84 2013-12-10 16:55:04.342 29291 INFO nova.virt.driver [-] Loading compute driver 'libvirt.LibvirtDriver' 2013-12-10 16:55:04.404 29291 DEBUG nova.openstack.common.rpc.amqp [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] Making synchronous call on conductor ... multicall /usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/amqp.py:553 2013-12-10 16:55:04.405 29291 DEBUG nova.openstack.common.rpc.amqp [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] MSG_ID is 8d9d7656f99e444cbf6034697cd9990e multicall /usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/amqp.py:556 2013-12-10 16:55:04.405 29291 DEBUG nova.openstack.common.rpc.amqp [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] UNIQUE_ID is 5d03fadb2430432d9872cb1670e719c7. _add_unique_id /usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/amqp.py:341 2013-12-10 16:55:04.425 29291 DEBUG amqp [-] Start from server, version: 0.9, properties: {u'information': u'Licensed under the MPL. See http://www.rabbitmq.com/', u'product': u'RabbitMQ', u'copyright': u'Copyright (C) 2007-2012 VMware, Inc.', u'capabilities': {u'exchange_exchange_bindings': True, u'consumer_cancel_notify': True, u'publisher_confirms': True, u'basic.nack': True}, u'platform': u'Erlang/OTP', u'version': u'2.8.4'}, mechanisms: [u'PLAIN', u'AMQPLAIN'], locales: [u'en_US'] _start /usr/lib/python2.7/dist-packages/amqp/connection.py:706 2013-12-10 16:55:04.427 29291 DEBUG amqp [-] Open OK! _open_ok /usr/lib/python2.7/dist-packages/amqp/connection.py:592 2013-12-10 16:55:04.427 29291 DEBUG amqp [-] using channel_id: 1 __init__ /usr/lib/python2.7/dist-packages/amqp/channel.py:70 2013-12-10 16:55:04.428 29291 DEBUG amqp [-] Channel open _open_ok /usr/lib/python2.7/dist-packages/amqp/channel.py:420 2013-12-10 16:55:04.428 29291 INFO nova.openstack.common.rpc.common [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] Connected to AMQP server on 10.0.2.0:5672 2013-12-10 16:55:04.433 29291 DEBUG nova.openstack.common.rpc.amqp [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] Pool creating new connection create /usr/lib/python2.7/dist-packages/nova/openstack/common/rpc/amqp.py:75 2013-12-10 16:55:04.440 29291 DEBUG amqp [-] Start from server, version: 0.9, properties: {u'information': u'Licensed under the MPL. See http://www.rabbitmq.com/', u'product': u'RabbitMQ', u'copyright': u'Copyright (C) 2007-2012 VMware, Inc.', u'capabilities': {u'exchange_exchange_bindings': True, u'consumer_cancel_notify': True, u'publisher_confirms': True, u'basic.nack': True}, u'platform': u'Erlang/OTP', u'version': u'2.8.4'}, mechanisms: [u'PLAIN', u'AMQPLAIN'], locales: [u'en_US'] _start /usr/lib/python2.7/dist-packages/amqp/connection.py:706 2013-12-10 16:55:04.442 29291 DEBUG amqp [-] Open OK! _open_ok /usr/lib/python2.7/dist-packages/amqp/connection.py:592 2013-12-10 16:55:04.442 29291 DEBUG amqp [-] using channel_id: 1 __init__ /usr/lib/python2.7/dist-packages/amqp/channel.py:70 2013-12-10 16:55:04.444 29291 DEBUG amqp [-] Channel open _open_ok /usr/lib/python2.7/dist-packages/amqp/channel.py:420 2013-12-10 16:55:04.444 29291 INFO nova.openstack.common.rpc.common [req-3213bd9f-b67e-41aa-9973-c4490d9dce97 None None] Connected to AMQP server on 10.0.2.0:5672 root@cloudseed-1:/var/log/nova# ps -eaf|grep nova-compute nova 29291 1 0 16:55 ? 00:00:00 /usr/bin/python /usr/bin/nova-compute --config-file=/etc/nova/nova.conf --log-file=/var/log/nova/nova-compute.log --config-file=/etc/nova/nova-compute.conf root 29452 27493 0 17:21 pts/1 00:00:00 grep nova-compute root@cloudseed-1:/var/log/nova# strace -f -p 29291 Process 29291 attached - interrupt to quit epoll_wait(8, ^C <unfinished ...> Process 29291 detached root@cloudseed-1:/var/log/nova# cd /proc/29291/fd root@cloudseed-1:/proc/29291/fd# ls -l total 0 lrwx------ 1 nova nova 64 Dec 10 17:01 0 -> /dev/null lrwx------ 1 nova nova 64 Dec 10 17:01 1 -> /dev/null lrwx------ 1 nova nova 64 Dec 10 17:01 2 -> /dev/null lrwx------ 1 nova nova 64 Dec 10 17:01 3 -> socket:[9163150] l-wx------ 1 nova nova 64 Dec 10 17:01 4 -> /var/log/nova/nova-compute.log lr-x------ 1 nova nova 64 Dec 10 17:01 5 -> /dev/urandom lrwx------ 1 nova nova 64 Dec 10 17:01 6 -> socket:[9163155] lr-x------ 1 nova nova 64 Dec 10 17:01 7 -> /dev/urandom lrwx------ 1 nova nova 64 Dec 10 17:01 8 -> anon_inode:[eventpoll] lrwx------ 1 nova nova 64 Dec 10 17:01 9 -> socket:[9163157] root@cloudseed-1:/proc/29291/fd# netstat -tapen | grep 29291 tcp 0 0 10.0.0.1:48701 10.0.2.0:5672 ESTABLISHED 107 9163157 29291/python tcp 0 0 10.0.0.1:48700 10.0.2.0:5672 ESTABLISHED 107 9163155 29291/python root@cloudseed-1:/proc/29291/fd# root@cloudseed-1:/proc/29291/fd# nova-manage service list 2013-12-10 17:22:10.660 29465 DEBUG nova.servicegroup.api [-] ServiceGroup driver defined as an instance of db __new__ /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:62 2013-12-10 17:22:10.713 29465 DEBUG stevedore.extension [-] found extension EntryPoint.parse('file = nova.image.download.file') _load_plugins /usr/lib/python2.7/dist-packages/stevedore/extension.py:84 2013-12-10 17:22:10.725 29465 DEBUG stevedore.extension [-] found extension EntryPoint.parse('file = nova.image.download.file') _load_plugins /usr/lib/python2.7/dist-packages/stevedore/extension.py:84 2013-12-10 17:22:10.726 29465 DEBUG nova.openstack.common.lockutils [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Got semaphore "dbapi_backend" lock /usr/lib/python2.7/dist-packages/nova/openstack/common/lockutils.py:166 2013-12-10 17:22:10.726 29465 DEBUG nova.openstack.common.lockutils [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Got semaphore / lock "__get_backend" inner /usr/lib/python2.7/dist-packages/nova/openstack/common/lockutils.py:245 Binary Host Zone Status State Updated_At 2013-12-10 17:22:10.956 29465 DEBUG nova.servicegroup.api [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Check if the given member [{'binary': u'nova-scheduler', 'availability_zone': 'internal', 'deleted': 0L, 'created_at': datetime.datetime(2013, 11, 26, 16, 3, 16), 'updated_at': datetime.datetime(2013, 12, 10, 17, 22, 3), 'report_count': 121382L, 'topic': u'scheduler', 'host': u'cloudseed-dashboard', 'disabled': False, 'deleted_at': None, 'disabled_reason': None, 'id': 1L}] is part of the ServiceGroup, is up service_is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:94 2013-12-10 17:22:10.957 29465 DEBUG nova.servicegroup.drivers.db [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] DB_Driver.is_up last_heartbeat = 2013-12-10 17:22:03 elapsed = 7.957 is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/drivers/db.py:71 nova-scheduler cloudseed-dashboard internal enabled :-) 2013-12-10 17:22:03 2013-12-10 17:22:10.957 29465 DEBUG nova.servicegroup.api [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Check if the given member [{'binary': u'nova-console', 'availability_zone': 'internal', 'deleted': 0L, 'created_at': datetime.datetime(2013, 11, 26, 16, 3, 16), 'updated_at': datetime.datetime(2013, 12, 10, 17, 22, 3), 'report_count': 121370L, 'topic': u'console', 'host': u'cloudseed-dashboard', 'disabled': False, 'deleted_at': None, 'disabled_reason': None, 'id': 2L}] is part of the ServiceGroup, is up service_is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:94 2013-12-10 17:22:10.958 29465 DEBUG nova.servicegroup.drivers.db [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] DB_Driver.is_up last_heartbeat = 2013-12-10 17:22:03 elapsed = 7.958185 is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/drivers/db.py:71 nova-console cloudseed-dashboard internal enabled :-) 2013-12-10 17:22:03 2013-12-10 17:22:10.958 29465 DEBUG nova.servicegroup.api [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Check if the given member [{'binary': u'nova-conductor', 'availability_zone': 'internal', 'deleted': 0L, 'created_at': datetime.datetime(2013, 11, 26, 18, 0, 49), 'updated_at': datetime.datetime(2013, 12, 10, 17, 22, 4), 'report_count': 120691L, 'topic': u'conductor', 'host': u'cloudseed-dashboard', 'disabled': False, 'deleted_at': None, 'disabled_reason': None, 'id': 3L}] is part of the ServiceGroup, is up service_is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:94 2013-12-10 17:22:10.959 29465 DEBUG nova.servicegroup.drivers.db [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] DB_Driver.is_up last_heartbeat = 2013-12-10 17:22:04 elapsed = 6.959261 is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/drivers/db.py:71 nova-conductor cloudseed-dashboard internal enabled :-) 2013-12-10 17:22:04 2013-12-10 17:22:10.959 29465 DEBUG nova.servicegroup.api [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] Check if the given member [{'binary': u'nova-compute', 'availability_zone': 'nova', 'deleted': 0L, 'created_at': datetime.datetime(2013, 11, 26, 18, 1, 22), 'updated_at': datetime.datetime(2013, 12, 10, 16, 6, 15), 'report_count': 119833L, 'topic': u'compute', 'host': u'cloudseed-1', 'disabled': False, 'deleted_at': None, 'disabled_reason': None, 'id': 4L}] is part of the ServiceGroup, is up service_is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/api.py:94 2013-12-10 17:22:10.960 29465 DEBUG nova.servicegroup.drivers.db [req-5a9ac552-013a-4f86-a03f-b56f67bad2d6 None None] DB_Driver.is_up last_heartbeat = 2013-12-10 16:06:15 elapsed = 4555.960334 is_up /usr/lib/python2.7/dist-packages/nova/servicegroup/drivers/db.py:71 nova-compute cloudseed-1 nova enabled XXX 2013-12-10 16:06:15 root@cloudseed-1:/proc/29291/fd# ---- root@cloudseed-dashboard:/var/log/nova# rabbitmqctl list_consumers Listing consumers ... network.cloudseed-0025b501a001 <'rabbit@cloudseed-dashboard'.3.384.0> 2 true compute_fanout_c276b64fa980438e83bb6a0b08757659 <'rabbit@cloudseed-dashboard'.3.293.0> 3 true compute <'rabbit@cloudseed-dashboard'.3.392.0> 1 true compute <'rabbit@cloudseed-dashboard'.3.356.0> 1 true compute <'rabbit@cloudseed-dashboard'.3.324.0> 1 true compute <'rabbit@cloudseed-dashboard'.3.293.0> 1 true conductor.cloudseed-dashboard <'rabbit@cloudseed-dashboard'.3.29820.103> 2 true scheduler_fanout_d1bd43c6d7ee426cb2d36e9bf58dddaf <'rabbit@cloudseed-dashboard'.3.538.0> 3 true network_fanout_7084e29745c7428cb0999e3696967f4b <'rabbit@cloudseed-dashboard'.3.384.0> 3 true compute_fanout_e49a53e237d44905820a5a9bc18eb518 <'rabbit@cloudseed-dashboard'.3.324.0> 3 true network.cloudseed-0025b501a000 <'rabbit@cloudseed-dashboard'.3.284.0> 2 true compute.cloudseed-0025b501a001 <'rabbit@cloudseed-dashboard'.3.392.0> 2 true q-plugin <'rabbit@cloudseed-dashboard'.3.11666.0> 2 true q-l3-plugin <'rabbit@cloudseed-dashboard'.3.11666.0> 1 true compute.cloudseed-0025b501a000 <'rabbit@cloudseed-dashboard'.3.293.0> 2 true ceilometer.agent.compute <'rabbit@cloudseed-dashboard'.3.11804.0> 1 true conductor_fanout_eac080cfb745445faa54b35c9b262739 <'rabbit@cloudseed-dashboard'.3.29820.103> 3 true compute_fanout_122c101f3b00495c9f52ddf80c9494e9 <'rabbit@cloudseed-dashboard'.3.356.0> 3 true network_fanout_70d1ef577e1341b49eccbe1863de9175 <'rabbit@cloudseed-dashboard'.3.284.0> 3 true network_fanout_63868efcc3454d7b884b668cf015459d <'rabbit@cloudseed-dashboard'.3.317.0> 3 true reply_dfb751ccb8054d28a215d80cade34f50 <'rabbit@cloudseed-dashboard'.3.29404.103> 1 true scheduler <'rabbit@cloudseed-dashboard'.3.538.0> 1 true console.cloudseed-dashboard <'rabbit@cloudseed-dashboard'.3.549.0> 2 true compute.cloudseed-0025b501a005 <'rabbit@cloudseed-dashboard'.3.356.0> 2 true network.cloudseed-0025b501a005 <'rabbit@cloudseed-dashboard'.3.352.0> 2 true scheduler.cloudseed-dashboard <'rabbit@cloudseed-dashboard'.3.538.0> 2 true ceilometer.agent.compute_fanout_67b1a234f0d14f2fb954b82818b9c2f0 <'rabbit@cloudseed-dashboard'.3.11804.0> 3 true console <'rabbit@cloudseed-dashboard'.3.549.0> 1 true reply_1b5207e85ed344a9818a9fe67dfd101a <'rabbit@cloudseed-dashboard'.3.11962.103> 1 true network <'rabbit@cloudseed-dashboard'.3.384.0> 1 true network <'rabbit@cloudseed-dashboard'.3.352.0> 1 true network <'rabbit@cloudseed-dashboard'.3.317.0> 1 true network <'rabbit@cloudseed-dashboard'.3.284.0> 1 true compute_fanout_818c19dd593b4fc098e3b7a35e9a6a9f <'rabbit@cloudseed-dashboard'.3.392.0> 3 true ceilometer.agent.compute.cloudseed-1 <'rabbit@cloudseed-dashboard'.3.11804.0> 2 true compute.cloudseed-0025b501a003 <'rabbit@cloudseed-dashboard'.3.324.0> 2 true console_fanout_d3969082d6884ea5aec3a1967932c7ef <'rabbit@cloudseed-dashboard'.3.549.0> 3 true network_fanout_9ff8449c0e0547d1b84dff8f609b00b9 <'rabbit@cloudseed-dashboard'.3.352.0> 3 true network.cloudseed-0025b501a003 <'rabbit@cloudseed-dashboard'.3.317.0> 2 true conductor <'rabbit@cloudseed-dashboard'.3.29820.103> 1 true ...done. root@cloudseed-dashboard:/var/log/nova# rabbitmqctl list_consumers|grep compute.cloudseed-1 ceilometer.agent.compute.cloudseed-1 <'rabbit@cloudseed-dashboard'.3.11804.0> 2 true root@cloudseed-dashboard:/var/log/nova# |