Improve server/modules termination and how to wait for an hypervisor.

connection.
System to avoid duplicated name for nodes.
Reload and idle-pc support for Dynamips routers.
Hypervisor allocation for other Dynamips devices.
This commit is contained in:
grossmj 2014-02-20 17:39:03 -07:00
parent df798f4bea
commit 72d303069c
16 changed files with 257 additions and 56 deletions

View File

@ -22,6 +22,7 @@ Base class (interface) for modules
import gns3server.jsonrpc as jsonrpc
import multiprocessing
import zmq
import signal
import logging
log = logging.getLogger(__name__)
@ -101,6 +102,13 @@ class IModule(multiprocessing.Process):
Starts the event loop
"""
def signal_handler(signum=None, frame=None):
log.warning("Module {} got signal {}, exiting...".format(self.name, signum))
self.stop()
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT]:
signal.signal(sig, signal_handler)
log.info("{} module running with PID {}".format(self.name, self.pid))
self._setup()
try:
@ -113,7 +121,8 @@ class IModule(multiprocessing.Process):
Stops the event loop.
"""
self._ioloop.stop()
if self._ioloop:
self._ioloop.stop()
def send_response(self, results):
"""

View File

@ -105,9 +105,11 @@ class Dynamips(IModule):
self._atm_switches = {}
self._ethernet_hubs = {}
# def __del__(self):
#
# self._hypervisor_manager.stop_all_hypervisors()
def stop(self):
if self._hypervisor_manager:
self._hypervisor_manager.stop_all_hypervisors()
IModule.stop(self)
@IModule.route("dynamips.reset")
def reset(self, request):
@ -153,7 +155,6 @@ class Dynamips(IModule):
:param request: JSON request
"""
print("Create")
if not self._hypervisor_manager:
self._hypervisor_manager = HypervisorManager(request["path"], "/tmp")
@ -183,6 +184,11 @@ class Dynamips(IModule):
rhost = request["rhost"]
rport = request["rport"]
nio = NIO_UDP(node.hypervisor, lport, rhost, rport)
# elif request["nio"] == "NIO_UDP_Auto":
# lhost = request["lhost"]
# lport_start = request["lport_start"]
# lport_end = request["lport_end"]
# nio = NIO_UDP_auto(node.hypervisor, lhost, lport_start, lport_end)
elif request["nio"] == "NIO_GenericEthernet":
ethernet_device = request["ethernet_device"]
nio = NIO_GenericEthernet(node.hypervisor, ethernet_device)
@ -223,6 +229,20 @@ class Dynamips(IModule):
return response
# def allocate_udp_port_auto(self, node, lport_start, lport_end):
# """
# Allocates a UDP port in order to create an UDP NIO Auto.
#
# :param node: the node that needs to allocate an UDP port
# """
#
# self._nio_udp_auto = NIO_UDP_auto(node.hypervisor, node.hypervisor.host, lport_start, lport_end)
#
# response = {"lport": self._nio_udp_auto.lport,
# "lhost": self._nio_udp_auto.lhost}
#
# return response
def set_ghost_ios(self, router):
if not router.mmap:
@ -249,8 +269,9 @@ class Dynamips(IModule):
ghost.stop()
ghost.delete()
router.ghost_status = 2
router.ghost_file = ghost_instance
if router.ghost_file != ghost_instance:
router.ghost_status = 2
router.ghost_file = ghost_instance
@IModule.route("dynamips.nio.get_interfaces")
def nio_get_interfaces(self, request):

View File

@ -39,7 +39,7 @@ class ATMSW(object):
name = request["name"]
try:
hypervisor = self._hypervisor_manager.allocate_hypervisor_for_switch()
hypervisor = self._hypervisor_manager.allocate_hypervisor_for_simulated_device()
atmsw = ATMSwitch(hypervisor, name)
except DynamipsError as e:
self.send_custom_error(str(e))
@ -65,6 +65,7 @@ class ATMSW(object):
atmsw = self._atm_switches[atmsw_id]
try:
atmsw.delete()
self._hypervisor_manager.unallocate_hypervisor_for_simulated_device(atmsw)
except DynamipsError as e:
self.send_custom_error(str(e))
return
@ -105,6 +106,7 @@ class ATMSW(object):
self.send_custom_error(str(e))
return
response["port_name"] = request["port_name"]
self.send_response(response)
@IModule.route("dynamips.atmsw.add_nio")
@ -121,7 +123,7 @@ class ATMSW(object):
atmsw = self._atm_switches[atmsw_id]
port = request["port"]
mapping = request["mapping"]
mappings = request["mappings"]
try:
nio = self.create_nio(atmsw, request)
@ -132,7 +134,7 @@ class ATMSW(object):
try:
atmsw.add_nio(nio, port)
pvc_entry = re.compile(r"""^([0-9]*):([0-9]*):([0-9]*)$""")
for source, destination in mapping.items():
for source, destination in mappings.items():
match_source_pvc = pvc_entry.search(source)
match_destination_pvc = pvc_entry.search(destination)
if match_source_pvc and match_destination_pvc:

View File

@ -38,7 +38,7 @@ class ETHHUB(object):
name = request["name"]
try:
hypervisor = self._hypervisor_manager.allocate_hypervisor_for_switch()
hypervisor = self._hypervisor_manager.allocate_hypervisor_for_simulated_device()
ethhub = Hub(hypervisor, name)
except DynamipsError as e:
self.send_custom_error(str(e))
@ -64,6 +64,7 @@ class ETHHUB(object):
ethhub = self._ethernet_hubs[ethhub_id]
try:
ethhub.delete()
self._hypervisor_manager.unallocate_hypervisor_for_simulated_device(ethhub)
except DynamipsError as e:
self.send_custom_error(str(e))
return
@ -106,6 +107,7 @@ class ETHHUB(object):
self.send_custom_error(str(e))
return
response["port_name"] = request["port_name"]
self.send_response(response)
@IModule.route("dynamips.ethhub.add_nio")

View File

@ -38,7 +38,7 @@ class ETHSW(object):
name = request["name"]
try:
hypervisor = self._hypervisor_manager.allocate_hypervisor_for_switch()
hypervisor = self._hypervisor_manager.allocate_hypervisor_for_simulated_device()
ethsw = EthernetSwitch(hypervisor, name)
except DynamipsError as e:
self.send_custom_error(str(e))
@ -64,6 +64,7 @@ class ETHSW(object):
ethsw = self._ethernet_switches[ethsw_id]
try:
ethsw.delete()
self._hypervisor_manager.unallocate_hypervisor_for_simulated_device(ethsw)
except DynamipsError as e:
self.send_custom_error(str(e))
return
@ -116,6 +117,7 @@ class ETHSW(object):
self.send_custom_error(str(e))
return
response["port_name"] = request["port_name"]
self.send_response(response)
@IModule.route("dynamips.ethsw.add_nio")

View File

@ -38,7 +38,7 @@ class FRSW(object):
name = request["name"]
try:
hypervisor = self._hypervisor_manager.allocate_hypervisor_for_switch()
hypervisor = self._hypervisor_manager.allocate_hypervisor_for_simulated_device()
frsw = FrameRelaySwitch(hypervisor, name)
except DynamipsError as e:
self.send_custom_error(str(e))
@ -64,6 +64,7 @@ class FRSW(object):
frsw = self._frame_relay_switches[frsw_id]
try:
frsw.delete()
self._hypervisor_manager.unallocate_hypervisor_for_simulated_device(frsw)
except DynamipsError as e:
self.send_custom_error(str(e))
return
@ -104,6 +105,7 @@ class FRSW(object):
self.send_custom_error(str(e))
return
response["port_name"] = request["port_name"]
self.send_response(response)
@IModule.route("dynamips.frsw.add_nio")
@ -120,7 +122,7 @@ class FRSW(object):
frsw = self._frame_relay_switches[frsw_id]
port = request["port"]
mapping = request["mapping"]
mappings = request["mappings"]
try:
nio = self.create_nio(frsw, request)
@ -132,7 +134,7 @@ class FRSW(object):
frsw.add_nio(nio, port)
# add the VCs mapped with this port/nio
for source, destination in mapping.items():
for source, destination in mappings.items():
source_port, source_dlci = map(int, source.split(':'))
destination_port, destination_dlci = map(int, destination.split(':'))
if frsw.has_port(destination_port):

View File

@ -99,18 +99,27 @@ class VM(object):
log.debug("received request {}".format(request))
#TODO: JSON schema validation
#name = request["name"]
name = None
if "name" in request:
name = request["name"]
platform = request["platform"]
image = request["image"]
ram = request["ram"]
try:
hypervisor = self._hypervisor_manager.allocate_hypervisor_for_router(image, ram)
router = PLATFORMS[platform](hypervisor)
router = PLATFORMS[platform](hypervisor, name)
router.ram = ram
router.image = image
router.sparsemem = self._hypervisor_manager.sparse_memory_support
router.mmap = self._hypervisor_manager.mmap_support
if "console" in request:
router.console = request["console"]
if "aux" in request:
router.aux = request["aux"]
if "mac_addr" in request:
router.mac_addr = request["mac_addr"]
# JIT sharing support
if self._hypervisor_manager.jit_sharing_support:
@ -161,6 +170,7 @@ class VM(object):
router = self._routers[router_id]
try:
router.delete()
self._hypervisor_manager.unallocate_hypervisor_for_router(router)
except DynamipsError as e:
self.send_custom_error(str(e))
return
@ -223,6 +233,27 @@ class VM(object):
return
self.send_response(request)
@IModule.route("dynamips.vm.reload")
def vm_reload(self, request):
"""
Reloads a VM (router)
:param request: JSON request
"""
#TODO: JSON schema validation for the request
log.debug("received request {}".format(request))
router_id = request["id"]
router = self._routers[router_id]
try:
if router.get_status() != "inactive":
router.stop()
router.start()
except DynamipsError as e:
self.send_custom_error(str(e))
return
self.send_response(request)
@IModule.route("dynamips.vm.update")
def vm_update(self, request):
"""
@ -286,6 +317,34 @@ class VM(object):
# for now send back the original request
self.send_response(request)
@IModule.route("dynamips.vm.idlepcs")
def vm_idlepcs(self, request):
"""
Get idle-pc proposals.
:param request: JSON request
"""
#TODO: JSON schema validation for the request
log.debug("received request {}".format(request))
router_id = request["id"]
router = self._routers[router_id]
try:
if "compute" in request and request["compute"] == False:
idlepcs = router.show_idle_pc_prop()
else:
# reset the current idlepc value
router.idlepc = "0x0"
idlepcs = router.get_idle_pc_prop()
except DynamipsError as e:
self.send_custom_error(str(e))
return
response = {"id": router_id,
"idlepcs": idlepcs}
self.send_response(response)
@IModule.route("dynamips.vm.allocate_udp_port")
def vm_allocate_udp_port(self, request):
"""
@ -306,6 +365,7 @@ class VM(object):
self.send_custom_error(str(e))
return
response["port_name"] = request["port_name"]
self.send_response(response)
@IModule.route("dynamips.vm.add_nio")
@ -324,8 +384,6 @@ class VM(object):
slot = request["slot"]
port = request["port"]
print(request)
try:
nio = self.create_nio(router, request)
except DynamipsError as e:

View File

@ -401,23 +401,30 @@ class HypervisorManager(object):
:param timeout: timeout value (default is 10 seconds)
"""
# try to connect 5 times
for _ in range(0, 5):
connection_success = False
begin = time.time()
# try to connect for 10 seconds
while(time.time() - begin < 10.0):
time.sleep(0.01)
sock = None
try:
s = socket.create_connection((host, port), timeout)
sock = socket.create_connection((host, port), timeout)
except socket.error as e:
time.sleep(0.5)
last_exception = e
#time.sleep(0.01)
continue
finally:
if sock:
sock.close()
connection_success = True
break
if connection_success:
s.close()
#time.sleep(0.1)
else:
if not connection_success:
# FIXME: throw exception here
log.critical("Couldn't connect to hypervisor on {}:{} :{}".format(host, port,
last_exception))
else:
log.info("Dynamips server ready after {:.4f} seconds".format(time.time() - begin))
def start_new_hypervisor(self):
"""
@ -501,9 +508,9 @@ class HypervisorManager(object):
hypervisor.stop()
self._hypervisors.remove(hypervisor)
def allocate_hypervisor_for_switch(self):
def allocate_hypervisor_for_simulated_device(self):
"""
Allocates a Dynamips hypervisor for a specific switch
Allocates a Dynamips hypervisor for a specific Dynamips simulated device.
:returns: the allocated hypervisor object
"""
@ -517,6 +524,18 @@ class HypervisorManager(object):
# no hypervisor, let's start one!
return self.start_new_hypervisor()
def unallocate_hypervisor_for_simulated_device(self, device):
"""
Unallocates a Dynamips hypervisor for a specific Dynamips simulated device.
:param device: device object
"""
hypervisor = device.hypervisor
if not hypervisor.devices:
hypervisor.stop()
self._hypervisors.remove(hypervisor)
def stop_all_hypervisors(self):
"""
Stops all hypervisors.

View File

@ -35,6 +35,7 @@ class ATMSwitch(object):
:param name: name for this switch
"""
_allocated_names = []
_instance_count = 1
def __init__(self, hypervisor, name=None):
@ -45,8 +46,15 @@ class ATMSwitch(object):
# let's create a unique name if none has been chosen
if not name:
name = "ATM" + str(self._id)
name_id = self._id
while True:
name = "ATM" + str(name_id)
# check if the name has already been allocated to another switch
if name not in self._allocated_names:
break
name_id += 1
self._allocated_names.append(name)
self._hypervisor = hypervisor
self._name = '"' + name + '"' # put name into quotes to protect spaces
self._hypervisor.send("atmsw create {}".format(self._name))
@ -61,10 +69,11 @@ class ATMSwitch(object):
@classmethod
def reset(cls):
"""
Reset the instance count.
Resets the instance count and the allocated names list.
"""
cls._instance_count = 1
cls._allocated_names.clear()
@property
def id(self):
@ -94,6 +103,7 @@ class ATMSwitch(object):
:param new_name: New name for this switch
"""
new_name_no_quotes = new_name
new_name = '"' + new_name + '"' # put the new name into quotes to protect spaces
self._hypervisor.send("atmsw rename {name} {new_name}".format(name=self._name,
new_name=new_name))
@ -102,7 +112,9 @@ class ATMSwitch(object):
id=self._id,
new_name=new_name))
self._allocated_names.remove(self.name)
self._name = new_name
self._allocated_names.append(new_name_no_quotes)
@property
def hypervisor(self):
@ -153,6 +165,7 @@ class ATMSwitch(object):
log.info("ATM switch {name} [id={id}] has been deleted".format(name=self._name,
id=self._id))
self._hypervisor.devices.remove(self)
self._allocated_names.remove(self.name)
def has_port(self, port):
"""

View File

@ -31,6 +31,8 @@ class Bridge(object):
:param name: name for this bridge
"""
_allocated_names = []
def __init__(self, hypervisor, name):
self._hypervisor = hypervisor
@ -57,11 +59,14 @@ class Bridge(object):
:param new_name: New name for this bridge
"""
new_name_no_quotes = new_name
new_name = '"' + new_name + '"' # put the new name into quotes to protect spaces
self._hypervisor.send("nio_bridge rename {name} {new_name}".format(name=self._name,
new_name=new_name))
self._allocated_names.remove(self.name)
self._name = new_name
self._allocated_names.append(new_name_no_quotes)
@property
def hypervisor(self):
@ -99,6 +104,7 @@ class Bridge(object):
self._hypervisor.send("nio_bridge delete {}".format(self._name))
self._hypervisor.devices.remove(self)
self._allocated_names.remove(self.name)
def add_nio(self, nio):
"""

View File

@ -44,16 +44,16 @@ class C2600(Router):
# adapters to insert by default corresponding the
# chosen chassis.
integrated_adapters = {'2610': C2600_MB_1E,
'2611': C2600_MB_2E,
'2620': C2600_MB_1FE,
'2621': C2600_MB_2FE,
'2610XM': C2600_MB_1FE,
'2611XM': C2600_MB_2FE,
'2620XM': C2600_MB_1FE,
'2621XM': C2600_MB_2FE,
'2650XM': C2600_MB_1FE,
'2651XM': C2600_MB_2FE}
integrated_adapters = {"2610": C2600_MB_1E,
"2611": C2600_MB_2E,
"2620": C2600_MB_1FE,
"2621": C2600_MB_2FE,
"2610XM": C2600_MB_1FE,
"2611XM": C2600_MB_2FE,
"2620XM": C2600_MB_1FE,
"2621XM": C2600_MB_2FE,
"2650XM": C2600_MB_1FE,
"2651XM": C2600_MB_2FE}
def __init__(self, hypervisor, name=None, chassis="2610"):
Router.__init__(self, hypervisor, name, platform="c2600")

View File

@ -36,6 +36,7 @@ class EthernetSwitch(object):
:param name: name for this switch
"""
_allocated_names = []
_instance_count = 1
def __init__(self, hypervisor, name=None):
@ -46,8 +47,15 @@ class EthernetSwitch(object):
# let's create a unique name if none has been chosen
if not name:
name = "SW" + str(self._id)
name_id = self._id
while True:
name = "SW" + str(name_id)
# check if the name has already been allocated to another switch
if name not in self._allocated_names:
break
name_id += 1
self._allocated_names.append(name)
self._hypervisor = hypervisor
self._name = '"' + name + '"' # put name into quotes to protect spaces
self._hypervisor.send("ethsw create {}".format(self._name))
@ -62,10 +70,11 @@ class EthernetSwitch(object):
@classmethod
def reset(cls):
"""
Reset the instance count.
Resets the instance count and the allocated names list.
"""
cls._instance_count = 1
cls._allocated_names.clear()
@property
def id(self):
@ -95,6 +104,7 @@ class EthernetSwitch(object):
:param new_name: New name for this switch
"""
new_name_no_quotes = new_name
new_name = '"' + new_name + '"' # put the new name into quotes to protect spaces
self._hypervisor.send("ethsw rename {name} {new_name}".format(name=self._name,
new_name=new_name))
@ -103,7 +113,9 @@ class EthernetSwitch(object):
id=self._id,
new_name=new_name))
self._allocated_names.remove(self.name)
self._name = new_name
self._allocated_names.append(new_name_no_quotes)
@property
def hypervisor(self):
@ -154,6 +166,7 @@ class EthernetSwitch(object):
log.info("Ethernet switch {name} [id={id}] has been deleted".format(name=self._name,
id=self._id))
self._hypervisor.devices.remove(self)
self._allocated_names.remove(self.name)
def add_nio(self, nio, port):
"""

View File

@ -35,6 +35,7 @@ class FrameRelaySwitch(object):
:param name: name for this switch
"""
_allocated_names = []
_instance_count = 1
def __init__(self, hypervisor, name=None):
@ -45,8 +46,15 @@ class FrameRelaySwitch(object):
# let's create a unique name if none has been chosen
if not name:
name = "FR" + str(self._id)
name_id = self._id
while True:
name = "FR" + str(name_id)
# check if the name has already been allocated to another switch
if name not in self._allocated_names:
break
name_id += 1
self._allocated_names.append(name)
self._hypervisor = hypervisor
self._name = '"' + name + '"' # put name into quotes to protect spaces
self._hypervisor.send("frsw create {}".format(self._name))
@ -61,10 +69,11 @@ class FrameRelaySwitch(object):
@classmethod
def reset(cls):
"""
Reset the instance count.
Resets the instance count and the allocated names list.
"""
cls._instance_count = 1
cls._allocated_names.clear()
@property
def id(self):
@ -94,6 +103,7 @@ class FrameRelaySwitch(object):
:param new_name: New name for this switch
"""
new_name_no_quotes = new_name
new_name = '"' + new_name + '"' # put the new name into quotes to protect spaces
self._hypervisor.send("frsw rename {name} {new_name}".format(name=self._name,
new_name=new_name))
@ -102,7 +112,9 @@ class FrameRelaySwitch(object):
id=self._id,
new_name=new_name))
self._allocated_names.remove(self.name)
self._name = new_name
self._allocated_names.append(new_name_no_quotes)
@property
def hypervisor(self):
@ -153,6 +165,7 @@ class FrameRelaySwitch(object):
log.info("Frame Relay switch {name} [id={id}] has been deleted".format(name=self._name,
id=self._id))
self._hypervisor.devices.remove(self)
self._allocated_names.remove(self.name)
def has_port(self, port):
"""

View File

@ -45,8 +45,15 @@ class Hub(Bridge):
# let's create a unique name if none has been chosen
if not name:
name = "Hub" + str(self._id)
name_id = self._id
while True:
name = "Hub" + str(name_id)
# check if the name has already been allocated to another switch
if name not in self._allocated_names:
break
name_id += 1
self._allocated_names.append(name)
self._mapping = {}
Bridge.__init__(self, hypervisor, name)
@ -56,10 +63,11 @@ class Hub(Bridge):
@classmethod
def reset(cls):
"""
Reset the instance count.
Resets the instance count and the allocated names list.
"""
cls._instance_count = 1
cls._allocated_names.clear()
@property
def id(self):

View File

@ -22,6 +22,7 @@ http://github.com/GNS3/dynamips/blob/master/README.hypervisor#L77
from __future__ import unicode_literals
from ..dynamips_error import DynamipsError
import time
import sys
import os
@ -39,6 +40,7 @@ class Router(object):
:param ghost_flag: used when creating a ghost IOS.
"""
_allocated_names = []
_instance_count = 1
_status = {0: "inactive",
1: "shutting down",
@ -53,8 +55,15 @@ class Router(object):
# let's create a unique name if none has been chosen
if not name:
name = "R" + str(self._id)
name_id = self._id
while True:
name = "R" + str(name_id)
# check if the name has already been allocated to another router
if name not in self._allocated_names:
break
name_id += 1
self._allocated_names.append(name)
self._hypervisor = hypervisor
self._name = '"' + name + '"' # put name into quotes to protect spaces
self._platform = platform
@ -106,10 +115,11 @@ class Router(object):
@classmethod
def reset(cls):
"""
Reset the instance count.
Resets the instance count and the allocated names list.
"""
cls._instance_count = 1
cls._allocated_names.clear()
def defaults(self):
"""
@ -179,6 +189,10 @@ class Router(object):
:param new_name: new name string
"""
if new_name in self._allocated_names:
raise DynamipsError('Name "{}" is already used by another router'.format(new_name))
new_name_no_quotes = new_name
new_name = '"' + new_name + '"' # put the new name into quotes to protect spaces
self._hypervisor.send("vm rename {name} {new_name}".format(name=self._name,
new_name=new_name))
@ -187,7 +201,9 @@ class Router(object):
id=self._id,
new_name=new_name))
self._allocated_names.remove(self.name)
self._name = new_name
self._allocated_names.append(new_name_no_quotes)
@property
def platform(self):
@ -237,6 +253,7 @@ class Router(object):
self._hypervisor.devices.remove(self)
log.info("router {name} [id={id}] has been deleted".format(name=self._name, id=self._id))
self._allocated_names.remove(self.name)
def start(self):
"""
@ -457,7 +474,7 @@ class Router(object):
old_ram=self._ram,
new_ram=ram))
self._hypervisor.decrease_memory_load(self._ram)
self._hypervisor.decrease_memory_load(ram)
self._ram = ram
self._hypervisor.increase_memory_load(self._ram)
@ -627,7 +644,17 @@ class Router(object):
:returns: list of idle PC proposal
"""
return self._hypervisor.send("vm get_idle_pc_prop {} 0".format(self._name))
if not self.is_running():
# router is not running
raise DynamipsError("router {name} is not running".format(name=self._name))
log.info("router {name} [id={id}] has started calculating idle-pc values".format(name=self._name, id=self._id))
begin = time.time()
idlepcs = self._hypervisor.send("vm get_idle_pc_prop {} 0".format(self._name))
log.info("router {name} [id={id}] has finished calculating idle-pc values after {time:.4f} seconds".format(name=self._name,
id=self._id,
time=time.time() - begin))
return idlepcs
def show_idle_pc_prop(self):
"""
@ -636,6 +663,10 @@ class Router(object):
:returns: list of idle PC proposal
"""
if not self.is_running():
# router is not running
raise DynamipsError("router {name} is not running".format(name=self._name))
return self._hypervisor.send("vm show_idle_pc_prop {} 0".format(self._name))
@property

View File

@ -96,7 +96,7 @@ class Server(object):
tornado.autoreload.add_reload_hook(functools.partial(self._cleanup, stop=False))
def signal_handler(signum=None, frame=None):
log.warning("Got signal {}, exiting...".format(signum))
log.warning("Server got signal {}, exiting...".format(signum))
self._cleanup()
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT]:
@ -139,17 +139,19 @@ class Server(object):
def _cleanup(self, stop=True):
"""
Shutdowns running module processes
Shutdowns any running module processes
and close remaining Tornado ioloop file descriptors
:param stop: Stop the ioloop if True (default)
:param stop: stops the ioloop if True (default)
"""
# terminate all modules
for module in self._modules:
log.info("terminating {}".format(module.name))
module.terminate()
module.join(timeout=1)
if module.is_alive():
log.info("terminating {}".format(module.name))
module.terminate()
module.join(timeout=1)
ioloop = tornado.ioloop.IOLoop.instance()
# close any fd that would have remained open...