1.3.8

Backup upload images projects

CHANGELOG

@@ -1,5 +1,42 @@
 # Change Log
 
+## 1.3.8 27/07/2015
+
+* Catch ProcessLookupError when updating iouyap config. Fixes #255.
+* Fixes IOS adapters and WICS cannot be removed. Fixes #282.
+* Makes sure the loop is running when closing the app.
+* Catch GeneratorExit exception. Fixes #231.
+* Fixes missing chipset info for VirtualBox VM. Fixes #254.
+* Fixes IOURC upload.
+* Restore images & projects tarballs
+* Allow users to backup projects and images.
+* Update gns3.conf.upstart.
+* Fix incorrect vboxmanage sudo command.
+* Backport from 1.4: option to drop nvram & disk files for IOS routers in order to save disk space.
+* Backport from 1.4: Remove timeout to wait for connections to finish.
+* Backport from 1.4: Fixes RuntimeError: Event loop is closed.
+* Backport from 1.4: Bind host on 0.0.0.0 when checking for a free UDP port.
+
+## 1.3.7 22/06/2015
+
+* Prevent install on Python 2
+
+## 1.3.6 16/06/2015
+
+* Fix an issue with 1.4dev compatibility
+
+## 1.3.5 16/06/15
+
+* Ignore invalid characters when reading the output of a process
+* Turn on / off authentication
+* Ensure no colored output on Windows
+* Do not stop saving IOS router configs when there is an exception while a project is committed.
+* Create a private config file if expected
+* Distribute our own version of netifaces working with python 3
+* Fix crash if a private config exist in IOS but no private config file
+* Basic Auth support
+* Fix crash when virtualbox list of VMS return an empty line
+
 ## 1.3.4 02/06/15
 
 * Drop useless dependencie dateutil

gns3server/crash_report.py

@@ -51,7 +51,7 @@ class CrashReport:
     Report crash to a third party service
     """
 
-    DSN = "sync+https://9e6f04df72c74b6894a6dcd2928d069e:2035d1beb1654136b170f1e91f05ee51@app.getsentry.com/38482"
+    DSN = "sync+https://c06a407cbe0d4036be7873b54ebfa0ef:d6129d3fe91a4ea0b96ecbcf99678b1b@app.getsentry.com/38482"
     if hasattr(sys, "frozen"):
         cacert = os.path.join(os.getcwd(), "cacert.pem")
         if os.path.isfile(cacert):

gns3server/handlers/upload_handler.py

@@ -18,6 +18,9 @@
 import os
 import aiohttp
 import stat
+import io
+import tarfile
+import asyncio
 
 from ..config import Config
 from ..web.route import Route
@@ -59,29 +62,97 @@ class UploadHandler:
             response.redirect("/upload")
             return
 
-        if data["type"] not in ["IOU", "IOURC", "QEMU", "IOS"]:
-            raise aiohttp.web.HTTPForbidden("You are not authorized to upload this kind of image {}".format(data["type"]))
+        if data["type"] not in ["IOU", "IOURC", "QEMU", "IOS", "IMAGES", "PROJECTS"]:
+            raise aiohttp.web.HTTPForbidden(text="You are not authorized to upload this kind of image {}".format(data["type"]))
 
-        if data["type"] == "IOURC":
-            destination_dir = os.path.expanduser("~/")
-            destination_path = os.path.join(destination_dir, ".iourc")
-        else:
-            destination_dir = os.path.join(UploadHandler.image_directory(), data["type"])
-            destination_path = os.path.join(destination_dir, data["file"].filename)
         try:
-            os.makedirs(destination_dir, exist_ok=True)
-            with open(destination_path, "wb+") as f:
-                chunk = data["file"].file.read()
-                f.write(chunk)
-            st = os.stat(destination_path)
-            os.chmod(destination_path, st.st_mode | stat.S_IXUSR)
+            if data["type"] == "IMAGES":
+                UploadHandler._restore_directory(data["file"], UploadHandler.image_directory())
+            elif data["type"] == "PROJECTS":
+                UploadHandler._restore_directory(data["file"], UploadHandler.project_directory())
+            else:
+                if data["type"] == "IOURC":
+                    destination_dir = os.path.expanduser("~/")
+                    destination_path = os.path.join(destination_dir, ".iourc")
+                else:
+                    destination_dir = os.path.join(UploadHandler.image_directory(), data["type"])
+                    destination_path = os.path.join(destination_dir, data["file"].filename)
+                os.makedirs(destination_dir, exist_ok=True)
+                with open(destination_path, "wb+") as f:
+                    chunk = data["file"].file.read()
+                    f.write(chunk)
+                st = os.stat(destination_path)
+                os.chmod(destination_path, st.st_mode | stat.S_IXUSR)
         except OSError as e:
             response.html("Could not upload file: {}".format(e))
             response.set_status(200)
             return
         response.redirect("/upload")
 
+    @classmethod
+    @Route.get(
+        r"/backup/images.tar",
+        description="Backup GNS3 images",
+        api_version=None
+    )
+    def backup_images(request, response):
+        yield from UploadHandler._backup_directory(request, response, UploadHandler.image_directory())
+
+    @classmethod
+    @Route.get(
+        r"/backup/projects.tar",
+        description="Backup GNS3 projects",
+        api_version=None
+    )
+    def backup_images(request, response):
+        yield from UploadHandler._backup_directory(request, response, UploadHandler.project_directory())
+
+    @staticmethod
+    def _restore_directory(file, directory):
+        """
+        Extract from HTTP stream the content of a tar
+        """
+        destination_path = os.path.join(directory, "archive.tar")
+        os.makedirs(directory, exist_ok=True)
+        with open(destination_path, "wb+") as f:
+            chunk = file.file.read()
+            f.write(chunk)
+        t = tarfile.open(destination_path)
+        t.extractall(directory)
+        t.close()
+        os.remove(destination_path)
+
+    @staticmethod
+    @asyncio.coroutine
+    def _backup_directory(request, response, directory):
+        """
+        Return a tar archive from a directory
+        """
+        response.content_type = 'application/x-gtar'
+        response.set_status(200)
+        response.enable_chunked_encoding()
+        # Very important: do not send a content length otherwise QT close the connection but curl can consume the Feed
+        response.content_length = None
+        response.start(request)
+
+        buffer = io.BytesIO()
+        with tarfile.open('arch.tar', 'w', fileobj=buffer) as tar:
+            for root, dirs, files in os.walk(directory):
+                for file in files:
+                    path = os.path.join(root, file)
+                    tar.add(os.path.join(root, file), arcname=os.path.relpath(path, directory))
+                    response.write(buffer.getvalue())
+                    yield from response.drain()
+                    buffer.truncate(0)
+                    buffer.seek(0)
+        yield from response.write_eof()
+
     @staticmethod
     def image_directory():
         server_config = Config.instance().get_section_config("Server")
         return os.path.expanduser(server_config.get("images_path", "~/GNS3/images"))
+
+    @staticmethod
+    def project_directory():
+        server_config = Config.instance().get_section_config("Server")
+        return os.path.expanduser(server_config.get("projects_path", "~/GNS3/projects"))

gns3server/modules/base_manager.py

@@ -113,7 +113,7 @@ class BaseManager:
             for future in done:
                 try:
                     future.result()
-                except Exception as e:
+                except (Exception, GeneratorExit) as e:
                     log.error("Could not close VM {}".format(e), exc_info=1)
                     continue
 

gns3server/modules/dynamips/__init__.py

@@ -130,7 +130,7 @@ class Dynamips(BaseManager):
             for future in done:
                 try:
                     future.result()
-                except Exception as e:
+                except (Exception, GeneratorExit) as e:
                     log.error("Could not stop device hypervisor {}".format(e), exc_info=1)
                     continue
 
@@ -154,7 +154,7 @@ class Dynamips(BaseManager):
             for future in done:
                 try:
                     future.result()
-                except Exception as e:
+                except (Exception, GeneratorExit) as e:
                     log.error("Could not delete device {}".format(e), exc_info=1)
 
     @asyncio.coroutine
@@ -171,8 +171,11 @@ class Dynamips(BaseManager):
         files = glob.glob(os.path.join(project_dir, "*.ghost"))
         files += glob.glob(os.path.join(project_dir, "*_lock"))
         files += glob.glob(os.path.join(project_dir, "ilt_*"))
-        files += glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_*_rommon_vars"))
-        files += glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_*_ssa"))
+        files += glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_i[0-9]*_rommon_vars"))
+        files += glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_i[0-9]*_ssa"))
+        files += glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_i[0-9]*_log.txt"))
+        files += glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_i[0-9]*_rom"))
+        files += glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_i[0-9]*_bootflash"))
         for file in files:
             try:
                 log.debug("Deleting file {}".format(file))
@@ -210,7 +213,11 @@ class Dynamips(BaseManager):
         # save the configs when the project is committed
         for vm in self._vms.copy().values():
             if vm.project.id == project.id:
-                yield from vm.save_configs()
+                try:
+                    yield from vm.save_configs()
+                except DynamipsError as e:
+                    log.warning(e)
+                    continue
 
     @property
     def dynamips_path(self):

gns3server/modules/dynamips/nodes/router.py

@@ -87,6 +87,7 @@ class Router(BaseVM):
             self._exec_area = 64  # 64 MB on other systems
         self._disk0 = 0  # Megabytes
         self._disk1 = 0  # Megabytes
+        self._auto_delete_disks = False
         self._aux = aux
         self._mac_addr = ""
         self._system_id = "FTX0945W0MY"  # processor board ID in IOS
@@ -145,6 +146,7 @@ class Router(BaseVM):
                        "exec_area": self._exec_area,
                        "disk0": self._disk0,
                        "disk1": self._disk1,
+                       "auto_delete_disks": self._auto_delete_disks,
                        "console": self._console,
                        "aux": self._aux,
                        "mac_addr": self._mac_addr,
@@ -158,7 +160,7 @@ class Router(BaseVM):
         for slot in self._slots:
             if slot:
                 slot = str(slot)
-                router_info["slot" + str(slot_number)] = slot
+            router_info["slot" + str(slot_number)] = slot
             slot_number += 1
 
         # add the wics
@@ -166,6 +168,8 @@ class Router(BaseVM):
             for wic_slot_number in range(0, len(self._slots[0].wics)):
                 if self._slots[0].wics[wic_slot_number]:
                     router_info["wic" + str(wic_slot_number)] = str(self._slots[0].wics[wic_slot_number])
+                else:
+                    router_info["wic" + str(wic_slot_number)] = None
 
         return router_info
 
@@ -182,6 +186,16 @@ class Router(BaseVM):
     @asyncio.coroutine
     def create(self):
 
+        # delete any previous file with same Dynamips identifier
+        project_dir = os.path.join(self.project.module_working_directory(self.manager.module_name.lower()))
+        for file in glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_i{}_*".format(self._dynamips_id))):
+            try:
+                log.debug("Deleting file {}".format(file))
+                yield from wait_run_in_executor(os.remove, file)
+            except OSError as e:
+                log.warn("Could not delete file {}: {}".format(file, e))
+                continue
+
         if not self._hypervisor:
             module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
             self._hypervisor = yield from self.manager.start_new_hypervisor(working_dir=module_workdir)
@@ -342,6 +356,20 @@ class Router(BaseVM):
                 pass
             yield from self.hypervisor.stop()
 
+        if self._auto_delete_disks:
+            # delete nvram and disk files
+            project_dir = os.path.join(self.project.module_working_directory(self.manager.module_name.lower()))
+            files = glob.glob(os.path.join(project_dir, "{}_i{}_disk[0-1]".format(self.platform, self.dynamips_id)))
+            files += glob.glob(os.path.join(project_dir, "{}_i{}_slot[0-1]".format(self.platform, self.dynamips_id)))
+            files += glob.glob(os.path.join(project_dir, "{}_i{}_nvram".format(self.platform, self.dynamips_id)))
+            files += glob.glob(os.path.join(project_dir, "{}_i{}_flash[0-1]".format(self.platform, self.dynamips_id)))
+            for file in files:
+                try:
+                    log.debug("Deleting file {}".format(file))
+                    yield from wait_run_in_executor(os.remove, file)
+                except OSError as e:
+                    log.warn("Could not delete file {}: {}".format(file, e))
+                    continue
         self._closed = True
 
     @property
@@ -851,6 +879,30 @@ class Router(BaseVM):
                                                                                                     new_disk1=disk1))
         self._disk1 = disk1
 
+    @property
+    def auto_delete_disks(self):
+        """
+        Returns True if auto delete disks is enabled on this router.
+
+        :returns: boolean either auto delete disks is activated or not
+        """
+
+        return self._auto_delete_disks
+
+    @asyncio.coroutine
+    def set_auto_delete_disks(self, auto_delete_disks):
+        """
+        Enable/disable use of auto delete disks
+
+        :param auto_delete_disks: activate/deactivate auto delete disks (boolean)
+        """
+
+        if auto_delete_disks:
+            log.info('Router "{name}" [{id}]: auto delete disks enabled'.format(name=self._name, id=self._id))
+        else:
+            log.info('Router "{name}" [{id}]: auto delete disks disabled'.format(name=self._name, id=self._id))
+        self._auto_delete_disks = auto_delete_disks
+
     @asyncio.coroutine
     def set_console(self, console):
         """
@@ -1178,11 +1230,20 @@ class Router(BaseVM):
         if not adapter.port_exists(port_number):
             raise DynamipsError("Port {port_number} does not exist in adapter {adapter}".format(adapter=adapter,
                                                                                                 port_number=port_number))
-
-        yield from self._hypervisor.send('vm slot_add_nio_binding "{name}" {slot_number} {port_number} {nio}'.format(name=self._name,
-                                                                                                                     slot_number=slot_number,
-                                                                                                                     port_number=port_number,
-                                                                                                                     nio=nio))
+        try:
+            yield from self._hypervisor.send('vm slot_add_nio_binding "{name}" {slot_number} {port_number} {nio}'.format(name=self._name,
+                                                                                                                         slot_number=slot_number,
+                                                                                                                         port_number=port_number,
+                                                                                                                         nio=nio))
+        except DynamipsError:
+            # in case of error try to remove and add the nio binding
+            yield from self._hypervisor.send('vm slot_remove_nio_binding "{name}" {slot_number} {port_number}'.format(name=self._name,
+                                                                                                                      slot_number=slot_number,
+                                                                                                                      port_number=port_number))
+            yield from self._hypervisor.send('vm slot_add_nio_binding "{name}" {slot_number} {port_number} {nio}'.format(name=self._name,
+                                                                                                                         slot_number=slot_number,
+                                                                                                                         port_number=port_number,
+                                                                                                                         nio=nio))
 
         log.info('Router "{name}" [{id}]: NIO {nio_name} bound to port {slot_number}/{port_number}'.format(name=self._name,
                                                                                                            id=self._id,
@@ -1499,6 +1560,9 @@ class Router(BaseVM):
             module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
             startup_config_base64, private_config_base64 = yield from self.extract_config()
             if startup_config_base64:
+                if not self.startup_config:
+                    self._startup_config = os.path.join("configs", "i{}_startup-config.cfg".format(self._dynamips_id))
+
                 try:
                     config = base64.b64decode(startup_config_base64).decode("utf-8", errors="replace")
                     config = "!\n" + config.replace("\r", "")
@@ -1510,6 +1574,9 @@ class Router(BaseVM):
                     raise DynamipsError("Could not save the startup configuration {}: {}".format(config_path, e))
 
             if private_config_base64:
+                if not self.private_config:
+                    self._private_config = os.path.join("configs", "i{}_private-config.cfg".format(self._dynamips_id))
+
                 try:
                     config = base64.b64decode(private_config_base64).decode("utf-8", errors="replace")
                     config = "!\n" + config.replace("\r", "")

gns3server/modules/iou/iou_vm.py

@@ -862,7 +862,10 @@ class IOUVM(BaseVM):
                                                                                              port_number=port_number))
         if self.is_iouyap_running():
             self._update_iouyap_config()
-            os.kill(self._iouyap_process.pid, signal.SIGHUP)
+            try:
+                os.kill(self._iouyap_process.pid, signal.SIGHUP)
+            except ProcessLookupError:
+                log.error("Could not update iouyap configuration: process (PID={}) not found".format(self._iouyap_process.pid))
 
     def adapter_remove_nio_binding(self, adapter_number, port_number):
         """
@@ -894,8 +897,10 @@ class IOUVM(BaseVM):
                                                                                                  port_number=port_number))
         if self.is_iouyap_running():
             self._update_iouyap_config()
-            os.kill(self._iouyap_process.pid, signal.SIGHUP)
-
+            try:
+                os.kill(self._iouyap_process.pid, signal.SIGHUP)
+            except ProcessLookupError:
+                log.error("Could not update iouyap configuration: process (PID={}) not found".format(self._iouyap_process.pid))
         return nio
 
     @property
@@ -1052,7 +1057,10 @@ class IOUVM(BaseVM):
 
         if self.is_iouyap_running():
             self._update_iouyap_config()
-            os.kill(self._iouyap_process.pid, signal.SIGHUP)
+            try:
+                os.kill(self._iouyap_process.pid, signal.SIGHUP)
+            except ProcessLookupError:
+                log.error("Could not update iouyap configuration: process (PID={}) not found".format(self._iouyap_process.pid))
 
     @asyncio.coroutine
     def stop_capture(self, adapter_number, port_number):
@@ -1085,4 +1093,7 @@ class IOUVM(BaseVM):
                                                                                                          port_number=port_number))
         if self.is_iouyap_running():
             self._update_iouyap_config()
-            os.kill(self._iouyap_process.pid, signal.SIGHUP)
+            try:
+                os.kill(self._iouyap_process.pid, signal.SIGHUP)
+            except ProcessLookupError:
+                log.error("Could not update iouyap configuration: process (PID={}) not found".format(self._iouyap_process.pid))

gns3server/modules/port_manager.py

@@ -34,8 +34,9 @@ class PortManager:
     def __init__(self, host="127.0.0.1"):
 
         self._console_host = host
-        self._udp_host = host
 
+        # UDP host must be 0.0.0.0, reason: https://github.com/GNS3/gns3-server/issues/265
+        self._udp_host = "0.0.0.0"
         self._used_tcp_ports = set()
         self._used_udp_ports = set()
 

gns3server/modules/project.py

@@ -363,7 +363,7 @@ class Project:
             for future in done:
                 try:
                     future.result()
-                except Exception as e:
+                except (Exception, GeneratorExit) as e:
                     log.error("Could not close VM or device {}".format(e), exc_info=1)
 
         if cleanup and os.path.exists(self.path):

gns3server/modules/virtualbox/__init__.py

@@ -97,7 +97,7 @@ class VirtualBox(BaseManager):
                 vbox_user = self.config.get_section_config("VirtualBox").get("vbox_user")
                 if vbox_user:
                     # TODO: test & review this part
-                    sudo_command = "sudo -i -u {}".format(vbox_user) + " ".join(command)
+                    sudo_command = "sudo -i -u {} ".format(vbox_user) + " ".join(command)
                     process = yield from asyncio.create_subprocess_shell(sudo_command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
                 else:
                     process = yield from asyncio.create_subprocess_exec(*command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
@@ -168,7 +168,7 @@ class VirtualBox(BaseManager):
         vms = []
         result = yield from self.execute("list", ["vms"])
         for line in result:
-            if line[0] != '"' or line[-1:] != "}":
+            if len(line) == 0 or line[0] != '"' or line[-1:] != "}":
                 continue # Broken output (perhaps a carriage return in VM name
             vmname, _ = line.rsplit(' ', 1)
             vmname = vmname.strip('"')

gns3server/modules/virtualbox/virtualbox_vm.py

@@ -574,10 +574,11 @@ class VirtualBoxVM(BaseVM):
 
         # check the maximum number of adapters supported by the VM
         vm_info = yield from self._get_vm_info()
-        chipset = vm_info["chipset"]
         maximum_adapters = 8
-        if chipset == "ich9":
-            maximum_adapters = int(self._system_properties["Maximum ICH9 Network Adapter count"])
+        if "chipset" in vm_info:
+            chipset = vm_info["chipset"]
+            if chipset == "ich9":
+                maximum_adapters = int(self._system_properties["Maximum ICH9 Network Adapter count"])
         return maximum_adapters
 
     def _get_pipe_name(self):

gns3server/schemas/dynamips_vm.py

@@ -120,6 +120,10 @@ VM_CREATE_SCHEMA = {
             "description": "disk1 size in MB",
             "type": "integer"
         },
+        "auto_delete_disks": {
+            "description": "automatically delete nvram and disk files",
+            "type": "boolean"
+        },
         "console": {
             "description": "console TCP port",
             "type": "integer",
@@ -337,6 +341,10 @@ VM_UPDATE_SCHEMA = {
             "description": "disk1 size in MB",
             "type": "integer"
         },
+        "auto_delete_disks": {
+            "description": "automatically delete nvram and disk files",
+            "type": "boolean"
+        },
         "console": {
             "description": "console TCP port",
             "type": "integer",
@@ -591,6 +599,10 @@ VM_OBJECT_SCHEMA = {
             "description": "disk1 size in MB",
             "type": "integer"
         },
+        "auto_delete_disks": {
+            "description": "automatically delete nvram and disk files",
+            "type": "boolean"
+        },
         "console": {
             "description": "console TCP port",
             "type": "integer",

gns3server/server.py

@@ -86,6 +86,7 @@ class Server:
 
         if self._handler:
             yield from self._handler.finish_connections()
+            self._handler = None
 
         for module in MODULES:
             log.debug("Unloading module {}".format(module.__name__))
@@ -175,6 +176,10 @@ class Server:
         yield from embed(globals(), locals(), return_asyncio_coroutine=True, patch_stdout=True)
 
     def _exit_handling(self):
+        """
+        Makes sure the asyncio loop is closed.
+        """
+
         def close_asyncio_loop():
             loop = None
             try:
@@ -226,7 +231,8 @@ class Server:
 
         log.info("Starting server on {}:{}".format(self._host, self._port))
         self._handler = app.make_handler(handler=RequestHandler)
-        self._loop.run_until_complete(self._run_application(self._handler, ssl_context))
+        server = self._run_application(self._handler, ssl_context)
+        self._loop.run_until_complete(server)
         self._signal_handling()
 
         self._exit_handling()
@@ -245,3 +251,9 @@ class Server:
             # on Windows when the process gets the SIGBREAK signal
             # TypeError: async() takes 1 positional argument but 3 were given
             log.warning("TypeError exception in the loop {}".format(e))
+        finally:
+            if self._handler and self._loop.is_running():
+                self._loop.run_until_complete(self._handler.finish_connections())
+            server.close()
+            if self._loop.is_running():
+                self._loop.run_until_complete(app.finish())

gns3server/templates/index.html

@@ -6,6 +6,6 @@
 <ul>
     <li><a href="http://community.gns3.com">Community</a></li>
     <li><a href="http://api.gns3.net">API documentation</a></li>
-    <li><a href="/upload">Upload images</a></li>
+    <li><a href="/upload">Upload images & backup</a></li>
 </ul>
 {% endblock %}

gns3server/templates/layout.html

@@ -4,6 +4,15 @@
 <title>GNS3 Server</title>
 </head>
 <body>
+<div>
+    <a href="/">Home</a>
+    |
+    <a href="/upload">Upload</a>
+    |    
+    <a href="/backup/images.tar">Backup images</a>
+    |
+    <a href="/backup/projects.tar">Backup projects</a>
+</div>
 {% block body %}{% endblock %}
 </body>
 <small>

gns3server/templates/upload.html

@@ -8,6 +8,8 @@
         <option value="IOURC">IOU licence (iourc)</option>
         <option value="IOS">IOS</option>
         <option value="QEMU">Qemu</option>
+        <option value="IMAGES">GNS3 images backup (.tar)</option>
+        <option value="PROJECTS">GNS3 projects backup (.tar)</option>
     </select>
     <br />
     <br />

gns3server/utils/asyncio.py

@@ -51,7 +51,10 @@ def subprocess_check_output(*args, cwd=None, env=None):
     output = yield from proc.stdout.read()
     if output is None:
         return ""
-    return output.decode("utf-8")
+    # If we received garbage we ignore invalid characters
+    # it should happend only when user try to use another binary
+    # and the code of VPCS, dynamips... Will detect it's not the correct binary
+    return output.decode("utf-8", errors="ignore")
 
 
 @asyncio.coroutine

gns3server/version.py

@@ -23,5 +23,5 @@
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
 
-__version__ = "1.3.4"
-__version_info__ = (1, 3, 4, 0)
+__version__ = "1.3.8"
+__version_info__ = (1, 3, 8, 0)

gns3server/web/logger.py

@@ -34,7 +34,7 @@ class ColouredFormatter(logging.Formatter):
 
         message = super().format(record)
 
-        if not colour:
+        if not colour or sys.platform.startswith("win"):
             return message.replace("#RESET#", "")
 
         level_no = record.levelno
@@ -90,7 +90,7 @@ class WinStreamHandler(logging.StreamHandler):
 
         stream = self.stream
         try:
-            msg =  self.formatter.format(record, stream.isatty())
+            msg = self.formatter.format(record, stream.isatty())
             stream.write(msg.encode(stream.encoding, errors="replace").decode(stream.encoding))
             stream.write(self.terminator)
             self.flush()

gns3server/web/route.py

@@ -83,6 +83,31 @@ class Route(object):
     def delete(cls, path, *args, **kw):
         return cls._route('DELETE', path, *args, **kw)
 
+    @classmethod
+    def authenticate(cls, request, route, server_config):
+        """
+        Ask user for authentication
+
+        :returns: Response if you need to auth the user otherwise None
+        """
+        if not server_config.getboolean("auth", False):
+            return
+
+        user = server_config.get("user", "").strip()
+        password = server_config.get("password", "").strip()
+
+        if len(user) == 0:
+            return
+
+        if "AUTHORIZATION" in request.headers:
+            if request.headers["AUTHORIZATION"] == aiohttp.helpers.BasicAuth(user, password).encode():
+                return
+
+        response = Response(request=request, route=route)
+        response.set_status(401)
+        response.headers["WWW-Authenticate"] = 'Basic realm="GNS3 server"'
+        return response
+
     @classmethod
     def _route(cls, method, path, *args, **kw):
         # This block is executed only the first time
@@ -118,6 +143,13 @@ class Route(object):
             def control_schema(request):
                 # This block is executed at each method call
 
+                server_config = Config.instance().get_section_config("Server")
+
+                # Authenticate
+                response = cls.authenticate(request, route, server_config)
+                if response:
+                    return response
+
                 # Non API call
                 if api_version is None:
                     response = Response(request=request, route=route, output_schema=output_schema)
@@ -127,7 +159,6 @@ class Route(object):
                 # API call
                 try:
                     request = yield from parse_request(request, input_schema)
-                    server_config = Config.instance().get_section_config("Server")
                     record_file = server_config.get("record")
                     if record_file:
                         try:

init/gns3.conf.upstart

@@ -6,7 +6,7 @@ stop on shutdown
 
 script
     echo $$ > /var/run/gns3.pid
-    exec start-stop-daemon --start -c gns3 --exec /usr/local/bin/gns3server --log /var/log/gns3.log
+    exec start-stop-daemon --start -c gns3 --exec /usr/local/bin/gns3server -- --log /var/log/gns3.log
 end script
 
 pre-start script

requirements.txt

@@ -1,4 +1,4 @@
-netifaces==0.10.4
+gns3-netifaces==0.10.4.1
 jsonschema==2.4.0
 aiohttp==0.14.4
 Jinja2==2.7.3

setup.py

@@ -19,6 +19,10 @@ import sys
 from setuptools import setup, find_packages
 from setuptools.command.test import test as TestCommand
 
+# we only support Python 3 version >= 3.3
+if sys.version_info < (3, 3):
+    raise SystemExit("Python 3.3 or higher is required")
+
 
 class PyTest(TestCommand):
 
@@ -34,12 +38,14 @@ class PyTest(TestCommand):
         sys.exit(errcode)
 
 
-dependencies = ["aiohttp>=0.14.4",
-                "jsonschema>=2.4.0",
-                "Jinja2>=2.7.3",
-                "raven>=5.2.0"]
+dependencies = [
+    # "gns3-netifaces>=0.10.4.1",
+    "aiohttp>=0.14.4",
+    "jsonschema>=2.4.0",
+    "Jinja2>=2.7.3",
+    "raven>=5.2.0"]
 
-#if not sys.platform.startswith("win"):
+# if not sys.platform.startswith("win"):
 #    dependencies.append("netifaces==0.10.4")
 
 if sys.version_info == (3, 3):

tests/conftest.py

@@ -136,6 +136,7 @@ def run_around_tests(monkeypatch):
     config = Config.instance()
     config.clear()
     config.set("Server", "project_directory", tmppath)
+    config.set("Server", "auth", False)
 
     # Prevent exectuions of the VM if we forgot to mock something
     config.set("VirtualBox", "vboxmanage_path", tmppath)

tests/handlers/api/base.py

@@ -88,7 +88,10 @@ class Query:
                 except ValueError:
                     response.json = None
             else:
-                response.html = response.body.decode("utf-8")
+                try:
+                    response.html = response.body.decode("utf-8")
+                except UnicodeDecodeError:
+                    response.html = None
         else:
             response.json = {}
             response.html = ""

tests/handlers/test_upload.py

@@ -17,10 +17,15 @@
 
 
 import aiohttp
+import asyncio
 import os
+import tarfile
 from unittest.mock import patch
 
 
+from gns3server.config import Config
+
+
 def test_index_upload(server):
     response = server.get('/upload', api_version=None)
     assert response.status == 200
@@ -37,10 +42,141 @@ def test_upload(server, tmpdir):
     body.add_field("type", "QEMU")
     body.add_field("file", open(str(tmpdir / "test"), "rb"), content_type="application/iou", filename="test2")
 
-    with patch("gns3server.config.Config.get_section_config", return_value={"images_path": str(tmpdir)}):
-        response = server.post('/upload', api_version=None, body=body, raw=True)
+    Config.instance().set("Server", "images_path", str(tmpdir))
+    response = server.post('/upload', api_version=None, body=body, raw=True)
 
     with open(str(tmpdir / "QEMU" / "test2")) as f:
         assert f.read() == "TEST"
 
     assert "test2" in response.body.decode("utf-8")
+
+
+def test_upload_images_backup(server, tmpdir):
+    Config.instance().set("Server", "images_path", str(tmpdir / 'images'))
+    os.makedirs(str(tmpdir / 'images' / 'IOU'))
+    # An old IOU image that we need to replace
+    with open(str(tmpdir / 'images' / 'IOU' / 'b.img'), 'w+') as f:
+        f.write('bad')
+
+    os.makedirs(str(tmpdir / 'old' / 'QEMU'))
+    with open(str(tmpdir / 'old' / 'QEMU' / 'a.img'), 'w+') as f:
+        f.write('hello')
+    os.makedirs(str(tmpdir / 'old' / 'IOU'))
+    with open(str(tmpdir / 'old' / 'IOU' / 'b.img'), 'w+') as f:
+        f.write('world')
+
+    os.chdir(str(tmpdir / 'old'))
+    with tarfile.open(str(tmpdir / 'test.tar'), 'w') as tar:
+        tar.add('.', recursive=True)
+
+    body = aiohttp.FormData()
+    body.add_field('type', 'IMAGES')
+    body.add_field('file', open(str(tmpdir / 'test.tar'), 'rb'), content_type='application/x-gtar', filename='test.tar')
+    response = server.post('/upload', api_version=None, body=body, raw=True)
+    assert response.status == 200
+
+    with open(str(tmpdir / 'images' / 'QEMU' / 'a.img')) as f:
+        assert f.read() == 'hello'
+    with open(str(tmpdir / 'images' / 'IOU' / 'b.img')) as f:
+        assert f.read() == 'world'
+
+    assert 'a.img' in response.body.decode('utf-8')
+    assert 'b.img' in response.body.decode('utf-8')
+    assert not os.path.exists(str(tmpdir / 'images' / 'archive.tar'))
+
+
+def test_upload_projects_backup(server, tmpdir):
+    Config.instance().set("Server", "projects_path", str(tmpdir / 'projects'))
+    os.makedirs(str(tmpdir / 'projects' / 'b'))
+    # An old b image that we need to replace
+    with open(str(tmpdir / 'projects' / 'b' / 'b.img'), 'w+') as f:
+        f.write('bad')
+
+    os.makedirs(str(tmpdir / 'old' / 'a'))
+    with open(str(tmpdir / 'old' / 'a' / 'a.img'), 'w+') as f:
+        f.write('hello')
+    os.makedirs(str(tmpdir / 'old' / 'b'))
+    with open(str(tmpdir / 'old' / 'b' / 'b.img'), 'w+') as f:
+        f.write('world')
+
+    os.chdir(str(tmpdir / 'old'))
+    with tarfile.open(str(tmpdir / 'test.tar'), 'w') as tar:
+        tar.add('.', recursive=True)
+
+    body = aiohttp.FormData()
+    body.add_field('type', 'PROJECTS')
+    body.add_field('file', open(str(tmpdir / 'test.tar'), 'rb'), content_type='application/x-gtar', filename='test.tar')
+    response = server.post('/upload', api_version=None, body=body, raw=True)
+    assert response.status == 200
+
+    with open(str(tmpdir / 'projects' / 'a' / 'a.img')) as f:
+        assert f.read() == 'hello'
+    with open(str(tmpdir / 'projects' / 'b' / 'b.img')) as f:
+        assert f.read() == 'world'
+
+    assert 'a.img' not in response.body.decode('utf-8')
+    assert 'b.img' not in response.body.decode('utf-8')
+    assert not os.path.exists(str(tmpdir / 'projects' / 'archive.tar'))
+
+
+def test_backup_images(server, tmpdir, loop):
+    Config.instance().set('Server', 'images_path', str(tmpdir))
+
+    os.makedirs(str(tmpdir / 'QEMU'))
+    with open(str(tmpdir / 'QEMU' / 'a.img'), 'w+') as f:
+        f.write('hello')
+    with open(str(tmpdir / 'QEMU' / 'b.img'), 'w+') as f:
+        f.write('world')
+
+    response = server.get('/backup/images.tar', api_version=None, raw=True)
+    assert response.status == 200
+    assert response.headers['CONTENT-TYPE'] == 'application/x-gtar'
+
+    with open(str(tmpdir / 'images.tar'), 'wb+') as f:
+        print(len(response.body))
+        f.write(response.body)
+
+    tar = tarfile.open(str(tmpdir / 'images.tar'), 'r')
+    os.makedirs(str(tmpdir / 'extract'))
+    os.chdir(str(tmpdir / 'extract'))
+    # Extract to current working directory
+    tar.extractall()
+    tar.close()
+
+    assert os.path.exists(os.path.join('QEMU', 'a.img'))
+    open(os.path.join('QEMU', 'a.img')).read() == 'hello'
+
+    assert os.path.exists(os.path.join('QEMU', 'b.img'))
+    open(os.path.join('QEMU', 'b.img')).read() == 'world'
+
+
+def test_backup_projects(server, tmpdir, loop):
+    Config.instance().set('Server', 'projects_path', str(tmpdir))
+
+    os.makedirs(str(tmpdir / 'a'))
+    with open(str(tmpdir / 'a' / 'a.gns3'), 'w+') as f:
+        f.write('hello')
+    os.makedirs(str(tmpdir / 'b'))
+    with open(str(tmpdir / 'b' / 'b.gns3'), 'w+') as f:
+        f.write('world')
+
+    response = server.get('/backup/projects.tar', api_version=None, raw=True)
+    assert response.status == 200
+    assert response.headers['CONTENT-TYPE'] == 'application/x-gtar'
+
+    with open(str(tmpdir / 'projects.tar'), 'wb+') as f:
+        print(len(response.body))
+        f.write(response.body)
+
+    tar = tarfile.open(str(tmpdir / 'projects.tar'), 'r')
+    os.makedirs(str(tmpdir / 'extract'))
+    os.chdir(str(tmpdir / 'extract'))
+    # Extract to current working directory
+    tar.extractall()
+    tar.close()
+
+    assert os.path.exists(os.path.join('a', 'a.gns3'))
+    open(os.path.join('a', 'a.gns3')).read() == 'hello'
+
+    assert os.path.exists(os.path.join('b', 'b.gns3'))
+    open(os.path.join('b', 'b.gns3')).read() == 'world'

tests/modules/virtualbox/test_virtualbox_manager.py

@@ -75,9 +75,9 @@ def test_get_list(manager, loop):
     vm_list = ['"Windows 8.1" {27b4d095-ff5f-4ac4-bb9d-5f2c7861c1f1}',
                '"Carriage',
                'Return" {27b4d095-ff5f-4ac4-bb9d-5f2c7861c1f1}',
+               '',
                '"<inaccessible>" {42b4d095-ff5f-4ac4-bb9d-5f2c7861c1f1}',
-               '"Linux Microcore 4.7.1" {ccd8c50b-c172-457d-99fa-dd69371ede0e}'
-              ]
+               '"Linux Microcore 4.7.1" {ccd8c50b-c172-457d-99fa-dd69371ede0e}']
 
     @asyncio.coroutine
     def execute_mock(cmd, args):