Merge branch '1.5' into 2.0

CHANGELOG

@@ -1,5 +1,16 @@
 # Change Log
 
+## 1.4.6 28/04/2016
+
+* More robust save/restore for VirtualBox linked clone VM hard disks.
+* Prevent non linked cloned hard disks to be detached when using VirtualBox linked cloned VMs. Fixes #1184.
+* Stricter checks to match VMware version to the right vmrun (VIX library) version. Also checks the VIX library version when only using the GNS3 VM running in VMware.
+* Allow only .pcap to be downloaded from remote stream API
+* Fix incrementation of qemu mac address
+* Clear warnings about using linked clones with VMware Player.
+* Alternative method to find the Documents folder on Windows.
+* Add IOU support and install config in /etc
+
 ## 1.4.5 23/03/2016
 
 * Stop the VMware VM if there is an error while setting up the network connections or console.

gns3server/compute/docker/__init__.py

@@ -49,7 +49,7 @@ class Docker(BaseManager):
 
     @asyncio.coroutine
     def connector(self):
-        if not self._connected:
+        if not self._connected or self._connector.closed:
             try:
                 self._connector = aiohttp.connector.UnixConnector(self._server_url)
                 self._connected = True

gns3server/compute/qemu/qemu_vm.py

@@ -1261,6 +1261,8 @@ class QemuVM(BaseVM):
                                                                             "backing_file={}".format(disk_image),
                                                                             "-f", "qcow2", disk)
                         retcode = yield from process.wait()
+                        if retcode is not None and retcode != 0:
+                            raise QemuError("Could not create {} disk image".format(disk_name))
                         log.info("{} returned with {}".format(qemu_img_path, retcode))
                     except (OSError, subprocess.SubprocessError) as e:
                         raise QemuError("Could not create {} disk image {}".format(disk_name, e))

gns3server/compute/virtualbox/virtualbox_vm.py

@@ -158,7 +158,7 @@ class VirtualBoxVM(BaseVM):
             if self.id and os.path.isdir(os.path.join(self.working_dir, self._vmname)):
                 vbox_file = os.path.join(self.working_dir, self._vmname, self._vmname + ".vbox")
                 yield from self.manager.execute("registervm", [vbox_file])
-                yield from self._reattach_hdds()
+                yield from self._reattach_linked_hdds()
             else:
                 yield from self._create_linked_clone()
 
@@ -313,7 +313,10 @@ class VirtualBoxVM(BaseVM):
         return hdds
 
     @asyncio.coroutine
-    def _reattach_hdds(self):
+    def _reattach_linked_hdds(self):
+        """
+        Reattach linked cloned hard disks.
+        """
 
         hdd_info_file = os.path.join(self.working_dir, self._vmname, "hdd_info.json")
         try:
@@ -332,20 +335,86 @@ class VirtualBoxVM(BaseVM):
                                                                                                                     device=hdd_info["device"],
                                                                                                                     medium=hdd_file))
 
+                try:
                     yield from self._storage_attach('--storagectl "{}" --port {} --device {} --type hdd --medium "{}"'.format(hdd_info["controller"],
                                                                                                                               hdd_info["port"],
                                                                                                                               hdd_info["device"],
                                                                                                                               hdd_file))
 
+                except VirtualBoxError as e:
+                    log.warn("VirtualBox VM '{name}' [{id}] error reattaching HDD {controller} {port} {device} {medium}: {error}".format(name=self.name,
+                                                                                                                                         id=self.id,
+                                                                                                                                         controller=hdd_info["controller"],
+                                                                                                                                         port=hdd_info["port"],
+                                                                                                                                         device=hdd_info["device"],
+                                                                                                                                         medium=hdd_file,
+                                                                                                                                         error=e))
+                    continue
+
+    @asyncio.coroutine
+    def save_linked_hdds_info(self):
+        """
+        Save linked cloned hard disks information.
+
+        :returns: disk table information
+        """
+
+        hdd_table = []
+        if self._linked_clone:
+            if os.path.exists(self.working_dir):
+                hdd_files = yield from self._get_all_hdd_files()
+                vm_info = yield from self._get_vm_info()
+                for entry, value in vm_info.items():
+                    match = re.search("^([\s\w]+)\-(\d)\-(\d)$", entry)  # match Controller-PortNumber-DeviceNumber entry
+                    if match:
+                        controller = match.group(1)
+                        port = match.group(2)
+                        device = match.group(3)
+                        if value in hdd_files and os.path.exists(os.path.join(self.working_dir, self._vmname, "Snapshots", os.path.basename(value))):
+                            log.info("VirtualBox VM '{name}' [{id}] detaching HDD {controller} {port} {device}".format(name=self.name,
+                                                                                                                       id=self.id,
+                                                                                                                       controller=controller,
+                                                                                                                       port=port,
+                                                                                                                       device=device))
+                            hdd_table.append(
+                                {
+                                    "hdd": os.path.basename(value),
+                                    "controller": controller,
+                                    "port": port,
+                                    "device": device,
+                                }
+                            )
+
+            if hdd_table:
+                try:
+                    hdd_info_file = os.path.join(self.working_dir, self._vmname, "hdd_info.json")
+                    with open(hdd_info_file, "w", encoding="utf-8") as f:
+                        json.dump(hdd_table, f, indent=4)
+                except OSError as e:
+                    log.warning("VirtualBox VM '{name}' [{id}] could not write HHD info file: {error}".format(name=self.name,
+                                                                                                              id=self.id,
+                                                                                                              error=e.strerror))
+
+        return hdd_table
+
     @asyncio.coroutine
     def close(self):
         """
         Closes this VirtualBox VM.
         """
 
+        if self._closed:
+            # VM is already closed
+            return
+
         if not (yield from super().close()):
             return False
 
+        log.debug("VirtualBox VM '{name}' [{id}] is closing".format(name=self.name, id=self.id))
+        if self._console:
+            self._manager.port_manager.release_tcp_port(self._console, self._project)
+            self._console = None
+
         for adapter in self._ethernet_adapters.values():
             if adapter is not None:
                 for nio in adapter.ports.values():
@@ -356,46 +425,31 @@ class VirtualBoxVM(BaseVM):
         yield from self.stop()
 
         if self._linked_clone:
-            hdd_table = []
+            hdd_table = yield from self.save_linked_hdds_info()
-            if os.path.exists(self.working_dir):
+            for hdd in hdd_table.copy():
-                hdd_files = yield from self._get_all_hdd_files()
-                vm_info = yield from self._get_vm_info()
-                for entry, value in vm_info.items():
-                    match = re.search("^([\s\w]+)\-(\d)\-(\d)$", entry)
-                    if match:
-                        controller = match.group(1)
-                        port = match.group(2)
-                        device = match.group(3)
-                        if value in hdd_files:
                 log.info("VirtualBox VM '{name}' [{id}] detaching HDD {controller} {port} {device}".format(name=self.name,
                                                                                                            id=self.id,
-                                                                                                                       controller=controller,
+                                                                                                           controller=hdd["controller"],
-                                                                                                                       port=port,
+                                                                                                           port=hdd["port"],
-                                                                                                                       device=device))
+                                                                                                           device=hdd["device"]))
-                            yield from self._storage_attach('--storagectl "{}" --port {} --device {} --type hdd --medium none'.format(controller,
+                try:
-                                                                                                                                      port,
+                    yield from self._storage_attach('--storagectl "{}" --port {} --device {} --type hdd --medium none'.format(hdd["controller"],
-                                                                                                                                      device))
+                                                                                                                              hdd["port"],
-                            hdd_table.append(
+                                                                                                                              hdd["device"]))
-                                {
+                except VirtualBoxError as e:
-                                    "hdd": os.path.basename(value),
+                    log.warn("VirtualBox VM '{name}' [{id}] error detaching HDD {controller} {port} {device}: {error}".format(name=self.name,
-                                    "controller": controller,
+                                                                                                                              id=self.id,
-                                    "port": port,
+                                                                                                                              controller=hdd["controller"],
-                                    "device": device,
+                                                                                                                              port=hdd["port"],
-                                }
+                                                                                                                              device=hdd["device"],
-                            )
+                                                                                                                              error=e))
+                    continue
 
             log.info("VirtualBox VM '{name}' [{id}] unregistering".format(name=self.name, id=self.id))
             yield from self.manager.execute("unregistervm", [self._name])
 
-            if hdd_table:
+        log.info("VirtualBox VM '{name}' [{id}] closed".format(name=self.name, id=self.id))
-                try:
+        self._closed = True
-                    hdd_info_file = os.path.join(self.working_dir, self._vmname, "hdd_info.json")
-                    with open(hdd_info_file, "w", encoding="utf-8") as f:
-                        json.dump(hdd_table, f, indent=4)
-                except OSError as e:
-                    log.warning("VirtualBox VM '{name}' [{id}] could not write HHD info file: {error}".format(name=self.name,
-                                                                                                              id=self.id,
-                                                                                                              error=e.strerror))
 
     @property
     def headless(self):

gns3server/compute/vmware/__init__.py

@@ -108,7 +108,7 @@ class VMware(BaseManager):
                 vmrun_path = shutil.which("vmrun")
 
         if not vmrun_path:
-            raise VMwareError("Could not find vmrun")
+            raise VMwareError("Could not find VMware vmrun, please make sure it is installed")
         if not os.path.isfile(vmrun_path):
             raise VMwareError("vmrun {} is not accessible".format(vmrun_path))
         if not os.access(vmrun_path, os.X_OK):
@@ -137,6 +137,50 @@ class VMware(BaseManager):
                 version = match.group(1)
         return version
 
+    @asyncio.coroutine
+    def _check_vmware_player_requirements(self, player_version):
+        """
+        Check minimum requirements to use VMware Player.
+
+        VIX 1.13 was the release for Player 6.
+        VIX 1.14 was the release for Player 7.
+        VIX 1.15 was the release for Workstation Player 12.
+
+        :param player_version: VMware Player major version.
+        """
+
+        player_version = int(player_version)
+        if player_version < 6:
+            raise VMwareError("Using VMware Player requires version 6 or above")
+        elif player_version == 6:
+            yield from self.check_vmrun_version(minimum_required_version="1.13")
+        elif player_version == 7:
+            yield from self.check_vmrun_version(minimum_required_version="1.14")
+        elif player_version >= 12:
+            yield from self.check_vmrun_version(minimum_required_version="1.15")
+
+    @asyncio.coroutine
+    def _check_vmware_workstation_requirements(self, ws_version):
+        """
+        Check minimum requirements to use VMware Workstation.
+
+        VIX 1.13 was the release for Workstation 10.
+        VIX 1.14 was the release for Workstation 11.
+        VIX 1.15 was the release for Workstation Pro 12.
+
+        :param ws_version: VMware Workstation major version.
+        """
+
+        ws_version = int(ws_version)
+        if ws_version < 10:
+            raise VMwareError("Using VMware Workstation requires version 10 or above")
+        elif ws_version == 10:
+            yield from self.check_vmrun_version(minimum_required_version="1.13")
+        elif ws_version == 11:
+            yield from self.check_vmrun_version(minimum_required_version="1.14")
+        elif ws_version >= 12:
+            yield from self.check_vmrun_version(minimum_required_version="1.15")
+
     @asyncio.coroutine
     def check_vmware_version(self):
         """
@@ -150,15 +194,12 @@ class VMware(BaseManager):
                 player_version = self._find_vmware_version_registry(r"SOFTWARE\Wow6432Node\VMware, Inc.\VMware Player")
                 if player_version:
                     log.debug("VMware Player version {} detected".format(player_version))
-                    if int(player_version) < 6:
+                    yield from self._check_vmware_player_requirements(player_version)
-                        raise VMwareError("Using VMware Player requires version 6 or above")
                 else:
                     log.warning("Could not find VMware version")
             else:
                 log.debug("VMware Workstation version {} detected".format(ws_version))
-                if int(ws_version) < 10:
+                yield from self._check_vmware_workstation_requirements(ws_version)
-                    raise VMwareError("Using VMware Workstation requires version 10 or above")
-            return
         else:
             if sys.platform.startswith("darwin"):
                 if not os.path.isdir("/Applications/VMware Fusion.app"):
@@ -174,16 +215,16 @@ class VMware(BaseManager):
                 match = re.search("VMware Workstation ([0-9]+)\.", output)
                 version = None
                 if match:
+                    # VMware Workstation has been detected
                     version = match.group(1)
                     log.debug("VMware Workstation version {} detected".format(version))
-                    if int(version) < 10:
+                    yield from self._check_vmware_workstation_requirements(version)
-                        raise VMwareError("Using VMware Workstation requires version 10 or above")
                 match = re.search("VMware Player ([0-9]+)\.", output)
                 if match:
+                    # VMware Player has been detected
                     version = match.group(1)
                     log.debug("VMware Player version {} detected".format(version))
-                    if int(version) < 6:
+                    yield from self._check_vmware_player_requirements(version)
-                        raise VMwareError("Using VMware Player requires version 6 or above")
                 if version is None:
                     log.warning("Could not find VMware version. Output of VMware: {}".format(output))
                     raise VMwareError("Could not find VMware version. Output of VMware: {}".format(output))
@@ -352,7 +393,17 @@ class VMware(BaseManager):
             return stdout_data.decode("utf-8", errors="ignore").splitlines()
 
     @asyncio.coroutine
-    def check_vmrun_version(self):
+    def check_vmrun_version(self, minimum_required_version="1.13"):
+        """
+        Checks the vmrun version.
+
+        VMware VIX library version must be at least >= 1.13 by default
+        VIX 1.13 was the release for VMware Fusion 6, Workstation 10, and Player 6.
+        VIX 1.14 was the release for VMware Fusion 7, Workstation 11 and Player 7.
+        VIX 1.15 was the release for VMware Fusion 8, Workstation Pro 12 and Workstation Player 12.
+
+        :param required_version: required vmrun version number
+        """
 
         with (yield from self._execute_lock):
             vmrun_path = self.vmrun_path
@@ -366,9 +417,9 @@ class VMware(BaseManager):
                 if match:
                     version = match.group(1)
                     log.debug("VMware vmrun version {} detected".format(version))
-                    if parse_version(version) < parse_version("1.13"):
+                    if parse_version(version) < parse_version(minimum_required_version):
-                        # VMware VIX library version must be at least >= 1.13
+
-                        raise VMwareError("VMware vmrun executable version must be >= version 1.13")
+                        raise VMwareError("VMware vmrun executable version must be >= version {}".format(minimum_required_version))
                 if version is None:
                     log.warning("Could not find VMware vmrun version. Output: {}".format(output))
                     raise VMwareError("Could not find VMware vmrun version. Output: {}".format(output))
@@ -595,6 +646,7 @@ class VMware(BaseManager):
 
         if sys.platform.startswith("win"):
             import ctypes
+            import ctypes.wintypes
             path = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
             ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, path)
             documents_folder = path.value

gns3server/handlers/api/compute/project_handler.py

@@ -405,8 +405,8 @@ class ProjectHandler:
 
         pm = ProjectManager.instance()
         project = pm.get_project(request.match_info["project_id"])
-        response.content_type = 'application/gns3z'
+        response.content_type = 'application/gns3project'
-        response.headers['CONTENT-DISPOSITION'] = 'attachment; filename="{}.gns3z"'.format(project.name)
+        response.headers['CONTENT-DISPOSITION'] = 'attachment; filename="{}.gns3project"'.format(project.name)
         response.enable_chunked_encoding()
         # Very important: do not send a content length otherwise QT close the connection but curl can consume the Feed
         response.content_length = None

setup.py

@@ -40,9 +40,6 @@ class PyTest(TestCommand):
 
 dependencies = open("requirements.txt", "r").read().splitlines()
 
-if sys.platform.startswith("win"):
-    dependencies.append("pywin32>=219")
-
 setup(
     name="gns3-server",
     version=__import__("gns3server").__version__,

tests/handlers/api/compute/test_project.py

@@ -245,8 +245,8 @@ def test_export(http_compute, tmpdir, loop, project):
 
     response = http_compute.get("/projects/{project_id}/export".format(project_id=project.id), raw=True)
     assert response.status == 200
-    assert response.headers['CONTENT-TYPE'] == 'application/gns3z'
+    assert response.headers['CONTENT-TYPE'] == 'application/gns3project'
-    assert response.headers['CONTENT-DISPOSITION'] == 'attachment; filename="{}.gns3z"'.format(project.name)
+    assert response.headers['CONTENT-DISPOSITION'] == 'attachment; filename="{}.gns3project"'.format(project.name)
 
     with open(str(tmpdir / 'project.zip'), 'wb+') as f:
         f.write(response.body)