Merge pull request #1537 from GNS3/import-export-refactoring

Project import/export refactoring. Fixes #1349

gns3server/compute/base_manager.py

@@ -20,6 +20,7 @@ import os
 import struct
 import stat
 import asyncio
+import aiofiles
 
 import aiohttp
 import socket
@@ -46,6 +47,8 @@ from .nios.nio_ethernet import NIOEthernet
 from ..utils.images import md5sum, remove_checksum, images_directories, default_images_directory, list_images
 from .error import NodeError, ImageMissingError
 
+CHUNK_SIZE = 1024 * 8  # 8KB
+
 
 class BaseManager:
 
@@ -456,7 +459,7 @@ class BaseManager:
             with open(path, "rb") as f:
                 await response.prepare(request)
                 while nio.capturing:
-                    data = f.read(4096)
+                    data = f.read(CHUNK_SIZE)
                     if not data:
                         await asyncio.sleep(0.1)
                         continue
@@ -594,18 +597,18 @@ class BaseManager:
         path = os.path.abspath(os.path.join(directory, *os.path.split(filename)))
         if os.path.commonprefix([directory, path]) != directory:
             raise aiohttp.web.HTTPForbidden(text="Could not write image: {}, {} is forbidden".format(filename, path))
-        log.info("Writing image file %s", path)
+        log.info("Writing image file to '{}'".format(path))
         try:
             remove_checksum(path)
             # We store the file under his final name only when the upload is finished
             tmp_path = path + ".tmp"
             os.makedirs(os.path.dirname(path), exist_ok=True)
-            with open(tmp_path, 'wb') as f:
+            async with aiofiles.open(tmp_path, 'wb') as f:
                 while True:
-                    packet = await stream.read(4096)
-                    if not packet:
+                    chunk = await stream.read(CHUNK_SIZE)
+                    if not chunk:
                         break
-                    f.write(packet)
+                    await f.write(chunk)
             os.chmod(tmp_path, stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
             shutil.move(tmp_path, path)
             await cancellable_wait_run_in_executor(md5sum, path)

gns3server/compute/docker/__init__.py

@@ -37,6 +37,7 @@ log = logging.getLogger(__name__)
 DOCKER_MINIMUM_API_VERSION = "1.25"
 DOCKER_MINIMUM_VERSION = "1.13"
 DOCKER_PREFERRED_API_VERSION = "1.30"
+CHUNK_SIZE = 1024 * 8  # 8KB
 
 
 class Docker(BaseManager):
@@ -206,7 +207,7 @@ class Docker(BaseManager):
         content = ""
         while True:
             try:
-                chunk = await response.content.read(1024)
+                chunk = await response.content.read(CHUNK_SIZE)
             except aiohttp.ServerDisconnectedError:
                 log.error("Disconnected from server while pulling Docker image '{}' from docker hub".format(image))
                 break