Use aiofiles where relevant.

gns3server/handlers/api/compute/dynamips_vm_handler.py

@@ -493,7 +493,7 @@ class DynamipsVMHandler:
         if filename[0] == ".":
             raise aiohttp.web.HTTPForbidden()
 
-        await response.file(image_path)
+        await response.stream_file(image_path)
 
     @Route.post(
         r"/projects/{project_id}/dynamips/nodes/{node_id}/duplicate",

gns3server/handlers/api/compute/iou_handler.py

@@ -451,4 +451,4 @@ class IOUHandler:
         if filename[0] == ".":
             raise aiohttp.web.HTTPForbidden()
 
-        await response.file(image_path)
+        await response.stream_file(image_path)

gns3server/handlers/api/compute/project_handler.py

@@ -37,6 +37,8 @@ from gns3server.schemas.project import (
 import logging
 log = logging.getLogger()
 
+CHUNK_SIZE = 1024 * 8  # 8KB
+
 
 class ProjectHandler:
 
@@ -248,64 +250,7 @@ class ProjectHandler:
             raise aiohttp.web.HTTPForbidden()
         path = os.path.join(project.path, path)
 
-        response.content_type = "application/octet-stream"
-        response.set_status(200)
-        response.enable_chunked_encoding()
-
-        try:
-            with open(path, "rb") as f:
-                await response.prepare(request)
-                while True:
-                    data = f.read(4096)
-                    if not data:
-                        break
-                    await response.write(data)
-
-        except FileNotFoundError:
-            raise aiohttp.web.HTTPNotFound()
-        except PermissionError:
-            raise aiohttp.web.HTTPForbidden()
-
-    @Route.get(
-        r"/projects/{project_id}/stream/{path:.+}",
-        description="Stream a file from a project",
-        parameters={
-            "project_id": "Project UUID",
-        },
-        status_codes={
-            200: "File returned",
-            403: "Permission denied",
-            404: "The file doesn't exist"
-        })
-    async def stream_file(request, response):
-
-        pm = ProjectManager.instance()
-        project = pm.get_project(request.match_info["project_id"])
-        path = request.match_info["path"]
-        path = os.path.normpath(path)
-
-        # Raise an error if user try to escape
-        if path[0] == ".":
-            raise aiohttp.web.HTTPForbidden()
-        path = os.path.join(project.path, path)
-
-        response.content_type = "application/octet-stream"
-        response.set_status(200)
-        response.enable_chunked_encoding()
-
-        # FIXME: file streaming is never stopped
-        try:
-            with open(path, "rb") as f:
-                await response.prepare(request)
-                while True:
-                    data = f.read(4096)
-                    if not data:
-                        await asyncio.sleep(0.1)
-                    await response.write(data)
-        except FileNotFoundError:
-            raise aiohttp.web.HTTPNotFound()
-        except PermissionError:
-            raise aiohttp.web.HTTPForbidden()
+        await response.stream_file(path)
 
     @Route.post(
         r"/projects/{project_id}/files/{path:.+}",
@@ -338,7 +283,7 @@ class ProjectHandler:
             with open(path, 'wb+') as f:
                 while True:
                     try:
-                        chunk = await request.content.read(1024)
+                        chunk = await request.content.read(CHUNK_SIZE)
                     except asyncio.TimeoutError:
                         raise aiohttp.web.HTTPRequestTimeout(text="Timeout when writing to file '{}'".format(path))
                     if not chunk:
@@ -349,64 +294,3 @@ class ProjectHandler:
             raise aiohttp.web.HTTPNotFound()
         except PermissionError:
             raise aiohttp.web.HTTPForbidden()
-
-    @Route.get(
-        r"/projects/{project_id}/export",
-        description="Export a project as a portable archive",
-        parameters={
-            "project_id": "Project UUID",
-        },
-        raw=True,
-        status_codes={
-            200: "File returned",
-            404: "The project doesn't exist"
-        })
-    async def export_project(request, response):
-
-        pm = ProjectManager.instance()
-        project = pm.get_project(request.match_info["project_id"])
-        response.content_type = 'application/gns3project'
-        response.headers['CONTENT-DISPOSITION'] = 'attachment; filename="{}.gns3project"'.format(project.name)
-        response.enable_chunked_encoding()
-        await response.prepare(request)
-
-        include_images = bool(int(request.json.get("include_images", "0")))
-        for data in project.export(include_images=include_images):
-            await response.write(data)
-
-        #await response.write_eof() #FIXME: shound't be needed anymore
-
-    @Route.post(
-        r"/projects/{project_id}/import",
-        description="Import a project from a portable archive",
-        parameters={
-            "project_id": "Project UUID",
-        },
-        raw=True,
-        output=PROJECT_OBJECT_SCHEMA,
-        status_codes={
-            200: "Project imported",
-            403: "Forbidden to import project"
-        })
-    async def import_project(request, response):
-
-        pm = ProjectManager.instance()
-        project_id = request.match_info["project_id"]
-        project = pm.create_project(project_id=project_id)
-
-        # We write the content to a temporary location and after we extract it all.
-        # It could be more optimal to stream this but it is not implemented in Python.
-        # Spooled means the file is temporary kept in memory until max_size is reached
-        try:
-            with tempfile.SpooledTemporaryFile(max_size=10000) as temp:
-                while True:
-                    chunk = await request.content.read(1024)
-                    if not chunk:
-                        break
-                    temp.write(chunk)
-                project.import_zip(temp, gns3vm=bool(int(request.GET.get("gns3vm", "1"))))
-        except OSError as e:
-            raise aiohttp.web.HTTPInternalServerError(text="Could not import the project: {}".format(e))
-
-        response.json(project)
-        response.set_status(201)

gns3server/handlers/api/compute/qemu_handler.py

@@ -576,4 +576,4 @@ class QEMUHandler:
         if filename[0] == ".":
             raise aiohttp.web.HTTPForbidden()
 
-        await response.file(image_path)
+        await response.stream_file(image_path)

gns3server/handlers/api/controller/project_handler.py

@@ -16,11 +16,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import sys
 import aiohttp
 import asyncio
 import tempfile
 import zipfile
+import aiofiles
 import time
 
 from gns3server.web.route import Route
@@ -51,6 +51,8 @@ async def process_websocket(ws):
     except aiohttp.WSServerHandshakeError:
         pass
 
+CHUNK_SIZE = 1024 * 8  # 8KB
+
 
 class ProjectHandler:
 
@@ -304,7 +306,6 @@ class ProjectHandler:
         controller = Controller.instance()
         project = await controller.get_loaded_project(request.match_info["project_id"])
 
-
         try:
             begin = time.time()
             with tempfile.TemporaryDirectory() as tmp_dir:
@@ -321,8 +322,8 @@ class ProjectHandler:
                     async for chunk in zstream:
                         await response.write(chunk)
 
-            log.info("Project '{}' exported in {:.4f} seconds".format(project.id, time.time() - begin))
-                #await response.write_eof() #FIXME: shound't be needed anymore
+            log.info("Project '{}' exported in {:.4f} seconds".format(project.name, time.time() - begin))
+
         # Will be raise if you have no space left or permission issue on your temporary directory
         # RuntimeError: something was wrong during the zip process
         except (ValueError, OSError, RuntimeError) as e:
@@ -354,29 +355,23 @@ class ProjectHandler:
 
         # We write the content to a temporary location and after we extract it all.
         # It could be more optimal to stream this but it is not implemented in Python.
-        # Spooled means the file is temporary kept in memory until max_size is reached
-        # Cannot use tempfile.SpooledTemporaryFile(max_size=10000) in Python 3.7 due
-        # to a bug https://bugs.python.org/issue26175
         try:
-            if sys.version_info >= (3, 7) and sys.version_info < (3, 8):
-                with tempfile.TemporaryFile() as temp:
+            begin = time.time()
+            with tempfile.TemporaryDirectory() as tmpdir:
+                temp_project_path = os.path.join(tmpdir, "project.zip")
+                async with aiofiles.open(temp_project_path, 'wb') as f:
                     while True:
-                        chunk = await request.content.read(1024)
+                        chunk = await request.content.read(CHUNK_SIZE)
                         if not chunk:
                             break
-                        temp.write(chunk)
-                    project = await import_project(controller, request.match_info["project_id"], temp, location=path, name=name)
-            else:
-                with tempfile.SpooledTemporaryFile(max_size=10000) as temp:
-                    while True:
-                        chunk = await request.content.read(1024)
-                        if not chunk:
-                            break
-                        temp.write(chunk)
-                    project = await import_project(controller, request.match_info["project_id"], temp, location=path, name=name)
+                        await f.write(chunk)
+
+                with open(temp_project_path, "rb") as f:
+                    project = await import_project(controller, request.match_info["project_id"], f, location=path, name=name)
+
+            log.info("Project '{}' imported in {:.4f} seconds".format(project.name, time.time() - begin))
         except OSError as e:
             raise aiohttp.web.HTTPInternalServerError(text="Could not import the project: {}".format(e))
-
         response.json(project)
         response.set_status(201)
 
@@ -443,7 +438,7 @@ class ProjectHandler:
             with open(path, "rb") as f:
                 await response.prepare(request)
                 while True:
-                    data = f.read(4096)
+                    data = f.read(CHUNK_SIZE)
                     if not data:
                         break
                     await response.write(data)
@@ -483,7 +478,7 @@ class ProjectHandler:
             with open(path, 'wb+') as f:
                 while True:
                     try:
-                        chunk = await request.content.read(1024)
+                        chunk = await request.content.read(CHUNK_SIZE)
                     except asyncio.TimeoutError:
                         raise aiohttp.web.HTTPRequestTimeout(text="Timeout when writing to file '{}'".format(path))
                     if not chunk:

gns3server/handlers/api/controller/symbol_handler.py

@@ -53,7 +53,7 @@ class SymbolHandler:
 
         controller = Controller.instance()
         try:
-            await response.file(controller.symbols.get_path(request.match_info["symbol_id"]))
+            await response.stream_file(controller.symbols.get_path(request.match_info["symbol_id"]))
         except (KeyError, OSError) as e:
             log.warning("Could not get symbol file: {}".format(e))
             response.set_status(404)