Merge pull request #1800 from GNS3/docker-resource-constraints

Resource constraints for Docker VMs.

gns3server/compute/docker/docker_vm.py

@@ -71,7 +71,7 @@ class DockerVM(BaseNode):
 
     def __init__(self, name, node_id, project, manager, image, console=None, aux=None, start_command=None,
                  adapters=None, environment=None, console_type="telnet", console_resolution="1024x768",
-                 console_http_port=80, console_http_path="/", extra_hosts=None, extra_volumes=[]):
+                 console_http_port=80, console_http_path="/", extra_hosts=None, extra_volumes=[], memory=0, cpus=0):
 
         super().__init__(name, node_id, project, manager, console=console, aux=aux, allocate_aux=True, console_type=console_type)
 
@@ -94,6 +94,8 @@ class DockerVM(BaseNode):
         self._console_websocket = None
         self._extra_hosts = extra_hosts
         self._extra_volumes = extra_volumes or []
+        self._memory = memory
+        self._cpus = cpus
         self._permissions_fixed = False
         self._display = None
         self._closing = False
@@ -132,6 +134,8 @@ class DockerVM(BaseNode):
             "node_directory": self.working_path,
             "extra_hosts": self.extra_hosts,
             "extra_volumes": self.extra_volumes,
+            "memory": self.memory,
+            "cpus": self.cpus
         }
 
     def _get_free_display_port(self):
@@ -211,6 +215,22 @@ class DockerVM(BaseNode):
     def extra_volumes(self, extra_volumes):
         self._extra_volumes = extra_volumes
 
+    @property
+    def memory(self):
+        return self._memory
+
+    @memory.setter
+    def memory(self, memory):
+        self._memory = memory
+
+    @property
+    def cpus(self):
+        return self._cpus
+
+    @cpus.setter
+    def cpus(self, cpus):
+        self._cpus = cpus
+
     async def _get_container_state(self):
         """
         Returns the container state (e.g. running, paused etc.)
@@ -328,6 +348,10 @@ class DockerVM(BaseNode):
         if image_infos is None:
             raise DockerError("Cannot get information for image '{}', please try again.".format(self._image))
 
+        available_cpus = psutil.cpu_count(logical=True)
+        if self._cpus > available_cpus:
+            raise DockerError("You have allocated too many CPUs for the Docker container (max available is {} CPUs)".format(available_cpus))
+
         params = {
             "Hostname": self._name,
             "Name": self._name,
@@ -340,6 +364,8 @@ class DockerVM(BaseNode):
                 "CapAdd": ["ALL"],
                 "Privileged": True,
                 "Binds": self._mount_binds(image_infos),
+                "Memory": self._memory * (1024 * 1024),  # convert memory to bytes
+                "NanoCpus": int(self._cpus * 1e9)  # convert cpus to nano cpus
             },
             "Volumes": {},
             "Env": ["container=docker"],  # Systemd compliant: https://github.com/GNS3/gns3-server/issues/573

gns3server/handlers/api/compute/docker_handler.py

@@ -49,20 +49,22 @@ class DockerHandler:
     async def create(request, response):
         docker_manager = Docker.instance()
         container = await docker_manager.create_node(request.json.pop("name"),
-                                                          request.match_info["project_id"],
+                                                     request.match_info["project_id"],
-                                                          request.json.get("node_id"),
+                                                     request.json.get("node_id"),
-                                                          image=request.json.pop("image"),
+                                                     image=request.json.pop("image"),
-                                                          start_command=request.json.get("start_command"),
+                                                     start_command=request.json.get("start_command"),
-                                                          environment=request.json.get("environment"),
+                                                     environment=request.json.get("environment"),
-                                                          adapters=request.json.get("adapters"),
+                                                     adapters=request.json.get("adapters"),
-                                                          console=request.json.get("console"),
+                                                     console=request.json.get("console"),
-                                                          console_type=request.json.get("console_type"),
+                                                     console_type=request.json.get("console_type"),
-                                                          console_resolution=request.json.get("console_resolution", "1024x768"),
+                                                     console_resolution=request.json.get("console_resolution", "1024x768"),
-                                                          console_http_port=request.json.get("console_http_port", 80),
+                                                     console_http_port=request.json.get("console_http_port", 80),
-                                                          console_http_path=request.json.get("console_http_path", "/"),
+                                                     console_http_path=request.json.get("console_http_path", "/"),
-                                                          aux=request.json.get("aux"),
+                                                     aux=request.json.get("aux"),
-                                                          extra_hosts=request.json.get("extra_hosts"),
+                                                     extra_hosts=request.json.get("extra_hosts"),
-                                                          extra_volumes=request.json.get("extra_volumes"))
+                                                     extra_volumes=request.json.get("extra_volumes"),
+                                                     memory=request.json.get("memory", 0),
+                                                     cpus=request.json.get("cpus", 0))
         for name, value in request.json.items():
             if name != "node_id":
                 if hasattr(container, name) and getattr(container, name) != value:
@@ -317,7 +319,8 @@ class DockerHandler:
         props = [
             "name", "console", "aux", "console_type", "console_resolution",
             "console_http_port", "console_http_path", "start_command",
-            "environment", "adapters", "extra_hosts", "extra_volumes"
+            "environment", "adapters", "extra_hosts", "extra_volumes",
+            "memory", "cpus"
         ]
 
         changed = False

gns3server/schemas/docker.py

@@ -103,6 +103,14 @@ DOCKER_CREATE_SCHEMA = {
                 "type": "string"
             }
         },
+        "memory": {
+            "description": "Maximum amount of memory the container can use in MB",
+            "type": "integer",
+        },
+        "cpus": {
+            "description": "Maximum amount of CPU resources the container can use",
+            "type": "number",
+        },
         "container_id": {
             "description": "Docker container ID Read only",
             "type": "string",
@@ -214,6 +222,14 @@ DOCKER_OBJECT_SCHEMA = {
                 "type": "string",
             }
         },
+        "memory": {
+            "description": "Maximum amount of memory the container can use in MB",
+            "type": "integer",
+        },
+        "cpus": {
+            "description": "Maximum amount of CPU resources the container can use",
+            "type": "number",
+        },
         "node_directory": {
             "description": "Path to the node working directory  Read only",
             "type": "string"

gns3server/schemas/docker_template.py

@@ -82,6 +82,16 @@ DOCKER_TEMPLATE_PROPERTIES = {
         "type": "array",
         "default": []
     },
+    "memory": {
+        "description": "Maximum amount of memory the container can use in MB",
+        "type": "integer",
+        "default": 0
+    },
+    "cpus": {
+        "description": "Maximum amount of CPU resources the container can use",
+        "type": "number",
+        "default": 0
+    },
     "custom_adapters": CUSTOM_ADAPTERS_ARRAY_SCHEMA
 }
 

tests/compute/docker/test_docker_vm.py

@@ -66,6 +66,8 @@ def test_json(vm, compute_project):
         'console_http_path': '/',
         'extra_hosts': None,
         'extra_volumes': [],
+        'memory': 0,
+        'cpus': 0,
         'aux': vm.aux,
         'start_command': vm.start_command,
         'environment': vm.environment,
@@ -104,7 +106,9 @@ async def test_create(compute_project, manager):
                             "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -143,7 +147,9 @@ async def test_create_with_tag(compute_project, manager):
                             "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -186,7 +192,9 @@ async def test_create_vnc(compute_project, manager):
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             '/tmp/.X11-unix/:/tmp/.X11-unix/'
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -301,7 +309,9 @@ async def test_create_start_cmd(compute_project, manager):
                             "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "Entrypoint": ["/gns3/init.sh"],
@@ -400,7 +410,9 @@ async def test_create_image_not_available(compute_project, manager):
                             "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -444,7 +456,9 @@ async def test_create_with_user(compute_project, manager):
                             "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -528,7 +542,9 @@ async def test_create_with_extra_volumes_duplicate_1_image(compute_project, mana
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -568,7 +584,9 @@ async def test_create_with_extra_volumes_duplicate_2_user(compute_project, manag
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -608,7 +626,9 @@ async def test_create_with_extra_volumes_duplicate_3_subdir(compute_project, man
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")),
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -648,7 +668,9 @@ async def test_create_with_extra_volumes_duplicate_4_backslash(compute_project,
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
                             "{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")),
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -687,7 +709,9 @@ async def test_create_with_extra_volumes_duplicate_5_subdir_issue_1595(compute_p
                             "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                             "{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")),
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -726,7 +750,9 @@ async def test_create_with_extra_volumes_duplicate_6_subdir_issue_1595(compute_p
                             "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                             "{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")),
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -773,7 +799,9 @@ async def test_create_with_extra_volumes(compute_project, manager):
                             "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
                             "{}:/gns3volumes/vol/2".format(os.path.join(vm.working_dir, "vol", "2")),
                         ],
-                        "Privileged": True
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 0
                     },
                 "Volumes": {},
                 "NetworkDisabled": True,
@@ -996,7 +1024,9 @@ async def test_update(vm):
                 "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                 "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
             ],
-            "Privileged": True
+            "Privileged": True,
+            "Memory": 0,
+            "NanoCpus": 0
         },
         "Volumes": {},
         "NetworkDisabled": True,
@@ -1065,7 +1095,9 @@ async def test_update_running(vm):
                 "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                 "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
             ],
-            "Privileged": True
+            "Privileged": True,
+            "Memory": 0,
+            "NanoCpus": 0
         },
         "Volumes": {},
         "NetworkDisabled": True,
@@ -1422,3 +1454,85 @@ async def test_read_console_output_with_binary_mode(vm):
     with asyncio_patch('gns3server.compute.docker.docker_vm.DockerVM.stop'):
         await vm._read_console_output(input_stream, output_stream)
         output_stream.feed_data.assert_called_once_with(b"test")
+
+
+async def test_cpus(compute_project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": []
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]):
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), compute_project, manager, "ubuntu:latest", cpus=0.5)
+            await vm.create()
+            mock.assert_called_with("POST", "containers/create", data={
+                "Tty": True,
+                "OpenStdin": True,
+                "StdinOnce": False,
+                "HostConfig":
+                    {
+                        "CapAdd": ["ALL"],
+                        "Binds": [
+                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
+                        ],
+                        "Privileged": True,
+                        "Memory": 0,
+                        "NanoCpus": 500000000
+                    },
+                "Volumes": {},
+                "NetworkDisabled": True,
+                "Name": "test",
+                "Hostname": "test",
+                "Image": "ubuntu:latest",
+                "Env": [
+                    "container=docker",
+                    "GNS3_MAX_ETHERNET=eth0",
+                    "GNS3_VOLUMES=/etc/network"
+                    ],
+                "Entrypoint": ["/gns3/init.sh"],
+                "Cmd": ["/bin/sh"]
+            })
+        assert vm._cid == "e90e34656806"
+
+
+async def test_memory(compute_project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": []
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]):
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), compute_project, manager, "ubuntu:latest", memory=32)
+            await vm.create()
+            mock.assert_called_with("POST", "containers/create", data={
+                "Tty": True,
+                "OpenStdin": True,
+                "StdinOnce": False,
+                "HostConfig":
+                    {
+                        "CapAdd": ["ALL"],
+                        "Binds": [
+                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network"))
+                        ],
+                        "Privileged": True,
+                        "Memory": 33554432,  # 32MB in bytes
+                        "NanoCpus": 0
+                    },
+                "Volumes": {},
+                "NetworkDisabled": True,
+                "Name": "test",
+                "Hostname": "test",
+                "Image": "ubuntu:latest",
+                "Env": [
+                    "container=docker",
+                    "GNS3_MAX_ETHERNET=eth0",
+                    "GNS3_VOLUMES=/etc/network"
+                    ],
+                "Entrypoint": ["/gns3/init.sh"],
+                "Cmd": ["/bin/sh"]
+            })
+        assert vm._cid == "e90e34656806"