Update IOUtools. Ref #1627

gns3server/compute/iou/utils/iou_export.py

@@ -37,19 +37,19 @@ optional arguments:
   -h, --help      show this help message and exit
 """
 
-import argparse
+import struct
-import sys
 
 
-# Uncompress data in .Z file format.
+# Uncompress data in LZC format, .Z file format
-# Ported from dynamips' fs_nvram.c to python
+# LZC uses the LZW compression algorithm with a variable dictionary size
-# Adapted from 7zip's ZDecoder.cpp, which is licensed under LGPL 2.1.
+# For LZW see https://en.wikipedia.org/wiki/Lempel–Ziv–Welch
+# Performance: about 1 MByte/sec, 15-50 times slower than C implementation
 def uncompress_LZC(data):
     LZC_NUM_BITS_MIN = 9
     LZC_NUM_BITS_MAX = 16
 
-    in_data = bytearray(data)
+    in_data  = bytearray(data)
-    in_len = len(in_data)
+    in_len   = len(in_data)
     out_data = bytearray()
 
     if in_len == 0:
@@ -59,143 +59,144 @@ def uncompress_LZC(data):
     if in_data[0] != 0x1F or in_data[1] != 0x9D:
         raise ValueError('invalid header')
 
-    maxbits = in_data[2] & 0x1F
+    max_bits = in_data[2] & 0x1F
-    numItems = 1 << maxbits
+    if max_bits < LZC_NUM_BITS_MIN or max_bits > LZC_NUM_BITS_MAX:
-    blockMode = (in_data[2] & 0x80) != 0
-    if maxbits < LZC_NUM_BITS_MIN or maxbits > LZC_NUM_BITS_MAX:
         raise ValueError('not supported')
+    num_items = 1 << max_bits
+    blockmode = (in_data[2] & 0x80) != 0
 
-    parents = [0] * numItems
+    in_pos    = 3
-    suffixes = [0] * numItems
+    start_pos = in_pos
-
+    num_bits  = LZC_NUM_BITS_MIN
-    in_pos = 3
+    dict_size = 1 << num_bits
-    numBits = LZC_NUM_BITS_MIN
+    head      = 256
-    head = 256
+    if blockmode:
-    if blockMode:
         head += 1
+    first_sym = True
 
-    needPrev = 0
+    # initialize dictionary
-    bitPos = 0
+    comp_dict = [None] * num_items
-    numBufBits = 0
+    for i in range(0, 256):
+        comp_dict[i] = bytes(bytearray([i]))
 
-    parents[256] = 0
+    buf = buf_bits = 0
-    suffixes[256] = 0
+    while in_pos < in_len:
-
+        # get next symbol
-    buf_extend = bytearray([0] * 3)
+        try:
-
+            while buf_bits < num_bits:
-    while True:
+                buf |= in_data[in_pos] << buf_bits
-        # fill buffer, when empty
+                buf_bits += 8
-        if numBufBits == bitPos:
+                in_pos += 1
-            buf_len = min(in_len - in_pos, numBits)
+            buf, symbol = divmod(buf, dict_size)
-            buf = in_data[in_pos:in_pos + buf_len] + buf_extend
+            buf_bits -= num_bits
-            numBufBits = buf_len << 3
+        except IndexError:
-            bitPos = 0
-            in_pos += buf_len
-
-        # extract next symbol
-        bytePos = bitPos >> 3
-        symbol = buf[bytePos] | buf[bytePos + 1] << 8 | buf[bytePos + 2] << 16
-        symbol >>= bitPos & 7
-        symbol &= (1 << numBits) - 1
-        bitPos += numBits
-
-        # check for special conditions: end, bad data, re-initialize dictionary
-        if bitPos > numBufBits:
-            break
-        if symbol >= head:
             raise ValueError('invalid data')
-        if blockMode and symbol == 256:
+
-            numBufBits = bitPos = 0
+        # re-initialize dictionary
-            numBits = LZC_NUM_BITS_MIN
+        if blockmode and symbol == 256:
+            # skip to next buffer boundary
+            buf = buf_bits = 0
+            in_pos += (start_pos - in_pos) % num_bits
+            # reset to LZC_NUM_BITS_MIN
             head = 257
-            needPrev = 0
+            num_bits = LZC_NUM_BITS_MIN
+            dict_size = 1 << num_bits
+            start_pos = in_pos
+            first_sym = True
             continue
 
-        # convert symbol to string
+        # first symbol
-        stack = []
+        if first_sym:
-        cur = symbol
+            first_sym = False
-        while cur >= 256:
+            if symbol >= 256:
-            stack.append(suffixes[cur])
+                raise ValueError('invalid data')
-            cur = parents[cur]
+            prev = symbol
-        stack.append(cur)
+            out_data.extend(comp_dict[symbol])
-        if needPrev:
+            continue
-            suffixes[head - 1] = cur
-            if symbol == head - 1:
-                stack[0] = cur
-        stack.reverse()
-        out_data.extend(stack)
 
-        # update parents, check for numBits change
+        # dictionary full
-        if head < numItems:
+        if head >= num_items:
-            needPrev = 1
+            out_data.extend(comp_dict[symbol])
-            parents[head] = symbol
+            continue
-            head += 1
+
-            if head > (1 << numBits):
+        # update compression dictionary
-                if numBits < maxbits:
+        if symbol < head:
-                    numBufBits = bitPos = 0
+            comp_dict[head] = comp_dict[prev] + comp_dict[symbol][0:1]
-                    numBits += 1
+        elif symbol == head:
+            comp_dict[head] = comp_dict[prev] + comp_dict[prev][0:1]
         else:
-            needPrev = 0
+            raise ValueError('invalid data')
+        prev = symbol
+
+        # output symbol
+        out_data.extend(comp_dict[symbol])
+
+        # update head, check for num_bits change
+        head += 1
+        if head >= dict_size and num_bits < max_bits:
+            num_bits += 1
+            dict_size = 1 << num_bits
+            start_pos = in_pos
 
     return out_data
 
 
-# extract 16 bit unsigned int from data
-def get_uint16(data, off):
-    return data[off] << 8 | data[off + 1]
-
-
-# extract 32 bit unsigned int from data
-def get_uint32(data, off):
-    return data[off] << 24 | data[off + 1] << 16 | data[off + 2] << 8 | data[off + 3]
-
-
 # export IOU NVRAM
+# NVRAM format: https://github.com/ehlers/IOUtools/blob/master/NVRAM.md
 def nvram_export(nvram):
     nvram = bytearray(nvram)
 
-    # extract startup config
     offset = 0
-    if len(nvram) < offset + 36:
+    # extract startup config
+    try:
+        (magic, data_format, _, _, _, _, length, _, _, _, _, _) = \
+            struct.unpack_from('>HHHHIIIIIHHI', nvram, offset=offset)
+        offset += 36
+        if magic != 0xABCD:
+            raise ValueError('no startup config')
+        if len(nvram) < offset+length:
+            raise ValueError('invalid length')
+        startup = nvram[offset:offset+length]
+    except struct.error:
         raise ValueError('invalid length')
-    if get_uint16(nvram, offset + 0) != 0xABCD:
-        raise ValueError('no startup config')
-    format = get_uint16(nvram, offset + 2)
-    length = get_uint32(nvram, offset + 16)
-    offset += 36
-    if len(nvram) < offset + length:
-        raise ValueError('invalid length')
-    startup = nvram[offset:offset + length]
 
-    # compressed startup config
+    # uncompress startup config
-    if format == 2:
+    if data_format == 2:
         try:
             startup = uncompress_LZC(startup)
         except ValueError as err:
             raise ValueError('uncompress startup: ' + str(err))
 
-    offset += length
-    # alignment to multiple of 4
-    offset = (offset + 3) & ~3
-    # check for additonal offset of 4
-    if len(nvram) >= offset + 8 and \
-       get_uint16(nvram, offset + 4) == 0xFEDC and \
-       get_uint16(nvram, offset + 6) == 1:
-        offset += 4
-
-    # extract private config
     private = None
-    if len(nvram) >= offset + 16 and get_uint16(nvram, offset + 0) == 0xFEDC:
+    try:
-        length = get_uint32(nvram, offset + 12)
+        # calculate offset of private header
+        length += (4 - length % 4) % 4          # alignment to multiple of 4
+        offset += length
+        # check for additonal offset of 4
+        (magic, data_format) = struct.unpack_from('>HH', nvram, offset=offset+4)
+        if magic == 0xFEDC and data_format == 1:
+            offset += 4
+
+        # extract private config
+        (magic, data_format, _, _, length) = \
+            struct.unpack_from('>HHIII', nvram, offset=offset)
         offset += 16
-        if len(nvram) >= offset + length:
+        if magic == 0xFEDC and data_format == 1:
-            private = nvram[offset:offset + length]
+            if len(nvram) < offset+length:
+                raise ValueError('invalid length')
+            private = nvram[offset:offset+length]
+
+    # missing private header is not an error
+    except struct.error:
+        pass
 
     return (startup, private)
 
 
 if __name__ == '__main__':
     # Main program
+    import argparse
+    import sys
 
     parser = argparse.ArgumentParser(description='%(prog)s exports startup/private configuration from IOU NVRAM file.')
     parser.add_argument('nvram', metavar='NVRAM',

gns3server/compute/iou/utils/iou_import.py

@@ -35,100 +35,82 @@ optional arguments:
                         create NVRAM file, size in kByte
 """
 
-import argparse
+import struct
-import sys
-
-
-# extract 16 bit unsigned int from data
-def get_uint16(data, off):
-    return data[off] << 8 | data[off + 1]
-
-
-# extract 32 bit unsigned int from data
-def get_uint32(data, off):
-    return data[off] << 24 | data[off + 1] << 16 | data[off + 2] << 8 | data[off + 3]
-
-
-# insert 16 bit unsigned int into data
-def put_uint16(data, off, value):
-    data[off] = (value >> 8) & 0xff
-    data[off + 1] = value & 0xff
-
-
-# insert 32 bit unsigned int into data
-def put_uint32(data, off, value):
-    data[off] = (value >> 24) & 0xff
-    data[off + 1] = (value >> 16) & 0xff
-    data[off + 2] = (value >> 8) & 0xff
-    data[off + 3] = value & 0xff
 
 
 # calculate padding
-def padding(off, ios, nvram_len):
+def padding(length, start_address):
-    pad = (4 - off % 4) % 4             # padding to alignment of 4
+    pad = -length % 4                   # padding to alignment of 4
-    # add 4 if IOS <= 15.0 or NVRAM area >= 64KB
+    # extra padding if pad != 0 and big start_address
-    if (ios <= 0x0F00 or nvram_len >= 64 * 1024) and pad != 0:
+    if pad != 0 and (start_address & 0x80000000) != 0:
         pad += 4
     return pad
 
 
 # update checksum
 def checksum(data, start, end):
-    put_uint16(data, start + 4, 0)      # set checksum to 0
-
     chk = 0
-    idx = start
+    # calculate checksum of first two words
-    while idx < end - 1:
+    for word in struct.unpack_from('>2H', data, start):
-        chk += get_uint16(data, idx)
+        chk += word
-        idx += 2
-    if idx < end:
-        chk += data[idx] << 8
 
+    # add remaining words, ignoring old checksum at offset 4
+    struct_format = '>{:d}H'.format((end - start - 6) // 2)
+    for word in struct.unpack_from(struct_format, data, start+6):
+        chk += word
+
+    # handle 16 bit overflow
     while chk >> 16:
         chk = (chk & 0xffff) + (chk >> 16)
-
     chk = chk ^ 0xffff
-    put_uint16(data, start + 4, chk)    # set checksum
+
+    # save checksum
+    struct.pack_into('>H', data, start+4, chk)
 
 
 # import IOU NVRAM
+# NVRAM format: https://github.com/ehlers/IOUtools/blob/master/NVRAM.md
 def nvram_import(nvram, startup, private, size):
-    BASE_ADDRESS = 0x10000000
+    DEFAULT_IOS  = 0x0F04               # IOS 15.4
-    DEFAULT_IOS = 0x0F04               # IOS 15.4
+    base_address = 0x10000000
 
     # check size parameter
     if size is not None and (size < 8 or size > 1024):
         raise ValueError('invalid size')
 
     # create new nvram if nvram is empty or has wrong size
-    if nvram is None or (size is not None and len(nvram) != size * 1024):
+    if nvram is None or (size is not None and len(nvram) != size*1024):
-        nvram = bytearray([0] * (size * 1024))
+        nvram = bytearray([0] * (size*1024))
     else:
         nvram = bytearray(nvram)
 
     # check nvram size
     nvram_len = len(nvram)
-    if nvram_len < 8 * 1024 or nvram_len > 1024 * 1024 or nvram_len % 1024 != 0:
+    if nvram_len < 8*1024 or nvram_len > 1024*1024 or nvram_len % 1024 != 0:
         raise ValueError('invalid NVRAM length')
     nvram_len = nvram_len // 2
 
     # get size of current config
     config_len = 0
-    ios = None
     try:
-        if get_uint16(nvram, 0) == 0xABCD:
+        (magic, _, _, ios, start_addr, _, length, _, _, _, _, _) = \
-            ios = get_uint16(nvram, 6)
+            struct.unpack_from('>HHHHIIIIIHHI', nvram, offset=0)
-            config_len = 36 + get_uint32(nvram, 16)
+        if magic == 0xABCD:
-            config_len += padding(config_len, ios, nvram_len)
+            base_address = start_addr - 36
-            if get_uint16(nvram, config_len) == 0xFEDC:
+            config_len = 36 + length + padding(length, base_address)
-                config_len += 16 + get_uint32(nvram, config_len + 12)
+            (magic, _, _, _, length) = \
-    except IndexError:
+                struct.unpack_from('>HHIII', nvram, offset=config_len)
+            if magic == 0xFEDC:
+                config_len += 16 + length
+        else:
+            ios = None
+    except struct.error:
         raise ValueError('unknown nvram format')
     if config_len > nvram_len:
         raise ValueError('unknown nvram format')
 
     # calculate max. config size
-    max_config = nvram_len - 2 * 1024             # reserve 2k for files
+    max_config = nvram_len - 2*1024             # reserve 2k for files
     idx = max_config
     empty_sector = bytearray([0] * 1024)
     while True:
@@ -136,45 +118,43 @@ def nvram_import(nvram, startup, private, size):
         if idx < config_len:
             break
         # if valid file header:
-        if get_uint16(nvram, idx + 0) == 0xDCBA and \
+        (magic, _, flags, length, _) = \
-           get_uint16(nvram, idx + 4) < 8 and \
+            struct.unpack_from('>HHHH24s', nvram, offset=idx)
-           get_uint16(nvram, idx + 6) <= 992:
+        if magic == 0xDCBA and flags < 8 and length <= 992:
             max_config = idx
-        elif nvram[idx:idx + 1024] != empty_sector:
+        elif nvram[idx:idx+1024] != empty_sector:
             break
 
     # import startup config
-    startup = bytearray(startup)
+    new_nvram = bytearray()
     if ios is None:
         # Target IOS version is unknown. As some IOU don't work nicely with
         # the padding of a different version, the startup config is padded
         # with '\n' to the alignment of 4.
         ios = DEFAULT_IOS
-        startup.extend([ord('\n')] * ((4 - len(startup) % 4) % 4))
+        startup += b'\n' * (-len(startup) % 4)
-    new_nvram = bytearray([0] * 36)                             # startup hdr
+    new_nvram.extend(struct.pack('>HHHHIIIIIHHI',
-    put_uint16(new_nvram, 0, 0xABCD)                           # magic
+        0xABCD,                             # magic
-    put_uint16(new_nvram, 2, 1)                                # raw data
+        1,                                  # raw data
-    put_uint16(new_nvram, 6, ios)                              # IOS version
+        0,                                  # checksum, not yet calculated
-    put_uint32(new_nvram, 8, BASE_ADDRESS + 36)                  # start address
+        ios,                                # IOS version
-    put_uint32(new_nvram, 12, BASE_ADDRESS + 36 + len(startup))   # end address
+        base_address + 36,                  # start address
-    put_uint32(new_nvram, 16, len(startup))                     # length
+        base_address + 36 + len(startup),   # end address
+        len(startup),                       # length
+        0, 0, 0, 0, 0))
     new_nvram.extend(startup)
-    new_nvram.extend([0] * padding(len(new_nvram), ios, nvram_len))
+    new_nvram.extend([0] * padding(len(new_nvram), base_address))
 
     # import private config
     if private is None:
-        private = bytearray()
+        private = b''
-    else:
-        private = bytearray(private)
     offset = len(new_nvram)
-    new_nvram.extend([0] * 16)                                  # private hdr
+    new_nvram.extend(struct.pack('>HHIII',
-    put_uint16(new_nvram, 0 + offset, 0xFEDC)                  # magic
+        0xFEDC,                             # magic
-    put_uint16(new_nvram, 2 + offset, 1)                       # raw data
+        1,                                  # raw data
-    put_uint32(new_nvram, 4 + offset,
+        base_address + offset + 16,         # start address
-               BASE_ADDRESS + offset + 16)                      # start address
+        base_address + offset + 16 + len(private),  # end address
-    put_uint32(new_nvram, 8 + offset,
+        len(private) ))                     # length
-               BASE_ADDRESS + offset + 16 + len(private))       # end address
-    put_uint32(new_nvram, 12 + offset, len(private))            # length
     new_nvram.extend(private)
 
     # add rest
@@ -190,6 +170,8 @@ def nvram_import(nvram, startup, private, size):
 
 if __name__ == '__main__':
     # Main program
+    import argparse
+    import sys
 
     def check_size(string):
         try: