Functionality:
KSM test start guests. They are connect to guest over ssh.
Copy and run allocator.py to guests.
Host can run any python command over allocator.py loop on client side.
Start run_ksm_overcommit.
Define host and guest reserve variables (host_reserver,guest_reserver).
Calculate amount of virtual machine and their memory based on variables
host_mem and overcommit.
Check KSM status.
Create and start virtual guests.
Test :
a] serial
1) initialize, merge all mem to single page
2) separate first guset mem
3) separate rest of guest up to fill all mem
4) kill all guests except for the last
5) check if mem of last guest is ok
6) kill guest
b] parallel
1) initialize, merge all mem to single page
2) separate mem of guest
3) verification of guest mem
4) merge mem to one block
5) verification of guests mem
6) separate mem of guests by 96B
7) check if mem is all right
8) kill guest
allocator.py (client side script)
After start they wait for command witch they make in client side.
mem_fill class implement commands to fill, check mem and return
error to host.
Signed-off-by: Jiri Zupka <jzupka@xxxxxxxxxx>
Signed-off-by: Lukáš Doktor<ldoktor@xxxxxxxxxx>
---
client/tests/kvm/kvm_test_utils.py | 36 ++-
client/tests/kvm/kvm_utils.py | 16 +
client/tests/kvm/kvm_vm.py | 17 +
client/tests/kvm/scripts/allocator.py | 230 ++++++++++++
client/tests/kvm/tests/ksm_overcommit.py | 573 ++++++++++++++++++++++++++++++
client/tests/kvm/tests_base.cfg.sample | 18 +
6 files changed, 889 insertions(+), 1 deletions(-)
create mode 100644 client/tests/kvm/scripts/allocator.py
create mode 100644 client/tests/kvm/tests/ksm_overcommit.py
diff --git a/client/tests/kvm/kvm_test_utils.py b/client/tests/kvm/kvm_test_utils.py
index 02ec0cf..7d96d6e 100644
--- a/client/tests/kvm/kvm_test_utils.py
+++ b/client/tests/kvm/kvm_test_utils.py
@@ -22,7 +22,8 @@ More specifically:
"""
import time, os, logging, re, commands
-from autotest_lib.client.common_lib import utils, error
+from autotest_lib.client.common_lib import error
+from autotest_lib.client.bin import utils
import kvm_utils, kvm_vm, kvm_subprocess
@@ -203,3 +204,36 @@ def get_time(session, time_command, time_filter_re, time_format):
s = re.findall(time_filter_re, s)[0]
guest_time = time.mktime(time.strptime(s, time_format))
return (host_time, guest_time)
+
+
+def get_memory_info(lvms):
+ """
+ Get memory information from host and guests in format:
+ Host: memfree = XXXM; Guests memsh = {XXX,XXX,...}
+
+ @params lvms: List of VM objects
+ @return: String with memory info report
+ """
+ if not isinstance(lvms, list):
+ raise error.TestError("Invalid list passed to get_stat: %s " % lvms)
+
+ try:
+ meminfo = "Host: memfree = "
+ meminfo += str(int(utils.freememtotal()) / 1024) + "M; "
+ meminfo += "swapfree = "
+ mf = int(utils.read_from_meminfo("SwapFree")) / 1024
+ meminfo += str(mf) + "M; "
+ except Exception, e:
+ raise error.TestFail("Could not fetch host free memory info, "
+ "reason: %s" % e)
+
+ meminfo += "Guests memsh = {"
+ for vm in lvms:
+ shm = vm.get_shared_meminfo()
+ if shm is None:
+ raise error.TestError("Could not get shared meminfo from "
+ "VM %s" % vm)
+ meminfo += "%dM; " % shm
+ meminfo = meminfo[0:-2] + "}"
+
+ return meminfo
diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py
index 2bbbe22..0118742 100644
--- a/client/tests/kvm/kvm_utils.py
+++ b/client/tests/kvm/kvm_utils.py
@@ -713,6 +713,22 @@ def generate_random_string(length):
return str
+def generate_tmp_file_name(file, ext=None, dir='/tmp/'):
+ """
+ Returns a temporary file name. The file is not created.
+ """
+ while True:
+ file_name = (file + '-' + time.strftime("%Y%m%d-%H%M%S-") +
+ generate_random_string(4))
+ if ext:
+ file_name += '.' + ext
+ file_name = os.path.join(dir, file_name)
+ if not os.path.exists(file_name):
+ break
+
+ return file_name
+
+
def format_str_for_message(str):
"""
Format str so that it can be appended to a message.
diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
index 7229b79..e52f826 100755
--- a/client/tests/kvm/kvm_vm.py
+++ b/client/tests/kvm/kvm_vm.py
@@ -690,6 +690,23 @@ class VM:
return self.process.get_pid()
+ def get_shared_meminfo(self):
+ """
+ Returns the VM's shared memory information.
+
+ @return: Shared memory used by VM (MB)
+ """
+ if self.is_dead():
+ logging.error("Could not get shared memory info from dead VM.")
+ return None
+
+ cmd = "cat /proc/%d/statm" % self.params.get('pid_' + self.name)
+ shm = int(os.popen(cmd).readline().split()[2])
+ # statm stores informations in pages, translate it to MB
+ shm = shm * 4 / 1024
+ return shm
+
+
def remote_login(self, nic_index=0, timeout=10):
"""
Log into the guest via SSH/Telnet/Netcat.
diff --git a/client/tests/kvm/scripts/allocator.py b/client/tests/kvm/scripts/allocator.py
new file mode 100644
index 0000000..e0b8c75
--- /dev/null
+++ b/client/tests/kvm/scripts/allocator.py
@@ -0,0 +1,230 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+Auxiliary script used to allocate memory on guests.
+
+@copyright: 2008-2009 Red Hat Inc.
+@author: Jiri Zupka (jzupka@xxxxxxxxxx)
+"""
+
+
+import os, array, sys, struct, random, copy, inspect, tempfile, datetime
+
+PAGE_SIZE = 4096 # machine page size
+
+
+class MemFill(object):
+ """
+ Fills guest memory according to certain patterns.
+ """
+ def __init__(self, mem, static_value, random_key):
+ """
+ Constructor of MemFill class.
+
+ @param mem: Amount of test memory in MB.
+ @param random_key: Seed of random series used for fill up memory.
+ @param static_value: Value used to fill all memory.
+ """
+ if (static_value < 0 or static_value > 255):
+ print ("FAIL: Initialization static value"
+ "can be only in range (0..255)")
+ return
+
+ self.tmpdp = tempfile.mkdtemp()
+ ret_code = os.system("mount -o size=%dM tmpfs %s -t tmpfs" %
+ ((mem + 25), self.tmpdp))
+ if ret_code != 0:
+ if os.getuid() != 0:
+ print ("FAIL: Unable to mount tmpfs "
+ "(likely cause: you are not root)")
+ else:
+ print "FAIL: Unable to mount tmpfs"
+ else:
+ self.f = tempfile.TemporaryFile(prefix='mem', dir=self.tmpdp)
+ self.allocate_by = 'L'
+ self.npages = (mem * 1024 * 1024) / PAGE_SIZE
+ self.random_key = random_key
+ self.static_value = static_value
+ print "PASS: Initialization"
+
+
+ def __del__(self):
+ if (os.path.ismount(self.tmpdp)):
+ self.f.close()
+ os.system("umount %s" % (self.tmpdp))
+
+
+ def compare_page(self, original, inmem):
+ """
+ Compare pages of memory.
+
+ @param original: Data that was expected to be in memory.
+ @param inmem: Data in memory.
+ """
+ for ip in range(PAGE_SIZE / original.itemsize):
+ if (not original[ip] == inmem[ip]): # find which item is wrong
+ originalp = array.array("B")
+ inmemp = array.array("B")
+ originalp.fromstring(original[ip:ip+1].tostring())
+ inmemp.fromstring(inmem[ip:ip+1].tostring())
+ for ib in range(len(originalp)): # find wrong byte in item
+ if not (originalp[ib] == inmemp[ib]):
+ position = (self.f.tell() - PAGE_SIZE + ip *
+ original.itemsize + ib)
+ print ("Mem error on position %d wanted 0x%Lx and is "
+ "0x%Lx" % (position, originalp[ib], inmemp[ib]))
+
+
+ def value_page(self, value):
+ """
+ Create page filled by value.
+
+ @param value: String we want to fill the page with.
+ @return: return array of bytes size PAGE_SIZE.
+ """
+ a = array.array("B")
+ for i in range(PAGE_SIZE / a.itemsize):
+ try:
+ a.append(value)
+ except:
+ print "FAIL: Value can be only in range (0..255)"
+ return a
+
+
+ def random_page(self, seed):
+ """
+ Create page filled by static random series.
+
+ @param seed: Seed of random series.
+ @return: Static random array series.
+ """
+ random.seed(seed)
+ a = array.array(self.allocate_by)
+ for i in range(PAGE_SIZE / a.itemsize):
+ a.append(random.randrange(0, sys.maxint))
+ return a
+
+
+ def value_fill(self, value=None):
+ """
+ Fill memory page by page, with value generated with value_page.
+
+ @param value: Parameter to be passed to value_page. None to just use
+ what's on the attribute static_value.
+ """
+ self.f.seek(0)
+ if value is None:
+ value = self.static_value
+ page = self.value_page(value)
+ for pages in range(self.npages):
+ page.tofile(self.f)
+ print "PASS: Mem value fill"
+
+
+ def value_check(self):
+ """
+ Check memory to see if data is correct.
+
+ @return: if data in memory is correct return PASS
+ else print some wrong data and return FAIL
+ """
+ self.f.seek(0)
+ e = 2
+ failure = False
+ page = self.value_page(self.static_value)
+ for pages in range(self.npages):
+ pf = array.array("B")
+ pf.fromfile(self.f, PAGE_SIZE / pf.itemsize)
+ if not (page == pf):
+ failure = True
+ self.compare_page(page, pf)
+ e = e - 1
+ if e == 0:
+ break
+ if failure:
+ print "FAIL: value verification"
+ else:
+ print "PASS: value verification"
+
+
+ def static_random_fill(self, n_bytes_on_end=PAGE_SIZE):
+ """
+ Fill memory by page with static random series with added special value
+ on random place in pages.
+
+ @param n_bytes_on_end: how many bytes on the end of page can be changed.
+ @return: PASS.
+ """
+ self.f.seek(0)
+ page = self.random_page(self.random_key)
+ random.seed(self.random_key)
+ p = copy.copy(page)
+
+ t_start = datetime.datetime.now()
+ for pages in range(self.npages):
+ rand = random.randint(((PAGE_SIZE / page.itemsize) - 1) -
+ (n_bytes_on_end / page.itemsize),
+ (PAGE_SIZE/page.itemsize) - 1)
+ p[rand] = pages
+ p.tofile(self.f)
+ p[rand] = page[rand]
+
+ t_end = datetime.datetime.now()
+ delta = t_end - t_start
+ milisec = delta.microseconds / 1e3 + delta.seconds * 1e3
+ print "PASS: filling duration = %Ld ms" % milisec
+
+
+ def static_random_verify(self, n_bytes_on_end=PAGE_SIZE):
+ """
+ Check memory to see if it contains correct contents.
+
+ @return: if data in memory is correct return PASS
+ else print some wrong data and return FAIL.
+ """
+ self.f.seek(0)
+ e = 2
+ page = self.random_page(self.random_key)
+ random.seed(self.random_key)
+ p = copy.copy(page)
+ failure = False
+ for pages in range(self.npages):
+ rand = random.randint(((PAGE_SIZE/page.itemsize) - 1) -
+ (n_bytes_on_end/page.itemsize),
+ (PAGE_SIZE/page.itemsize) - 1)
+ p[rand] = pages
+ pf = array.array(self.allocate_by)
+ pf.fromfile(self.f, PAGE_SIZE / pf.itemsize)
+ if not (p == pf):
+ failure = True
+ self.compare_page(p, pf)
+ e = e - 1
+ if e == 0:
+ break
+ p[rand] = page[rand]
+ if failure:
+ print "FAIL: Random series verification"
+ else:
+ print "PASS: Random series verification"
+
+
+def die():
+ """
+ Quit allocator.
+ """
+ exit(0)
+
+
+def main():
+ """
+ Main (infinite) loop of allocator.
+ """
+ print "PASS: Start"
+ end = False
+ while (not end):
+ str = raw_input()
+ exec str
+
+
+if __name__ == "__main__":
+ main()
diff --git a/client/tests/kvm/tests/ksm_overcommit.py b/client/tests/kvm/tests/ksm_overcommit.py
new file mode 100644
index 0000000..01b2271
--- /dev/null
+++ b/client/tests/kvm/tests/ksm_overcommit.py
@@ -0,0 +1,573 @@
+import logging, time, random, string, math, os, tempfile
+from autotest_lib.client.common_lib import error
+from autotest_lib.client.bin import utils
+import kvm_subprocess, kvm_test_utils, kvm_utils, kvm_preprocessing
+
+
+def run_ksm_overcommit(test, params, env):
+ """
+ Test how KSM (Kernel Shared Memory) act with more than physical memory is
+ used. In second part is also tested, how KVM can handle the situation,
+ when the host runs out of memory (it is expected to pause the guest system,
+ wait until some process returns the memory and bring the guest back to life)
+
+
+
+ @param test: kvm test object.
+ @param params: Dictionary with test parameters.
+ @param env: Dictionary with the test wnvironment.
+ """
+
+ def initialize_guests():
+ """
+ Initialize guests (fill their memories with specified patterns).
+ """
+ logging.info("Starting phase 1: filling with 0")
+ logging.info("Preparing the guests and fill in pages with 0")
+ for session in lsessions:
+ vm = lvms[lsessions.index(session)]
+
+ ret = session.get_command_status("swapoff -a", timeout=300)
+ if ret is None or ret:
+ raise error.TestFail("Failed to swapoff on VM %s" % vm.name)
+
+ # Start the allocator
+ session.sendline("python /tmp/allocator.py")
+ (ret, data) = session.read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ 60 * perf_ratio)
+ if ret is None or ret == 1:
+ raise error.TestFail("Could not run vksmd in %s errno: %s\n"
+ "Output:\n%s" % (vm.name, ret, data))
+
+ # Set allocator keys
+ for i in range(0, vmsc):
+ vm = lvms[i]
+
+ lsessions[i].sendline("mem = MemFill(%d, %s, %s)" %
+ (ksm_size, skeys[i], dkeys[i]))
+
+ (match, data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ 60 * perf_ratio)
+ if match == 1 or match is None:
+ raise error.TestFail("Could not allocate memory on guest %s\n"
+ "Output:\n%s" % (vm.name, data))
+
+ lsessions[i].sendline("mem.value_fill(%d)"% (skeys[0]))
+ (match,data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ 120 * perf_ratio)
+ if match == 1 or match is None:
+ raise error.TestFail("Could not allocate memory on guest %s\n"
+ "Output: %s" % (vm.name, data))
+
+ # Let kksmd works (until shared mem reaches expected value)
+ shm = 0
+ i = 0
+ while shm < ksm_size:
+ if i > 64:
+ logging.info(kvm_test_utils.get_memory_info(lvms))
+ raise error.TestError("SHM didn't merge the memory until "
+ "the DL on guest: %s"% (vm.name))
+ st = ksm_size / 200 * perf_ratio
+ logging.debug("Sleep (%d)" % st)
+ time.sleep(st)
+ shm = vm.get_shared_meminfo()
+ i = i + 1
+
+ # Keep some reserve
+ time.sleep(ksm_size / 200 * perf_ratio)
+
+ logging.info(kvm_test_utils.get_memory_info(lvms))
+ logging.info("Phase 1 => passed")
+
+
+ def separate_first_guest():
+ """
+ Separate memory of the first guest by generating special random series
+ """
+ logging.info("Starting phase 2: Split the pages on the first guest")
+
+ lsessions[0].sendline("mem.static_random_fill()")
+
+ (match, data) = lsessions[0].read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ 120 * perf_ratio)
+ if match == 1:
+ raise error.TestFail("Could not fill memory by random on guest %s"
+ "\nOutput:\n%s" % (vm.name, data))
+
+ if match == None:
+ raise error.TestFail("Generating random series timeout on "
+ "guest %s" % vm.name)
+
+ r_msg = data.splitlines()[-1]
+ logging.debug("Return message of static_random_fill: %s", r_msg)
+ out = int(r_msg.split()[4])
+ logging.info("PERFORMANCE: %dMB * 1000 / %dms = %dMB/s", ksm_size, out,
+ (ksm_size * 1000 / out))
+ logging.info(kvm_test_utils.get_memory_info(lvms))
+ logging.info("Phase 2 => passed")
+
+
+ def split_guest():
+ """
+ Sequential split of pages on guests up to memory limit
+ """
+ logging.info("Starting phase 3a: Sequentional split of pages on "
+ "guests up to memory limit")
+ last_vm = 0
+ session = None
+ vm = None
+ for i in range(1, vmsc):
+ vm = lvms[i]
+ session = lsessions[i]
+ session.sendline("mem.static_random_fill()")
+
+ out = ""
+ while not out.startswith("PASS") and not out.startswith("FAIL"):
+
+ free_mem = int(utils.read_from_meminfo("MemFree"))
+ if (ksm_swap):
+ free_mem = (free_mem +
+ int(utils.read_from_meminfo("SwapFree")))
+ logging.debug("FreeMem = %d" % (free_mem))
+ # We need to keep some memory for python to run.
+
+ if (free_mem < 64000) or (ksm_swap and free_mem < (450000*perf_ratio)):
+ logging.debug("Only %s free memory, killing 0 - %d hosts" %
+ (free_mem, (i-1)))
+ vm.send_monitor_cmd('stop')
+ for j in range(0, i):
+ lvms[j].destroy(gracefully = False)
+ time.sleep(20)
+ vm.send_monitor_cmd('c')
+ last_vm = i
+ break
+ out = session.read_nonblocking(0.1)
+
+ if last_vm != 0:
+ break
+
+ logging.info("Memory filled by the guest %s" % (vm.name))
+ logging.info("Phase 3a => passed")
+
+ # Check if memory in max loading guest is right
+ logging.info("Starting phase 3b")
+
+ # Kill rest of machines
+ for i in range(last_vm + 1, vmsc):
+ lsessions[i].close()
+ if i == (vmsc - 1):
+ logging.info(kvm_test_utils.get_memory_info([lvms[i]]))
+ lvms[i].destroy(gracefully = False)
+
+ # Verify last machine with randomly generated memory
+ session.sendline("mem.static_random_verify()")
+ (match, data) = session.read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ (mem / 200 * 50 * perf_ratio))
+ if (match == 1):
+ raise error.TestError("Memory error dump: %s" % data)
+ logging.info(kvm_test_utils.get_memory_info([lvms[last_vm]]))
+ (status,data) = lsessions[i].get_command_status_output("die()",20)
+ lvms[last_vm].destroy(gracefully = False)
+ logging.info("Phase 3b => passed")
+
+
+ def split_parallel():
+ """
+ Parallel page spliting
+ """
+ logging.info("Phase 1: parallel page spliting")
+ # We have to wait until allocator is finished (it waits 5 seconds to
+ # clean the socket
+
+ session = lsessions[0]
+ vm = lvms[0]
+ for i in range(1,max_alloc):
+ lsessions.append(kvm_utils.wait_for(vm.remote_login, 360, 0, 2))
+ if not lsessions[i]:
+ raise error.TestFail("Could not log into guest %s" %
+ lvms[i].name)
+
+ ret = session.get_command_status("swapoff -a", timeout=300)
+ if ret is None or ret:
+ raise error.TestFail("Failed to swapoff in the %s" % (vm.name))
+
+ for i in range(0, max_alloc):
+ lsessions[i].sendline("python /tmp/allocator.py")
+ (ret,data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"], (60 * perf_ratio))
+ if ret == None:
+ raise error.TestFail("Could not run vksmd in guest %s" %
+ vm.name)
+ if ret == 1:
+ raise error.TestFail("Could not run allocator in %s errno: %d"
+ "\nOutput:\n%s" % (vm.name, ret, data))
+
+ logging.info("Phase 4a: Simultaneous merging")
+ for i in range(0, max_alloc):
+ logging.info("Memory to guest allocator = %dMB" %
+ (ksm_size / max_alloc))
+ lsessions[i].sendline("mem = MemFill(%d, %s, %s)" %
+ ((ksm_size / max_alloc), skeys[i], dkeys[i]))
+
+ for i in range(0, max_alloc):
+ (match, data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"], (60 * perf_ratio))
+ if match == 1:
+ raise error.TestFail("Could not allocate memory on guest %s"
+ "\nOutput:\n%s" % (vm.name, data))
+
+ for i in range(0, max_alloc):
+ lsessions[i].sendline("mem.value_fill(%d)" % (skeys[0]))
+
+ for i in range(0, max_alloc):
+ (match,data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"], (90 * perf_ratio))
+ if match == 1:
+ raise error.TestFail("Could not allocate memory on guest %s"
+ "\nOutput:\n%s" % (vm.name, data))
+ # Wait until kksmd merges the pages (3 * ksm_size / 3)
+ shm = 0
+ i = 0
+ while shm < ksm_size:
+ if i > 64:
+ logging.info(kvm_test_utils.get_memory_info(lvms))
+ raise error.TestError("SHM didn't merge the memory until DL")
+ logging.debug("Sleep(%d)" % (ksm_size / 200 * perf_ratio))
+ time.sleep(ksm_size / 200 * perf_ratio)
+ shm = vm.get_shared_meminfo()
+ i = i + 1
+ logging.info(kvm_test_utils.get_memory_info([vm]))
+
+ logging.info("Phases 4b: Simultaneous spliting")
+ # Actual splitting
+ for i in range(0, max_alloc):
+ lsessions[i].sendline("mem.static_random_fill()")
+
+ for i in range(0, max_alloc):
+ (match,data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"], (90 * perf_ratio))
+ if match == 1:
+ raise error.TestFail("Could not fill memory with random "
+ "pattern on guest %s\n"
+ "Output:\n%s" % (vm.name, data))
+
+ if match == None:
+ raise error.TestFail("Generating random series timeout on "
+ "guest %s" % (vm.name))
+
+ data = data.splitlines()[-1]
+ logging.debug(data)
+ out = int(data.split()[4])
+ logging.info("PERFORMANCE: %dMB * 1000 / %dms = %dMB/s" %
+ ((ksm_size / max_alloc), out,
+ (ksm_size * 1000 / out / max_alloc)))
+ logging.info(kvm_test_utils.get_memory_info([vm]))
+
+ logging.info("Phase 4c: Simultaneous verification")
+ for i in range(0, max_alloc):
+ lsessions[i].sendline("mem.static_random_verify()")
+ for i in range(0, max_alloc):
+ (match, data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ (mem / 200 * 50 * perf_ratio))
+ if (match == 1):
+ raise error.TestError("Memory error dump: %s" % data)
+
+ logging.info("Phases 4d: Simultaneous merging")
+
+ # Actual splitting
+ for i in range(0, max_alloc):
+ lsessions[i].sendline("mem.value_fill(%d)" % (skeys[0]))
+ for i in range(0, max_alloc):
+ (match, data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ (120 * perf_ratio))
+ if match == 1:
+ raise error.TestFail("Could not fill memory by random on guest "
+ "%s\nOutput:\n%s" % vm.name)
+ logging.info(kvm_test_utils.get_memory_info([vm]))
+
+ logging.info("Phase 4e: Simultaneous verification")
+ for i in range(0, max_alloc):
+ lsessions[i].sendline("mem.value_fill(%d)" % (skeys[0]))
+ for i in range(0, max_alloc):
+ (match, data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ (mem / 200 * 50 * perf_ratio))
+ if match == 1:
+ raise error.TestError("Memory error dump: %s" % data)
+
+ logging.info("Phases 4f: Simultaneous spliting last 96B")
+
+ # Actual splitting
+ for i in range(0, max_alloc):
+ lsessions[i].sendline("mem.static_random_fill(96)")
+
+ for i in range(0, max_alloc):
+ (match, data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ (60 * perf_ratio))
+ if match == 1:
+ raise error.TestFail("Could not fill memory by zero on guest "
+ "%s\nOutput:\n%s" % (vm.name))
+
+ if match == None:
+ raise error.TestFail("Generating random series timeout on "
+ "guest %s" % vm.name)
+
+ data = data.splitlines()[-1]
+ out = int(data.split()[4])
+ logging.info("PERFORMANCE: %dMB * 1000 / %dms = %dMB/s",
+ ksm_size/max_alloc, out,
+ (ksm_size * 1000 / out / max_alloc))
+ logging.info(kvm_test_utils.get_memory_info([vm]))
+
+ logging.info("Phase 4g: Simultaneous verification last 96B")
+ for i in range(0, max_alloc):
+ lsessions[i].sendline("mem.static_random_verify(96)")
+ for i in range(0, max_alloc):
+ (match,data) = lsessions[i].read_until_last_line_matches(
+ ["PASS:","FAIL:"],
+ (mem / 200 * 50 * perf_ratio))
+ if (match == 1):
+ raise error.TestError("Memory error dump: %s" % data)
+
+ logging.info(kvm_test_utils.get_memory_info([vm]))
+
+ logging.info("Phase 4 => passed")
+
+ # Clean up
+ for i in range(0, max_alloc):
+ lsessions[i].get_command_status_output("die()", 20)
+ session.close()
+ vm.destroy(gracefully = False)
+
+
+ # Main test code
+ logging.info("Starting phase 0: Initialization")
+ # host_reserve: mem reserve kept for the host system to run
+ host_reserve = 256
+ # guest_reserve: mem reserve which is not used by allocator on the
+ # guests
+ guest_reserve = 256
+ max_vms = params.get("max_vms")
+ if max_vms:
+ max_vms = int(max_vms)
+ else:
+ max_vms = 2
+
+ overcommit = params.get("ksm_overcommit_ratio")
+ if overcommit:
+ overcommit = float(overcommit)
+ else:
+ overcommit = 2.0
+
+ max_alloc = params.get("ksm_parallel_ratio")
+ if max_alloc:
+ max_alloc = int(max_alloc)
+ else:
+ max_alloc = 1
+
+ # vmsc: count of all used VMs
+ vmsc = int(overcommit) + 1
+ vmsc = max(vmsc, max_vms)
+
+ if (params['ksm_test_size'] == "serial"):
+ max_alloc = vmsc
+
+ host_mem = (int(utils.memtotal()) / 1024 - host_reserve)
+
+ ksm_swap = False
+ if params.get("ksm_swap") == "yes":
+ ksm_swap = True
+
+ # Performance ratio
+ perf_ratio = params.get("ksm_perf_ratio")
+ if perf_ratio:
+ perf_ratio = float(perf_ratio)
+ else:
+ perf_ratio = 1
+
+ if (params['ksm_test_size'] == "parallel"):
+ vmsc = 1
+ overcommit = 1
+ mem = host_mem
+ # 32bit system adjustment
+ if not params['image_name'].endswith("64"):
+ logging.debug("Probably i386 guest architecture, "
+ "max allocator mem = 2G")
+ # Guest can have more than 2G but
+ # kvm mem + 1MB (allocator itself) can't
+ if (host_mem > 3100):
+ mem = 3100
+
+ if os.popen("uname -i").readline().startswith("i386"):
+ logging.debug("Host is i386 architecture, max guest mem is 2G")
+ # Guest system with qemu overhead (64M) can't have more than 2G
+ if mem > 3100 - 64:
+ mem = 3100 - 64
+
+ else:
+ # mem: Memory of the guest systems. Maximum must be less than
+ # host's physical ram
+ mem = int(overcommit * host_mem / vmsc)
+
+ # 32bit system adjustment
+ if not params['image_name'].endswith("64"):
+ logging.debug("Probably i386 guest architecture, "
+ "max allocator mem = 2G")
+ # Guest can have more than 2G but
+ # kvm mem + 1MB (allocator itself) can't
+ if mem - guest_reserve - 1 > 3100:
+ vmsc = int(math.ceil((host_mem * overcommit) /
+ (3100 + guest_reserve)))
+ mem = int(math.floor(host_mem * overcommit / vmsc))
+
+ if os.popen("uname -i").readline().startswith("i386"):
+ logging.debug("Host is i386 architecture, max guest mem is 2G")
+ # Guest system with qemu overhead (64M) can't have more than 2G
+ if mem > 3100 - 64:
+ vmsc = int(math.ceil((host_mem * overcommit) /
+ (3100 - 64.0)))
+ mem = int(math.floor(host_mem * overcommit / vmsc))
+
+ logging.debug("Checking KSM status...")
+ ksm_flag = 0
+ for line in os.popen('ksmctl info').readlines():
+ if line.startswith('flags'):
+ ksm_flag = int(line.split(' ')[1].split(',')[0])
+ if int(ksm_flag) != 1:
+ logging.info("KSM module is not loaded! Trying to load module and "
+ "start ksmctl...")
+ try:
+ utils.run("modprobe ksm")
+ utils.run("ksmctl start 5000 100")
+ except error.CmdError, e:
+ raise error.TestFail("Failed to load KSM: %s" % e)
+ logging.debug("KSM module loaded and ksmctl started")
+
+ logging.info("overcommit = %f", overcommit)
+ logging.info("true overcommit = %f ", (float(vmsc * mem) /
+ float(host_mem)))
+ logging.info("host mem = %dM", host_mem)
+ logging.info("mem = %dM", mem)
+ logging.info("using swap = %s", ksm_swap)
+ swap = int(utils.read_from_meminfo("SwapTotal")) / 1024
+ logging.info("swap = %dM", swap)
+ logging.info("max_vms = %d", max_vms)
+ logging.info("vmsc = %d", vmsc)
+ logging.info("performance_ratio = %f", perf_ratio)
+
+ # Generate unique keys for random series
+ skeys = []
+ dkeys = []
+ for i in range(0, max(vmsc, max_alloc)):
+ key = random.randrange(0, 255)
+ while key in skeys:
+ key = random.randrange(0, 255)
+ skeys.append(key)
+
+ key = random.randrange(0, 999)
+ while key in dkeys:
+ key = random.randrange(0, 999)
+ dkeys.append(key)
+
+ lvms = []
+ lsessions = []
+
+ # As we don't know the number and memory amount of VMs in advance,
+ # we need to specify and create them here (FIXME: not a nice thing)
+ vm_name = params.get("main_vm")
+ params['mem'] = mem
+ params['vms'] = vm_name
+ # Associate pidfile name
+ params['pid_' + vm_name] = kvm_utils.generate_tmp_file_name(vm_name,
+ 'pid')
+ if not params.get('extra_params'):
+ params['extra_params'] = ' '
+ params['extra_params_' + vm_name] = params.get('extra_params')
+ params['extra_params_' + vm_name] += (" -pidfile %s" %
+ (params.get('pid_' + vm_name)))
+ params['extra_params'] = params.get('extra_params_'+vm_name)
+
+ # ksm_size: amount of memory used by allocator
+ ksm_size = mem - guest_reserve
+ logging.info("ksm_size = %dM" % (ksm_size))
+
+ # Creating the first guest
+ kvm_preprocessing.preprocess_vm(test, params, env, vm_name)
+ lvms.append(kvm_utils.env_get_vm(env, vm_name))
+ if not lvms[0]:
+ raise error.TestError("VM object not found in environment")
+ if not lvms[0].is_alive():
+ raise error.TestError("VM seems to be dead; Test requires a living "
+ "VM")
+
+ logging.info("Booting the first guest %s", lvms[0].name)
+
+ lsessions.append(kvm_utils.wait_for(lvms[0].remote_login, 360, 0, 2))
+ if not lsessions[0]:
+ raise error.TestFail("Could not log into first guest")
+ # Associate vm PID
+ try:
+ tmp = open(params.get('pid_' + vm_name), 'r')
+ params['pid_' + vm_name] = int(tmp.readline())
+ except:
+ raise error.TestFail("Could not get PID of %s" % (vm_name))
+
+ # Creating other guest systems
+ for i in range(1, vmsc):
+ vm_name = "vm" + str(i + 1)
+ params['pid_' + vm_name] = kvm_utils.generate_tmp_file_name(vm_name,
+ 'pid')
+ params['extra_params_' + vm_name] = params.get('extra_params')
+ params['extra_params_' + vm_name] += (" -pidfile %s" %
+ (params.get('pid_' + vm_name)))
+ params['extra_params'] = params.get('extra_params_' + vm_name)
+
+ # Last VM is later used to run more allocators simultaneously
+ lvms.append(lvms[0].clone(vm_name, params))
+ kvm_utils.env_register_vm(env, vm_name, lvms[i])
+ params['vms'] += " " + vm_name
+
+ logging.info("Booting guest %s" % lvms[i].name)
+ if not lvms[i].create():
+ raise error.TestFail("Cannot create VM %s" % lvms[i].name)
+ if not lvms[i].is_alive():
+ raise error.TestError("VM %s seems to be dead; Test requires a"
+ "living VM" % lvms[i].name)
+
+ lsessions.append(kvm_utils.wait_for(lvms[i].remote_login, 360, 0, 2))
+ if not lsessions[i]:
+ raise error.TestFail("Could not log into guest %s" %
+ lvms[i].name)
+ try:
+ tmp = open(params.get('pid_' + vm_name), 'r')
+ params['pid_' + vm_name] = int(tmp.readline())
+ except:
+ raise error.TestFail("Could not get PID of %s" % (vm_name))
+
+ # Let systems take a rest :-)
+ time.sleep(vmsc * 2 * perf_ratio)
+ logging.info(kvm_test_utils.get_memory_info(lvms))
+
+ # Copy allocator.py into guests
+ pwd = os.path.join(os.environ['AUTODIR'],'tests/kvm')
+ vksmd_src = os.path.join(pwd, "scripts/allocator.py")
+ dst_dir = "/tmp"
+ for vm in lvms:
+ if not vm.copy_files_to(vksmd_src, dst_dir):
+ raise error.TestFail("copy_files_to failed %s" % vm.name)
+ logging.info("Phase 0 => passed")
+
+ if params['ksm_test_size'] == "parallel":
+ split_parallel()
+ elif params['ksm_test_size'] == "serial":
+ initialize_guests()
+ separate_first_guest()
+ split_guest()
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index b8f25f4..24c789d 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -270,6 +270,24 @@ variants:
type = physical_resources_check
catch_uuid_cmd = dmidecode | awk -F: '/UUID/ {print $2}'
+ - ksm_overcommit:
+ # Don't preprocess any vms as we need to change its params
+ vms = ''
+ image_snapshot = yes
+ kill_vm_gracefully = no
+ type = ksm_overcommit
+ ksm_swap = yes
+ no hugepages
+ # Overcommit of host memmory
+ ksm_overcommit_ratio = 3
+ # Max paralel runs machine
+ ksm_parallel_ratio = 4
+ variants:
+ - ksm_serial
+ ksm_test_size = "serial"
+ - ksm_parallel
+ ksm_test_size = "parallel"
+
# NICs
variants:
- @rtl8139:
--
1.6.5.2
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
