Hi, Updated few programs required for ginstaller (both in HPC and EP mode). Changelog: * 'gmountmap' changed to give out mount point spec at destination. * 'dest-hdd' rewritten to give out a script, which includes o partitioning o mounting different mountpoints o customizing fstab ? 'untar'ing and 'bootloader' part to be added * some small bug fixes in 'gmountmap' and 'gprobe' * TODO updated for this release Regards, Amar
Index: TODO =================================================================== RCS file: /cvsroot/gluster/gluster-alpha1/TODO,v retrieving revision 1.5 diff -p -u -r1.5 TODO --- TODO 4 Feb 2006 08:40:59 -0000 1.5 +++ TODO 28 Mar 2006 09:51:57 -0000 @@ -1,15 +1,24 @@ +TODO: -* complete configure.ac with the requirements written in comments -* put proper kernel config's to i386 and x86_64 kernels for a latest one -* convert core extension libs to `distutils' based package -* try building gluster on every possible distro and version to catch - build bugs - Currently tested on : <update this list if u successfully built on new> - 1. Slackware 10.2 i686 - 2. Debian Sarge i686 - 3. Scientific Linux x86_64 -* convert all scripts (python, shell, any) which refer to /usr/share with - @DATADIR@ and rename them to filename.in -* <DEL> key generates ~ in Live mode. fix it (some kind of terminal setting) -* make `gex' tool ignore empty lines and handle invalid input from a .gex - meta file +* GlusterHPC Scripts (High level script to control the flow) +* dest-hdd ( Untarring of the stream[action mode], Calling 'gmountmap' ) +* gpart (Fixes related to partition creation) - +* bootmanager (UI, destination script) +* Validating partition table after 'gpart', for boundary cases. +* Stage3 from different sources (Probing from cd is supported currently) +* 'fstab' creation (LABLE option) +* Network Config (code complete, framework, placement to be decided) +* HPC Apps (Native compilation) +* Picking a client node (for EP) + +* Gluster EP + - Ask repository + - Client Kick start + - source-tgz, dest-tgz scripts + +* Adding 'nfsd' to build (stage3, use it in tarring) + +* Testing + - Major work, test for all the simple -> complex combinations + +* UDPCAST (think about deamonizing it in clients) Index: autogen.sh =================================================================== RCS file: /cvsroot/gluster/gluster-alpha1/autogen.sh,v retrieving revision 1.4 diff -p -u -r1.4 autogen.sh --- autogen.sh 14 Feb 2006 08:16:46 -0000 1.4 +++ autogen.sh 28 Mar 2006 09:51:57 -0000 @@ -5,5 +5,5 @@ autoconf automake --add-missing mkdir -p tarballs -wget -Ptarballs/ -nH ftp://ftp.zresearch.com/pub/gluster/external/* +#wget -Ptarballs/ -nH ftp://ftp.zresearch.com/pub/gluster/external/* Index: extensions/ginstaller/dest-hdd =================================================================== RCS file: /cvsroot/gluster/gluster-alpha1/extensions/ginstaller/dest-hdd,v retrieving revision 1.4 diff -p -u -r1.4 dest-hdd --- extensions/ginstaller/dest-hdd 9 Mar 2006 07:36:41 -0000 1.4 +++ extensions/ginstaller/dest-hdd 28 Mar 2006 09:51:58 -0000 @@ -1,186 +1,117 @@ #!/usr/bin/python -import os, string, sys, getopt -from Gluster.GFrontEnd import dialog -from Gluster.GTmp import Dir -from Gluster import gmap - -dlg = dialog.Dialog () - -known_fs_types = ['reiserfs', 'ext3', 'ext2', 'xfs', 'jfs', 'swap'] - -port_base = 23465 -xfer_cmd = 'nc 127.0.0.1 %d' -fg = True -_parts = {} +try: + import sys, os, getopt, string +except ImportError, msg: + print "Error: ", msg + +flag = '' + +main_script = 'temp.sh' + +def sort_fn (line1): + return len (line1.split (":")[2]) def uniq (list): set = {} map (set.__setitem__, list, []) return set.keys () -def is_labels_ok (new_labels): - for p in new_labels.keys (): - if new_labels[p] == '': - dlg.msgbox ('label for %s is empty' % p) - return False - if new_labels.values ().count (new_labels[p]) != 1: - dlg.msgbox ('label %s used more than once' % new_labels[p]) - return False - return True - -def set_labels (_parts): - new_labels = {} - for p in _parts.keys (): - if _parts[p]['XFER'] == 'Y': - new_labels[p] = _parts[p]['LABEL'] - - for p in new_labels.keys (): - if new_labels[p] == '': - i = 1 - while True: - p_new = 'UNKNOWN%d' % i - if not p_new in new_labels.values (): - break - i = i + 1 - if _parts[p]['OS'] != '': - p_new = _parts[p]['OS'] - if p_new in new_labels.values (): - stem = p_new + '%d' - i = 1 - while True: - p_new = stem % i - if p_new not in new_labels.values (): - break - i = i + 1 - new_labels[p] = p_new - continue - for os in _parts.keys (): - if not 'FSTAB' in _parts[os].keys (): - continue - if p in _parts[os]['FSTAB'].keys (): - p_new = _parts[os]['FSTAB'][p] - if p_new in new_labels.values (): - stem = p_new + '%d' - i = 1 - while True: - p_new = stem % i - if not p_new in new_labels.values (): - break - i = i + 1 - new_labels[p] = p_new +def main (): - sel = '' + global flag + inp_file = 'tmpfile' + + (opt, args) = getopt.getopt (sys.argv[1:], "ahvi:", + ["auto", "help", "version", "input="]) + for (o, v) in opt: + if o == '-a' or o == '--auto': + flag = '--auto' + if o == '-i' or o == '--input': + inp_file = val + +# os.system ("../gpartdump/gprobe | ./source-hdd") + os.system ("cat %s | ../gpartdump/gmountmap %s \ + -d dump -m mnt -t tx -c part" % (inp_file, flag)) + + f = open (main_script, "w") + + f.write ("#!/bin/sh\n\n") + f.write ("# Partition \n") + tmpf = open ("part", "r") while True: - (ret, sel) = dlg.menu ('set unique labels to all partitions', - choices=new_labels.items (), - cancel='done', - ok_label='edit', - default_item=sel, - defaultno=True) - if ret != 0: - if is_labels_ok (new_labels): - break - continue - (ret, new_label) = dlg.inputbox ('input new label for %s' % sel, - init=new_labels[sel]) - if ret != 0: + line = tmpf.readline () + if line == '': + break + f.write (line) + tmpf.close () + f.write ("# End of Partitioning \n\n") + + f.write ("# Mounting\n") + tmpf = file ("mnt", "r") + tmpf1 = file ("tx", "r") + lines1 = tmpf.readlines () + lines2 = tmpf1.readlines () + lines = uniq (lines1 + lines2) + lines.sort (key=sort_fn) + for line in lines: + if line == '': + break + array = line.strip ().split (":") + if array[2] == 'swap': continue - new_labels[sel] = new_label - return - -def read_parts (): - _parts = {} - all_parts = sys.stdin.readlines () - for part in all_parts: - pieces = part.strip ().split (':') - if pieces[0] == 'fstab': - if not 'FSTAB' in _parts[pieces[1]].keys (): - _parts[pieces[1]]['FSTAB'] = {} - _parts[pieces[1]]['FSTAB'][pieces[2]] = pieces[3] + mp = "/tmp/ginstaller/%s%s" % (array[1], array[2]) + f.write ("mkdir -p %s\n" % mp) + f.write ("mount %s %s\n" % (array[0], mp)) + tmpf.close () + tmpf1.close () + f.write ("# End of Mounting section\n\n") + + f.write ("# Untarring\n") + + f.write ("# End of untarring section \n\n") + + f.write ("# fstab \n") + # first delete all the entrys in 'tx', later add all in 'mnt' + tmpf = open ("tx", "r") + swap_flag = True + fstab_prev = '' + for line in tmpf.readlines (): + array = line.strip ().split (":") + chroot = "/tmp/ginstaller/%s" % array[1] + fstab = "%s/etc/fstab" % chroot + f.write ("sed -i -e '\#[ \\t]%s[ \\t]#d' %s\n" % (array[2], fstab)) + # Handle 'swap' partition seperately + if fstab != fstab_prev: + swap_flag == True + fstab_prev = fstab + if swap_flag == True: + f.write ("sed -i -e '\#[ \\t]swap[ \\t]#d' %s\n" % fstab) + swap_flag = False + tmpf.close () + tmpf = open ("mnt", "r") + while True: + line = tmpf.readline () + if line == '': + break + array = line.strip ().split (":") + chroot = "/tmp/ginstaller/%s" % array[1] + fstab = "%s/etc/fstab" % chroot + if array[2] == '/': + f.write ("echo \"%s %s auto defaults 1 1\" >> %s\n" + % (array[0], array[2], fstab)) + elif array[2] == 'swap': + f.write ("echo \"%s none swap defaults 0 0\" >> %s\n" % + (array[0], fstab)) else: - if not pieces[0] in _parts.keys (): - _parts[pieces[0]] = {} - _parts[pieces[0]]['TYPE'] = pieces[1] - _parts[pieces[0]]['SIZE_MB'] = pieces[2] - _parts[pieces[0]]['LABEL'] = pieces[3] - _parts[pieces[0]]['OS'] = pieces[4] - _parts[pieces[0]]['DESC'] = pieces[5] - _parts[pieces[0]]['XFER'] = pieces[6] - return _parts - -def get_mpoint (part): - hnd = os.popen ("mount | grep '^ *%s ' | awk '{print $3}'" % part) - line = hnd.readline ().strip () - hnd.close () - if line: - return (False, line) - else: - tmpdir = Dir.TmpDir () - tmpdir.SetSticky (True) - ret = os.system ("mount %s %s 2>/dev/null" % (part, tmpdir.Name ())) - if ret: - return (False, None) - return (True, tmpdir.Name ()) - -def xfer_fn (pair): - global xfer_cmd - (need_umount, mp) = get_mpoint (pair[1]) - if not mp: - return - tar_cmd = 'tar -C %s -c .' % mp - full_cmd = '%s | %s' % (tar_cmd, xfer_cmd) - os.system (full_cmd % pair[0]) - - if need_umount: - os.system ("umount %s" % mp) + f.write ("echo \"%s %s auto defaults 1 2\" >> %s\n" + % (array[0], array[2], fstab)) + + tmpf.close () + f.write ("# End of fstab section \n\n") + + f.close () + return -def main (): - global port_base - global xfer_cmd - global fg - global _parts - try: - (opt, args) = getopt.getopt (sys.argv[1:], "rx:c:p:f", - ["recv", - "xfer=", - "conf=", - "port-base=", - "forground"]) - except getopt.GetoptError, (msg, opt): - print msg - sys.exit (1) - - do_xfer = False - - conf_to = 'cat' - for (o,val) in opt: - if o == '-r' or o == '--recv': - do_xfer = True - if o == '-x' or o == '--xfer': - xfer_cmd = val - if o == '-c' or o == '--conf': - conf_to = val - if o == '-p' or o == '--port-base': - port_base = string.atoi (val) - if o == '-f' or o == '--foreground': - fg = True - - _parts = read_parts () - - if do_xfer: - dev_list = [] - idx = 0 - for _part in _parts.keys (): - if _parts[_part]['XFER'] == 'Y': - idx += 2 - dev_list.append ((port_base + idx, _part)) - if fg == False and os.fork () != 0: - sys.exit (1) - gmap.gmap (xfer_fn, dev_list) - else: - new_labels = set_labels (_parts) - main () - Index: extensions/ginstaller/source-hdd =================================================================== RCS file: /cvsroot/gluster/gluster-alpha1/extensions/ginstaller/source-hdd,v retrieving revision 1.6 diff -p -u -r1.6 source-hdd --- extensions/ginstaller/source-hdd 21 Mar 2006 06:13:02 -0000 1.6 +++ extensions/ginstaller/source-hdd 28 Mar 2006 09:51:58 -0000 @@ -114,7 +114,7 @@ def main (): do_xfer = False - conf_to = 'cat' + conf_to = "tmpfile" for (o,val) in opt: if o == '-s' or o == '--send': do_xfer = True @@ -144,7 +144,10 @@ def main (): if not parts_selected: dlg.msgbox ("no partitions available for replication") return - cfd = os.popen (conf_to, "w") + cfd = file (conf_to, "w") + + for part in parts_selected: + _parts[part]['XFER'] = 'Y' write_dump (cfd, partlist_gdump (_parts)) return Index: extensions/gpartdump/gmountmap =================================================================== RCS file: /cvsroot/gluster/gluster-alpha1/extensions/gpartdump/gmountmap,v retrieving revision 1.5 diff -p -u -r1.5 gmountmap --- extensions/gpartdump/gmountmap 27 Mar 2006 10:23:07 -0000 1.5 +++ extensions/gpartdump/gmountmap 28 Mar 2006 09:51:59 -0000 @@ -17,8 +17,7 @@ auto = False # # # os_dict = { 'old_os_part' : { 'FSTAB' : { 'mp1' : [ None|'new_part', -# is_sticky, (2 mpoints), -# is_merged] }, +# is_sticky, (2 mpoints)]} # 'SWAP' : [ swap1, swap2 ], # 'OS' : "OpSys Name" }} # @@ -241,7 +240,11 @@ def run_map (os_dict, part_dict, source_ if nmp in os_dict[os]['FSTAB'].keys (): dlg.msgbox ('mount point (%s) exists.' % nmp) continue - + + if len (nmp) == 0 or nmp[0] != '/': + dlg.msgbox ('invalid mountpoint %s' % nmp) + continue + if nmp in ['/etc', '/lib', '/dev', @@ -404,6 +407,13 @@ def run_map (os_dict, part_dict, source_ break return +def get_key_from_os (part): + global key_to_os + for key in key_to_os.keys (): + if key_to_os[key] == part: + return key + return None + def fix_fstab (dest_partlist, os_dict): for p in dest_partlist.keys (): dest_partlist[p]['FSTAB'] = {} @@ -415,11 +425,26 @@ def fix_fstab (dest_partlist, os_dict): if not stuff[0]: continue dest_partlist[os_part]['FSTAB'][stuff[0]] = mp + dest_partlist[os_part]['OS'] = os_dict[os]['OS'] if 'SWAP' in os_dict[os].keys (): for sp in os_dict[os]['SWAP']: dest_partlist[os_part]['FSTAB'][sp] = 'swap' -def write_mountmap (map_fd, source_partlist, part_dict): +def write_mount (mount_fd, os_dict): + for k1,v1 in os_dict.iteritems (): + for k2,v2 in v1['FSTAB'].iteritems (): + if v2[0] is not None : + mount_fd.write ("%s:%s:%s\n" % (v2[0], + get_key_from_os(k1), + k2)) + for i in range (len (v1['SWAP'])): + mount_fd.write ("%s:%s:swap\n" % (v1['SWAP'][i], + get_key_from_os(k1))) + + + return + +def write_map (map_fd, source_partlist, part_dict): parts = source_partlist.keys () parts.sort () key_to_os_sorted = key_to_os.keys () @@ -459,12 +484,14 @@ def main (): dump_fd = sys.stdout commands_fd = sys.stdout map_fd = sys.stdout - (opt, args) = getopt.getopt (sys.argv[1:], "ai:d:c:m:", + mount_fd = sys.stdout + (opt, args) = getopt.getopt (sys.argv[1:], "ai:d:c:t:m:", ["auto", "input=", "dump=", "commands=", - "mountmap="]) + "transfer=", + "mount="]) for (o, val) in opt: if o == '-a' or o == '--auto': @@ -484,11 +511,16 @@ def main (): input_fd = file (val) except: sys.exit (1) - if o == '-m' or o == '--mountmap': + if o == '-t' or o == '--transfer': try: map_fd = file (val, "w") except: sys.exit (1) + if o == '-m' or o == '--mount': + try: + mount_fd = file(val, "w") + except: + sys.exit (1) source_dump = read_gdump (input_fd) source_partlist = gdump_partlist (source_dump) @@ -511,7 +543,9 @@ def main (): write_dump (dump_fd, partlist_gdump (dest_partlist)) - write_mountmap (map_fd, source_partlist, part_dict) + write_map (map_fd, source_partlist, part_dict) + + write_mount (mount_fd, os_dict) if auto: d = Dir.TmpDir () @@ -529,6 +563,7 @@ def main (): dump_fd.close () map_fd.close () + mount_fd.close () commands_fd.close () return Index: extensions/gpartdump/gprobe =================================================================== RCS file: /cvsroot/gluster/gluster-alpha1/extensions/gpartdump/gprobe,v retrieving revision 1.3 diff -p -u -r1.3 gprobe --- extensions/gpartdump/gprobe 23 Mar 2006 07:14:16 -0000 1.3 +++ extensions/gpartdump/gprobe 28 Mar 2006 09:51:59 -0000 @@ -33,12 +33,12 @@ def get_bootloader (part, _fstab): bootloader = {} if os.access ("%s/etc/lilo.conf" % mpoint, os.R_OK): - hnd = os.popen ("grep 'boot *=' %s/etc/lilo.conf | cut -f 2 -d '='" - % mpoint) - bootloader['lilo'] = hnd.readline ().strip () - hnd.close () + hnd = os.popen ("grep 'boot *=' %s/etc/lilo.conf | cut -f 2 -d '='" + % mpoint) + bootloader['lilo'] = hnd.readline ().strip () + hnd.close () if os.access ("%s/boot/grub/menu.lst" % mpoint, os.R_OK): - bootloader['grub'] = 'default' + bootloader['grub'] = 'default' # If /boot is mapped to another partition, then check for # grub/menu.lst in the path for k, v in _fstab.iteritems (): @@ -50,14 +50,14 @@ def get_bootloader (part, _fstab): if os.access ("%s/grub/menu.lst" % tmp_mpoint, os.R_OK): bootloader['grub'] = 'default' unget_mpoint (tmp_need_umount, tmp_mpoint) - + if os.access ("%s/ntldr" % mpoint, os.R_OK) \ or os.access ("%s/boot.ini" % mpoint, os.R_OK): bootloader['windows'] = part #need to check this still - + unget_mpoint (need_umount, mpoint) return bootloader - + def get_fstab (part, _parts): (need_umount, mpoint) = get_mpoint (part)
