[Gluster-devel] Updates in dest-hdd, gmountmap etc.
Amar S. Tumballi
amar at zresearch.com
Tue Mar 28 10:05:49 UTC 2006
Hi,
Updated few programs required for ginstaller (both in HPC and EP mode).
Changelog:
* 'gmountmap' changed to give out mount point spec at destination.
* 'dest-hdd' rewritten to give out a script, which includes
o partitioning
o mounting different mountpoints
o customizing fstab
? 'untar'ing and 'bootloader' part to be added
* some small bug fixes in 'gmountmap' and 'gprobe'
* TODO updated for this release
Regards,
Amar
-------------- next part --------------
Index: TODO
===================================================================
RCS file: /cvsroot/gluster/gluster-alpha1/TODO,v
retrieving revision 1.5
diff -p -u -r1.5 TODO
--- TODO 4 Feb 2006 08:40:59 -0000 1.5
+++ TODO 28 Mar 2006 09:51:57 -0000
@@ -1,15 +1,24 @@
+TODO:
-* complete configure.ac with the requirements written in comments
-* put proper kernel config's to i386 and x86_64 kernels for a latest one
-* convert core extension libs to `distutils' based package
-* try building gluster on every possible distro and version to catch
- build bugs
- Currently tested on : <update this list if u successfully built on new>
- 1. Slackware 10.2 i686
- 2. Debian Sarge i686
- 3. Scientific Linux x86_64
-* convert all scripts (python, shell, any) which refer to /usr/share with
- @DATADIR@ and rename them to filename.in
-* <DEL> key generates ~ in Live mode. fix it (some kind of terminal setting)
-* make `gex' tool ignore empty lines and handle invalid input from a .gex
- meta file
+* GlusterHPC Scripts (High level script to control the flow)
+* dest-hdd ( Untarring of the stream[action mode], Calling 'gmountmap' )
+* gpart (Fixes related to partition creation) -
+* bootmanager (UI, destination script)
+* Validating partition table after 'gpart', for boundary cases.
+* Stage3 from different sources (Probing from cd is supported currently)
+* 'fstab' creation (LABLE option)
+* Network Config (code complete, framework, placement to be decided)
+* HPC Apps (Native compilation)
+* Picking a client node (for EP)
+
+* Gluster EP
+ - Ask repository
+ - Client Kick start
+ - source-tgz, dest-tgz scripts
+
+* Adding 'nfsd' to build (stage3, use it in tarring)
+
+* Testing
+ - Major work, test for all the simple -> complex combinations
+
+* UDPCAST (think about deamonizing it in clients)
Index: autogen.sh
===================================================================
RCS file: /cvsroot/gluster/gluster-alpha1/autogen.sh,v
retrieving revision 1.4
diff -p -u -r1.4 autogen.sh
--- autogen.sh 14 Feb 2006 08:16:46 -0000 1.4
+++ autogen.sh 28 Mar 2006 09:51:57 -0000
@@ -5,5 +5,5 @@ autoconf
automake --add-missing
mkdir -p tarballs
-wget -Ptarballs/ -nH ftp://ftp.zresearch.com/pub/gluster/external/*
+#wget -Ptarballs/ -nH ftp://ftp.zresearch.com/pub/gluster/external/*
Index: extensions/ginstaller/dest-hdd
===================================================================
RCS file: /cvsroot/gluster/gluster-alpha1/extensions/ginstaller/dest-hdd,v
retrieving revision 1.4
diff -p -u -r1.4 dest-hdd
--- extensions/ginstaller/dest-hdd 9 Mar 2006 07:36:41 -0000 1.4
+++ extensions/ginstaller/dest-hdd 28 Mar 2006 09:51:58 -0000
@@ -1,186 +1,117 @@
#!/usr/bin/python
-import os, string, sys, getopt
-from Gluster.GFrontEnd import dialog
-from Gluster.GTmp import Dir
-from Gluster import gmap
-
-dlg = dialog.Dialog ()
-
-known_fs_types = ['reiserfs', 'ext3', 'ext2', 'xfs', 'jfs', 'swap']
-
-port_base = 23465
-xfer_cmd = 'nc 127.0.0.1 %d'
-fg = True
-_parts = {}
+try:
+ import sys, os, getopt, string
+except ImportError, msg:
+ print "Error: ", msg
+
+flag = ''
+
+main_script = 'temp.sh'
+
+def sort_fn (line1):
+ return len (line1.split (":")[2])
def uniq (list):
set = {}
map (set.__setitem__, list, [])
return set.keys ()
-def is_labels_ok (new_labels):
- for p in new_labels.keys ():
- if new_labels[p] == '':
- dlg.msgbox ('label for %s is empty' % p)
- return False
- if new_labels.values ().count (new_labels[p]) != 1:
- dlg.msgbox ('label %s used more than once' % new_labels[p])
- return False
- return True
-
-def set_labels (_parts):
- new_labels = {}
- for p in _parts.keys ():
- if _parts[p]['XFER'] == 'Y':
- new_labels[p] = _parts[p]['LABEL']
-
- for p in new_labels.keys ():
- if new_labels[p] == '':
- i = 1
- while True:
- p_new = 'UNKNOWN%d' % i
- if not p_new in new_labels.values ():
- break
- i = i + 1
- if _parts[p]['OS'] != '':
- p_new = _parts[p]['OS']
- if p_new in new_labels.values ():
- stem = p_new + '%d'
- i = 1
- while True:
- p_new = stem % i
- if p_new not in new_labels.values ():
- break
- i = i + 1
- new_labels[p] = p_new
- continue
- for os in _parts.keys ():
- if not 'FSTAB' in _parts[os].keys ():
- continue
- if p in _parts[os]['FSTAB'].keys ():
- p_new = _parts[os]['FSTAB'][p]
- if p_new in new_labels.values ():
- stem = p_new + '%d'
- i = 1
- while True:
- p_new = stem % i
- if not p_new in new_labels.values ():
- break
- i = i + 1
- new_labels[p] = p_new
+def main ():
- sel = ''
+ global flag
+ inp_file = 'tmpfile'
+
+ (opt, args) = getopt.getopt (sys.argv[1:], "ahvi:",
+ ["auto", "help", "version", "input="])
+ for (o, v) in opt:
+ if o == '-a' or o == '--auto':
+ flag = '--auto'
+ if o == '-i' or o == '--input':
+ inp_file = val
+
+# os.system ("../gpartdump/gprobe | ./source-hdd")
+ os.system ("cat %s | ../gpartdump/gmountmap %s \
+ -d dump -m mnt -t tx -c part" % (inp_file, flag))
+
+ f = open (main_script, "w")
+
+ f.write ("#!/bin/sh\n\n")
+ f.write ("# Partition \n")
+ tmpf = open ("part", "r")
while True:
- (ret, sel) = dlg.menu ('set unique labels to all partitions',
- choices=new_labels.items (),
- cancel='done',
- ok_label='edit',
- default_item=sel,
- defaultno=True)
- if ret != 0:
- if is_labels_ok (new_labels):
- break
- continue
- (ret, new_label) = dlg.inputbox ('input new label for %s' % sel,
- init=new_labels[sel])
- if ret != 0:
+ line = tmpf.readline ()
+ if line == '':
+ break
+ f.write (line)
+ tmpf.close ()
+ f.write ("# End of Partitioning \n\n")
+
+ f.write ("# Mounting\n")
+ tmpf = file ("mnt", "r")
+ tmpf1 = file ("tx", "r")
+ lines1 = tmpf.readlines ()
+ lines2 = tmpf1.readlines ()
+ lines = uniq (lines1 + lines2)
+ lines.sort (key=sort_fn)
+ for line in lines:
+ if line == '':
+ break
+ array = line.strip ().split (":")
+ if array[2] == 'swap':
continue
- new_labels[sel] = new_label
- return
-
-def read_parts ():
- _parts = {}
- all_parts = sys.stdin.readlines ()
- for part in all_parts:
- pieces = part.strip ().split (':')
- if pieces[0] == 'fstab':
- if not 'FSTAB' in _parts[pieces[1]].keys ():
- _parts[pieces[1]]['FSTAB'] = {}
- _parts[pieces[1]]['FSTAB'][pieces[2]] = pieces[3]
+ mp = "/tmp/ginstaller/%s%s" % (array[1], array[2])
+ f.write ("mkdir -p %s\n" % mp)
+ f.write ("mount %s %s\n" % (array[0], mp))
+ tmpf.close ()
+ tmpf1.close ()
+ f.write ("# End of Mounting section\n\n")
+
+ f.write ("# Untarring\n")
+
+ f.write ("# End of untarring section \n\n")
+
+ f.write ("# fstab \n")
+ # first delete all the entrys in 'tx', later add all in 'mnt'
+ tmpf = open ("tx", "r")
+ swap_flag = True
+ fstab_prev = ''
+ for line in tmpf.readlines ():
+ array = line.strip ().split (":")
+ chroot = "/tmp/ginstaller/%s" % array[1]
+ fstab = "%s/etc/fstab" % chroot
+ f.write ("sed -i -e '\#[ \\t]%s[ \\t]#d' %s\n" % (array[2], fstab))
+ # Handle 'swap' partition seperately
+ if fstab != fstab_prev:
+ swap_flag == True
+ fstab_prev = fstab
+ if swap_flag == True:
+ f.write ("sed -i -e '\#[ \\t]swap[ \\t]#d' %s\n" % fstab)
+ swap_flag = False
+ tmpf.close ()
+ tmpf = open ("mnt", "r")
+ while True:
+ line = tmpf.readline ()
+ if line == '':
+ break
+ array = line.strip ().split (":")
+ chroot = "/tmp/ginstaller/%s" % array[1]
+ fstab = "%s/etc/fstab" % chroot
+ if array[2] == '/':
+ f.write ("echo \"%s %s auto defaults 1 1\" >> %s\n"
+ % (array[0], array[2], fstab))
+ elif array[2] == 'swap':
+ f.write ("echo \"%s none swap defaults 0 0\" >> %s\n" %
+ (array[0], fstab))
else:
- if not pieces[0] in _parts.keys ():
- _parts[pieces[0]] = {}
- _parts[pieces[0]]['TYPE'] = pieces[1]
- _parts[pieces[0]]['SIZE_MB'] = pieces[2]
- _parts[pieces[0]]['LABEL'] = pieces[3]
- _parts[pieces[0]]['OS'] = pieces[4]
- _parts[pieces[0]]['DESC'] = pieces[5]
- _parts[pieces[0]]['XFER'] = pieces[6]
- return _parts
-
-def get_mpoint (part):
- hnd = os.popen ("mount | grep '^ *%s ' | awk '{print $3}'" % part)
- line = hnd.readline ().strip ()
- hnd.close ()
- if line:
- return (False, line)
- else:
- tmpdir = Dir.TmpDir ()
- tmpdir.SetSticky (True)
- ret = os.system ("mount %s %s 2>/dev/null" % (part, tmpdir.Name ()))
- if ret:
- return (False, None)
- return (True, tmpdir.Name ())
-
-def xfer_fn (pair):
- global xfer_cmd
- (need_umount, mp) = get_mpoint (pair[1])
- if not mp:
- return
- tar_cmd = 'tar -C %s -c .' % mp
- full_cmd = '%s | %s' % (tar_cmd, xfer_cmd)
- os.system (full_cmd % pair[0])
-
- if need_umount:
- os.system ("umount %s" % mp)
+ f.write ("echo \"%s %s auto defaults 1 2\" >> %s\n"
+ % (array[0], array[2], fstab))
+
+ tmpf.close ()
+ f.write ("# End of fstab section \n\n")
+
+ f.close ()
+
return
-def main ():
- global port_base
- global xfer_cmd
- global fg
- global _parts
- try:
- (opt, args) = getopt.getopt (sys.argv[1:], "rx:c:p:f",
- ["recv",
- "xfer=",
- "conf=",
- "port-base=",
- "forground"])
- except getopt.GetoptError, (msg, opt):
- print msg
- sys.exit (1)
-
- do_xfer = False
-
- conf_to = 'cat'
- for (o,val) in opt:
- if o == '-r' or o == '--recv':
- do_xfer = True
- if o == '-x' or o == '--xfer':
- xfer_cmd = val
- if o == '-c' or o == '--conf':
- conf_to = val
- if o == '-p' or o == '--port-base':
- port_base = string.atoi (val)
- if o == '-f' or o == '--foreground':
- fg = True
-
- _parts = read_parts ()
-
- if do_xfer:
- dev_list = []
- idx = 0
- for _part in _parts.keys ():
- if _parts[_part]['XFER'] == 'Y':
- idx += 2
- dev_list.append ((port_base + idx, _part))
- if fg == False and os.fork () != 0:
- sys.exit (1)
- gmap.gmap (xfer_fn, dev_list)
- else:
- new_labels = set_labels (_parts)
-
main ()
-
Index: extensions/ginstaller/source-hdd
===================================================================
RCS file: /cvsroot/gluster/gluster-alpha1/extensions/ginstaller/source-hdd,v
retrieving revision 1.6
diff -p -u -r1.6 source-hdd
--- extensions/ginstaller/source-hdd 21 Mar 2006 06:13:02 -0000 1.6
+++ extensions/ginstaller/source-hdd 28 Mar 2006 09:51:58 -0000
@@ -114,7 +114,7 @@ def main ():
do_xfer = False
- conf_to = 'cat'
+ conf_to = "tmpfile"
for (o,val) in opt:
if o == '-s' or o == '--send':
do_xfer = True
@@ -144,7 +144,10 @@ def main ():
if not parts_selected:
dlg.msgbox ("no partitions available for replication")
return
- cfd = os.popen (conf_to, "w")
+ cfd = file (conf_to, "w")
+
+ for part in parts_selected:
+ _parts[part]['XFER'] = 'Y'
write_dump (cfd, partlist_gdump (_parts))
return
Index: extensions/gpartdump/gmountmap
===================================================================
RCS file: /cvsroot/gluster/gluster-alpha1/extensions/gpartdump/gmountmap,v
retrieving revision 1.5
diff -p -u -r1.5 gmountmap
--- extensions/gpartdump/gmountmap 27 Mar 2006 10:23:07 -0000 1.5
+++ extensions/gpartdump/gmountmap 28 Mar 2006 09:51:59 -0000
@@ -17,8 +17,7 @@ auto = False
#
#
# os_dict = { 'old_os_part' : { 'FSTAB' : { 'mp1' : [ None|'new_part',
-# is_sticky, (2 mpoints),
-# is_merged] },
+# is_sticky, (2 mpoints)]}
# 'SWAP' : [ swap1, swap2 ],
# 'OS' : "OpSys Name" }}
#
@@ -241,7 +240,11 @@ def run_map (os_dict, part_dict, source_
if nmp in os_dict[os]['FSTAB'].keys ():
dlg.msgbox ('mount point (%s) exists.' % nmp)
continue
-
+
+ if len (nmp) == 0 or nmp[0] != '/':
+ dlg.msgbox ('invalid mountpoint %s' % nmp)
+ continue
+
if nmp in ['/etc',
'/lib',
'/dev',
@@ -404,6 +407,13 @@ def run_map (os_dict, part_dict, source_
break
return
+def get_key_from_os (part):
+ global key_to_os
+ for key in key_to_os.keys ():
+ if key_to_os[key] == part:
+ return key
+ return None
+
def fix_fstab (dest_partlist, os_dict):
for p in dest_partlist.keys ():
dest_partlist[p]['FSTAB'] = {}
@@ -415,11 +425,26 @@ def fix_fstab (dest_partlist, os_dict):
if not stuff[0]:
continue
dest_partlist[os_part]['FSTAB'][stuff[0]] = mp
+ dest_partlist[os_part]['OS'] = os_dict[os]['OS']
if 'SWAP' in os_dict[os].keys ():
for sp in os_dict[os]['SWAP']:
dest_partlist[os_part]['FSTAB'][sp] = 'swap'
-def write_mountmap (map_fd, source_partlist, part_dict):
+def write_mount (mount_fd, os_dict):
+ for k1,v1 in os_dict.iteritems ():
+ for k2,v2 in v1['FSTAB'].iteritems ():
+ if v2[0] is not None :
+ mount_fd.write ("%s:%s:%s\n" % (v2[0],
+ get_key_from_os(k1),
+ k2))
+ for i in range (len (v1['SWAP'])):
+ mount_fd.write ("%s:%s:swap\n" % (v1['SWAP'][i],
+ get_key_from_os(k1)))
+
+
+ return
+
+def write_map (map_fd, source_partlist, part_dict):
parts = source_partlist.keys ()
parts.sort ()
key_to_os_sorted = key_to_os.keys ()
@@ -459,12 +484,14 @@ def main ():
dump_fd = sys.stdout
commands_fd = sys.stdout
map_fd = sys.stdout
- (opt, args) = getopt.getopt (sys.argv[1:], "ai:d:c:m:",
+ mount_fd = sys.stdout
+ (opt, args) = getopt.getopt (sys.argv[1:], "ai:d:c:t:m:",
["auto",
"input=",
"dump=",
"commands=",
- "mountmap="])
+ "transfer=",
+ "mount="])
for (o, val) in opt:
if o == '-a' or o == '--auto':
@@ -484,11 +511,16 @@ def main ():
input_fd = file (val)
except:
sys.exit (1)
- if o == '-m' or o == '--mountmap':
+ if o == '-t' or o == '--transfer':
try:
map_fd = file (val, "w")
except:
sys.exit (1)
+ if o == '-m' or o == '--mount':
+ try:
+ mount_fd = file(val, "w")
+ except:
+ sys.exit (1)
source_dump = read_gdump (input_fd)
source_partlist = gdump_partlist (source_dump)
@@ -511,7 +543,9 @@ def main ():
write_dump (dump_fd, partlist_gdump (dest_partlist))
- write_mountmap (map_fd, source_partlist, part_dict)
+ write_map (map_fd, source_partlist, part_dict)
+
+ write_mount (mount_fd, os_dict)
if auto:
d = Dir.TmpDir ()
@@ -529,6 +563,7 @@ def main ():
dump_fd.close ()
map_fd.close ()
+ mount_fd.close ()
commands_fd.close ()
return
Index: extensions/gpartdump/gprobe
===================================================================
RCS file: /cvsroot/gluster/gluster-alpha1/extensions/gpartdump/gprobe,v
retrieving revision 1.3
diff -p -u -r1.3 gprobe
--- extensions/gpartdump/gprobe 23 Mar 2006 07:14:16 -0000 1.3
+++ extensions/gpartdump/gprobe 28 Mar 2006 09:51:59 -0000
@@ -33,12 +33,12 @@ def get_bootloader (part, _fstab):
bootloader = {}
if os.access ("%s/etc/lilo.conf" % mpoint, os.R_OK):
- hnd = os.popen ("grep 'boot *=' %s/etc/lilo.conf | cut -f 2 -d '='"
- % mpoint)
- bootloader['lilo'] = hnd.readline ().strip ()
- hnd.close ()
+ hnd = os.popen ("grep 'boot *=' %s/etc/lilo.conf | cut -f 2 -d '='"
+ % mpoint)
+ bootloader['lilo'] = hnd.readline ().strip ()
+ hnd.close ()
if os.access ("%s/boot/grub/menu.lst" % mpoint, os.R_OK):
- bootloader['grub'] = 'default'
+ bootloader['grub'] = 'default'
# If /boot is mapped to another partition, then check for
# grub/menu.lst in the path
for k, v in _fstab.iteritems ():
@@ -50,14 +50,14 @@ def get_bootloader (part, _fstab):
if os.access ("%s/grub/menu.lst" % tmp_mpoint, os.R_OK):
bootloader['grub'] = 'default'
unget_mpoint (tmp_need_umount, tmp_mpoint)
-
+
if os.access ("%s/ntldr" % mpoint, os.R_OK) \
or os.access ("%s/boot.ini" % mpoint, os.R_OK):
bootloader['windows'] = part #need to check this still
-
+
unget_mpoint (need_umount, mpoint)
return bootloader
-
+
def get_fstab (part, _parts):
(need_umount, mpoint) = get_mpoint (part)
More information about the Gluster-devel
mailing list