multiple changes:
authorPaul Tonelli <paul.tonelli@logilab.fr>
Thu, 21 Mar 2013 14:33:03 +0100
changeset 1 8b17affcf529
parent 0 b8651d994515
child 2 23272de16625
multiple changes: - added _modules and _states - create vm_create
_modules/openstack_client.py
_modules/vm_create.py
_states/flavors.py
bootstrap-script.sh
init-client.sls
init-machine.sls
init-server.sls
minion.patch
others/vm-create.bash
readme.txt
top.sls
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/_modules/openstack_client.py	Thu Mar 21 14:33:03 2013 +0100
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+'''
+translation of a bash script to connect to nova
+'''
+
+from os import environ
+from glance import client as glance_client
+
+def set_vars():
+    '''
+    really ugly, but works
+    until credentials expire
+    '''
+    environ['OS_AUTH_URL']='http://control.example.com:5000/v2.0'
+    environ['OS_TENANT_ID']='00000000000000000000000000000000'
+    environ['OS_TENANT_NAME']='tenant'
+    environ['OS_USERNAME']='username'
+    environ['OS_PASSWORD']='password'
+
+def get_running_instances():
+    set_vars()
+    return glance_client.get_client(host='0.0.0.0')
+
+def test_pillars():
+    pillars_val = __salt__['pillar.raw']()['master']
+    print pillars_val
+    for i in pillars_val:
+        if i.find("keystone")!=-1:
+            print i + " : " + pillars_val[i]
+    return True
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/_modules/vm_create.py	Thu Mar 21 14:33:03 2013 +0100
@@ -0,0 +1,162 @@
+#!/usr/bin/env python
+'''
+translation of a bash script to create a flavored virtual machine
+'''
+
+from os import listdir, remove, stat, environ
+def _get_images(path):
+    '''
+    find .img files in path
+    '''
+    images = [element for element in listdir(path) 
+            if element.split(".")[-1] == "img"]
+    return images
+
+def _credentials_defined():
+    '''
+    check if glance credentials have been set in env
+    '''
+    for keyword in environ:
+        if keyword.find('OS_TENANT_NAME') !=1:
+            return True
+    return False
+
+
+def _create_apt_proxy(path="/etc/apt/apt.conf.d/01proxy",
+        proxy="http://proxy.logilab.priv:3142"):
+    proxy_file = open(path,'w')
+    proxy_file.write(proxy)
+    proxy_file.close()
+
+def _chroot_exec(chroot_path, command):
+    return __salt__['cmd.run']("chroot " + chroot_path + " " + command)
+
+def umount_images(path):
+    '''
+    umounts all images in path
+    '''
+    mounts = sorted([mount for mount in __salt__['mount.active']() 
+        if mount.find(path) != -1], reverse = True)
+    for mount in mounts:
+        print "umounting " + mount
+        print __salt__['mount.umount'](mount)
+
+def remove_images(path):
+    for image in _get_images(path):
+        print "removing " + path + "/" + image
+        remove(path + "/"+ image)
+
+def revert(path="/mnt", files_to_delete=()):
+    '''
+    umounts and delete all images in path
+    '''
+    umount_images(path)
+    remove_images(path)
+    for file_to_delete in files_to_delete:
+        remove(path + "/" +file_to_delete)
+       
+def resize_and_check(path, image_name, new_size=0):
+    '''
+    resize partition and filesystem whenever possible (size in Mb) 
+    and check the integrity (for ext* fs) 
+    '''
+    current_size = stat(path + "/" + image_name).st_size/pow(1024, 2)
+    if new_size > current_size:
+        print __salt__['cmd.run']("dd if=/dev/zero bs=1024k count=" 
+                + str(new_size - current_size) + " >> " + path + "/" 
+                + image_name)
+        minimize = ""
+    else:
+        minimize = "-M "
+    print __salt__['cmd.run']("e2fsck -n -f " + path + "/" + image_name)
+    print __salt__['cmd.run']("resize2fs -f " + minimize + path + "/" 
+            + image_name)
+
+def mount_image(archive_name, path, new_size = 4):
+    '''
+    unrar, resize and mount image in path
+    '''
+    print __salt__['file.makedirs'](path + "/") 
+    assert archive_name.split('.')[-1] == "gz"
+    remove_images(path)
+    assert len(_get_images(path)) == 0
+    print __salt__['cmd.run']("tar xzf " + archive_name + " --wildcards -C " 
+            + path + """ "*.img" """)
+    images = _get_images(path)
+    assert len(images) == 1
+    image_name = images[0]
+    resize_and_check(path, image_name, new_size * 1024)
+    image_dirname = image_name.replace(".img","")
+    print __salt__['file.makedirs'](path + "/" + image_dirname + "/") 
+    print __salt__['mount.mount'](path + "/" + image_dirname, path + "/" 
+            + image_name, opts="loop")
+    print __salt__['mount.mount'](path + "/" + image_dirname + "/proc", "none",
+            fstype="proc")
+    print __salt__['mount.mount'](path + "/" + image_dirname + "/dev/pts", 
+            "none", fstype="devpts")
+    return image_dirname, image_name
+
+def update_image(path, image_dirname):
+    _create_apt_proxy(path + "/" + image_dirname 
+            + "/etc/apt/apt.conf.d/01proxy")
+    _chroot_exec(image_dirname, "apt-get remove whoopsie")
+    _chroot_exec(image_dirname, "apt-get update && apt-get upgrade")
+    _chroot_exec(image_dirname, "apt-get install salt-minion")
+
+def flavor_image(path, image_dirname, flavor_filename):
+    '''
+    do all code relevant to personalizing the machine here
+    salt-minion should be available
+    '''
+    pass
+
+def upload_to_glance(path, filename, disk_format, other_params, 
+        image_name=None):
+    if image_name == None:
+        image_name = filename.replace(".img","")
+    #glance upload initrd, kernel and get ids
+    print "sending "+ disk_format + " to glance"
+    output_add = __salt__['cmd.run']("glance add is_public=false disk_format=" 
+            + disk_format + " container_format=" + disk_format + " name=\""
+            + image_name + "\" " + other_params + " < " + path + "/" + filename)
+    id_image = output_add.split("ID: ")[1].split('\n')[0]
+    print "Uploaded " + disk_format + " with id : " + id_image
+    return id_image
+
+def delete_image(id_image):
+    print __salt__['cmd.run']("glance delete " + id_image)
+
+def get_initrd_kernel(path, image_dirname):
+    return_files = []
+    for filetype in ("vmlinuz","initrd"):
+        myfiles = [myfile for myfile in listdir(path + "/" + image_dirname 
+            + "/boot/") 
+                if myfile.find(filetype) != -1]
+        assert len(myfiles) == 1
+        myfile = myfiles[0]
+        __salt__['cmd.run']("cp " + path + "/" + image_dirname + "/boot/" 
+                + myfile + " " + path)
+        return_files.append(myfile)
+    return tuple(return_files)
+
+def create_flavor(archive_name, flavor_filename, path="/mnt"):
+    '''
+    complete function to create the new flavor
+    '''
+    image_dirname, image_name =  mount_image(archive_name, path)
+    update_image(path, image_dirname)
+    flavor_image(path, image_dirname, flavor_filename)
+    kernel_name, ramdisk_name = get_initrd_kernel(path, image_dirname)
+    umount_images(image_dirname)
+    resize_and_check(path, image_name)
+    if _credentials_defined():
+        print "will now upload to glance server"
+        id_kernel = upload_to_glance(path, kernel_name, "aki", "", image_dirname)
+        id_ramdisk = upload_to_glance(path, ramdisk_name, "ari", "", image_dirname)
+        id_image = upload_to_glance(path, image_name, "ami", "kernel_id="
+                + id_kernel + " ramdisk_id=" + id_ramdisk)
+    else:
+        print "cannot upload, no credentials available"
+    #clean
+    revert(path, (kernel_name, ramdisk_name))
+    return True
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/_states/flavors.py	Thu Mar 21 14:33:03 2013 +0100
@@ -0,0 +1,22 @@
+
+#def flavor_sls_exist(name, value):
+#    if __opts__['test']:
+#        ret['result'] = True
+#        ret['comment'] = 'State flavor_sls_exist will check if a flavor file for {0} is available'.format(value)
+#        return ret
+#    return True
+
+def created(name, value):
+    ret = {'name': name,
+            'changes': {},
+            'result': True,
+            'comment':  ''}
+    if __opts__['test']:
+        ret['result'] = None
+        ret['comment'] = 'State created will check if flavor {0} is available on logilab openstack'.format(value)
+        return ret
+    ret['comment'] = "should have added the flavor value"
+    return ret
+
+
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bootstrap-script.sh	Thu Mar 21 14:33:03 2013 +0100
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+### download image ###
+image_name=raring-server-cloudimg-amd64.tar.gz 
+image_address=http://cloud-images.ubuntu.com/raring/current/$image_name
+wget $image_address 
+mv $image_name /mnt
+
+#### install and configure salt-minion ####
+wget http://ftp.fr.debian.org/debian/pool/main/s/salt/salt-minion_0.12.1-1_all.deb
+wget http://ftp.fr.debian.org/debian/pool/main/s/salt/salt-common_0.12.1-1_all.deb
+dpkg -i salt-minion_0.12.1-1_all.deb salt-common_0.12.1-1_all.deb
+apt-get update
+apt-get -f --force-yes --yes install
+aptitude install salt-minion
+patch /etc/salt/minion minion.patch
+mkdir /etc/salt/srv
+cp -r _modules _states init-server.sls top.sls /etc/salt/srv 
+salt-call state.highstate
+salt-call vm_create.create_flavor $image_name gna
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/init-client.sls	Thu Mar 21 14:33:03 2013 +0100
@@ -0,0 +1,13 @@
+# /srv/salt/init_machine.sls
+ami-packages:
+  pkg.installed:
+    - pkgs:
+      - salt-minion
+      - python
+      - mercurial
+
+http://hg.logilab.fr/users/ptonelli/automatic_ami_creation:
+   hg.latest:
+     - target: 
+
+salt.states.file.absent(name)
--- a/init-machine.sls	Wed Feb 20 18:48:57 2013 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,13 +0,0 @@
-# /srv/salt/init_machine.sls
-ami-packages:
-  pkg.installed:
-    - pkgs:
-      - salt-minion
-      - python
-      - mercurial
-
-http://hg.logilab.fr/users/ptonelli/automatic_ami_creation:
-   hg.latest:
-     - target: 
-
-salt.states.file.absent(name)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/init-server.sls	Thu Mar 21 14:33:03 2013 +0100
@@ -0,0 +1,19 @@
+# /srv/salt/init_machine.sls
+my_pkgs:
+  pkg.installed:
+    - pkgs:
+      - salt-minion
+      - python
+      - mercurial
+      - tar
+      - glance
+#      - eca2ools
+
+my_flavors:
+  flavors.created:
+    - value: gna
+
+http://hg.logilab.fr/users/ptonelli/srv_salt:
+   hg.latest:
+     - target: /mnt/srv
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/minion.patch	Thu Mar 21 14:33:03 2013 +0100
@@ -0,0 +1,12 @@
+226c226
+< #file_client: remote
+---
+> file_client: local
+244,246c244,246
+< #file_roots:
+< #  base:
+< #    - /srv/salt
+---
+> file_roots:
+>   base:
+>     - /etc/salt/srv
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/others/vm-create.bash	Thu Mar 21 14:33:03 2013 +0100
@@ -0,0 +1,148 @@
+#!/bin/bash
+
+set -e
+
+#install necessary software to work
+sudo aptitude install glance mercurial
+
+# Basic arg check
+archive=$1
+[ ! -f "$archive" ] && echo -e "[USAGE]\n$0 EXISTING_IMAGE.tar.gz" && exit 1
+
+#create directory to work in
+[ ! -d mnt ] && mkdir mnt
+
+#extract image only to mnt
+tar xzf $archive --wildcards -C mnt "*.img"
+#check
+cd mnt
+image=`ls *.img`
+
+# increase the image size of 4G (append 0s)
+echo "adding space to image to reach 4G"
+dd if=/dev/zero bs=4096k count=1024 >> $image
+# check
+e2fsck -n -f $image
+[ $? != 0 ] && echo "Incorrect filesystem on image $image" && exit 1
+# resize the image filesystem to the size of the file
+resize2fs -f $image
+[ $? != 0 ] && echo "Could not resize filesystem in $image" && exit 1
+ls -l $image
+
+# mount as loopback
+imagedir=`echo $image | sed 's:\.img$::'`
+mkdir $imagedir
+sudo mount -o loop $image $imagedir
+CHROOT="sudo chroot $imagedir"
+$CHROOT df -h .
+
+# Prepare actual install work
+
+$CHROOT mount -t proc none /proc
+$CHROOT mount -t devpts none /dev/pts
+echo "Acquire::http::Proxy \"http://proxy.logilab.priv:3142/\";" > 01proxy
+
+sudo cp 01proxy $imagedir/etc/apt/apt.conf.d/ && \rm 01proxy
+$CHROOT mv /etc/resolv.conf /etc/resolv.conf.orig
+sudo cp /etc/resolv.conf $imagedir/etc/resolv.conf
+
+#no questions 
+DEBIAN_FRONTEND=noninteractive
+
+#forbid apt to add scripts to boot (will need to add salt minion to boot)
+#doc in /usr/share/doc/sysv-rc/README.policy-rc.d.gz
+echo "exit 101" > policy-rc.d
+chmod +x policy-rc.d
+sudo cp policy-rc.d $imagedir/usr/sbin && rm policy-rc.d
+
+
+#special treatment for whoopsie which fails to update
+#it is a crash report util from ubuntu
+exist_whoopsie=`$CHROOT dpkg -l |grep " whoopsie "|wc -l`
+[ $exist_whoopsie -ge 1 ] && echo "I need to delete whoopsie"
+[ $exist_whoopsie -ge 1 ] && $CHROOT apt-get remove -y whoopsie
+
+#upgrade machine
+$CHROOT apt-get update &&
+sudo -E chroot $imagedir apt-get dist-upgrade -y
+
+# =========================
+# Install work is done HERE
+# =========================
+
+#install salt-minion
+$CHROOT apt-get install -y salt-minion
+
+#patch salt minion config
+echo "making salt minion work locally"
+patch $imagedir/etc/salt/minion `find /mnt -name minion.patch`
+[ $? != 0 ] && echo "error while patching minion config file, is the minion.patch file available ?" && exit 1
+
+#install using salt call, --local require salt-minion >=12.0 and is not necessary if previous patch applied
+$CHROOT salt-call --local pkg.install -l warning pkgs='["mercurial"]'
+
+#you should modify the files in imagedir to change the configuration of the client
+cp -r ../srv $imagedir/etc/salt/
+[ $? != 0 ] && echo "error while moving the minion config folder  ?" && exit 1
+$CHROOT salt-call state.highstate
+
+
+# ==========================================
+# getting files for uploading / ami creation
+# ==========================================
+
+kernel_filepath=`find $imagedir/boot -name "*vmlinuz*"`
+kernel_filename=`basename $kernel_filename`
+initrd_filepath=`find $imagedir/boot -name "*initrd*"`
+initrd_filename=`basename $initrd_filename`
+
+cp $kernel_filepath $kernel_filename
+cp $kernel_filepath $initrd_filename
+
+# =========================
+# End of install work
+# =========================
+
+# Cleanup
+$CHROOT rm /usr/sbin/policy-rc.d
+$CHROOT service rsyslog stop
+$CHROOT umount /proc
+$CHROOT umount /dev/pts
+sudo \rm $imagedir/etc/apt/apt.conf.d/01proxy
+$CHROOT mv -f /etc/resolv.conf.orig /etc/resolv.conf
+
+sudo umount $imagedir
+
+echo "WARNING: image $image was left in the tmpfs mounted directory mnt"
+
+# =================================
+# unmounting image and creating ami
+# =================================
+
+echo "unmounting chroot"
+sudo umount $imagedir/proc
+sudo umount $imagedir/dev/pts
+sudo umount $imagedir
+
+# check filesystem and shrink to minimal size
+sudo e2fsck -f $image
+sudo resize2fs -M -f $image 
+
+#glance upload initrd, kernel and get ids
+echo "sending kernel"
+output=`glance add is_public=false disk_format=aki container_format=aki\
+    name="$imagedir-kernel" < $kernel_filename`
+kernel_id=`echo $output|awk -F':' '{print $2}'|sed 's/ //g'`
+
+echo "sending initrd"
+output=`glance add is_public=false disk_format=ari container_format=ari\
+    name="$imagedir-initrd" < $initrd_filename`
+initrd_id=`echo $output|awk -F':' '{print $2}'|sed 's/ //g'`
+
+echo "sending image"
+#glance upload corresponding image
+glance add disk_format=ami container_format=ami\
+    name="debian-squeeze-uec-x86_64"\
+    kernel_id=$kernel_id\
+    ramdisk_id=$initrd_id\
+    < $image 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/readme.txt	Thu Mar 21 14:33:03 2013 +0100
@@ -0,0 +1,23 @@
+
+Objectif: créer de manière automatique des variantes machines virtuelles en
+fonction de fichiers disponibles sur un repository hg.
+
+Fonctionnement souhaité:
+commiter un fichier salt state contenant la variante puis exécuter le script de
+mise à jour des variantes si nécessaire (doit pouvoir être lancé
+automatiquement en cas de commit)
+
+
+Fonctionnement actuel:
+- lancer l'instance openstack ami image creation (id : d4062f13-f26a-4a9a-a61c-613911b22c77 ) et se logger dessus
+- lancer le script de creation (salt-minion a été enlevé de wheezy...):
+
+$ salt-call vm_create.create_flavor $image_name None
+
+ou
+
+- créer une machine wheezy 64 bits, faire un checkout de ce repo et lancer le script de bootstrap
+
+$ chmod +x bootstrap-script.sh
+$ ./bootstrap-script.sh 
+
--- a/top.sls	Wed Feb 20 18:48:57 2013 +0100
+++ b/top.sls	Thu Mar 21 14:33:03 2013 +0100
@@ -1,4 +1,4 @@
 # /srv/salt/top.sls
 base:
   '*':
-    - editor 
+    - init-server