DevOps with Fabric

DevOps with Fabric
Pycon5
Simone Federici

$ fab taskA taskB
from fabric.api import run, env
env.hosts = ['host1', 'host2']def taskA():
run('ls')def taskB():
run('whoami')

$ fab mytask:roles=role1
from fabric.api import env
env.roledefs = {
'web': ['www1', 'www2', 'www3'],
'dns': ['ns1', 'ns2']}
def mytask():
run('ls /var/www')

$ fab
mytask:roles=role1,exclude_hosts="a;c
"from fabric.api import env, hosts, roles, run
env.roledefs = {'role1': ['b', 'c']}
@hosts('a', 'b')
@roles('role1')def mytask():
run('ls /var/www')

$ fab migrate update
from fabric.api import run, roles
env.roledefs = {
'db': ['db1', 'db2'],
'web': ['web1', 'web2', 'web3'],}
@roles('db')def migrate():
# Database stuff here.
pass
@roles('web')def update():
# Code updates here.
pass

fab deploy
from fabric.api import run, roles, executedef deploy():
execute(migrate)
execute(update)
migrate on db1
migrate on db2
update on web1
update on web2
update on web3

$ fab deploy:app or $ fab deploy:db
from fabric.api import run, execute, task# For example, code talking to an HTTP API, or a database, or ...from mylib import
external_datastore# This is the actual algorithm involved. It does not care about host# lists at all.def do_work():
run("something interesting on a host")# This is the user-facing task invoked on the command line.
@taskdef deploy(lookup_param):
# This is the magic you don't get with @hosts or @roles.
# Even lazy-loading roles require you to declare available roles
# beforehand. Here, the sky is the limit.
host_list = external_datastore.query(lookup_param)
# Put this dynamically generated host list together with the work to be
# done.
execute(do_work, hosts=host_list)

$ fab set_hosts:app do_work
from fabric.api import run, taskfrom mylib import external_datastore# Marked as a publicly visible task, but otherwise unchanged:
still just# "do the work, let somebody else worry about what hosts to run on".
@taskdef do_work():
run("something interesting on a host")
@taskdef set_hosts(lookup_param):
# Update env.hosts instead of calling execute()
env.hosts = external_datastore.query(lookup_param)

Combining stdout and stderr
run("cmd", pty=False, combine_stderr=True):
run("cmd", pty=False, combine_stderr=False):
run("cmd", pty=True, combine_stderr=False):

$ fab -H host1,host2,host3
runs_in_parallel runs_serially
from fabric.api import *
@paralleldef runs_in_parallel():
passdef runs_serially():
pass
runs_in_parallel on host1, host2, and host3
runs_serially on host1

$ fab -P -z 5 heavy_task
from fabric.api import *
@parallel(pool_size=5)def heavy_task():
# lots of heavy local lifting or lots of IO here

@task(alias=’short’)
from fabric.api import task, run
@taskdef mytask():
run("a command")
@task(alias='dwm')def deploy_with_migrations():
pass

Submodule deploy.py
@task(default=True)def full_deploy():
pass
$ fab --list
Available commands:
deploy
deploy.full_deploy
deploy.migrate
deploy.provision
deploy.push

Class Task
class MyTask(Task):
name = "deploy"
def run(self, environment, domain="whatever.com"):
run("git clone foo")
sudo("service apache2 restart")
instance = MyTask()
VS
@taskdef deploy(environment, domain="whatever.com"):
run("git clone foo")
sudo("service apache2 restart")

Colors
from fabric.colors import greenprint(green("This text is green!"))
fabric.colors.blue(text, bold=False)
fabric.colors.cyan(text, bold=False)
fabric.colors.green(text, bold=False)
fabric.colors.magenta(text, bold=False)
fabric.colors.red(text, bold=False)
fabric.colors.white(text, bold=False)
fabric.colors.yellow(text, bold=False)

Context managers
def mytask():
with cd('/path/to/app'), prefix('workon myvenv'):
run('./manage.py syncdb')
run('./manage.py loaddata myfixture')
with cd('/var/www'):
run('ls') # cd /var/www && ls
with cd('website1'):
run('ls') # cd /var/www/website1 && ls
with hide('running', 'stdout', 'stderr'):
run('ls /var/www')
# Map localhost:6379 on the server to localhost:6379 on the client,
# so that the remote 'redis-cli' program ends up speaking to the local
# redis-server.
with remote_tunnel(6379):
run("redis-cli -i")

Contrib
Django Integration
Rsync Project
Upload Project
Console Confirm y/n
Files and Directory

So… what is Fabric?
● Deploy
● Manage multiple server
● Clustering
● Multiplatform
● Parallel
● Testing
● SSH Authentication

SSH + Bash Power + Python =
Rocks**3 = Fabric

django integration
from fabric.contrib import django
django.settings_module('myproject.settings')from django.conf import settingsdef
dump_production_database():
run(pg_dump -U %s -w %s > /bck/prod-db.sql' % (
settings.DATABASE_USER,
settings.DATABASE_NAME
))

fab deploy
@hosts('user@example.com')def deploy():
with cd("/opt/myproject"):
run("git pull")
run("django-admin.py collectstatic --noinput")
run("django-admin.py migrate --noinput")
run("/etc/init.d/uwsgi stop || echo 'done'")
run("/etc/init.d/uwsgi start")

fab backup_and_publish
@task
@hosts('www-data@example.com')def backup_and_publish():
run('''tar cjvf /var/wwwbackups/www.%s.tar.bz2 --
exclude=/var/www/download --exclude=/var/www/backup* /var/www''' %
today().strftime("%Y%m%d%H%M%S"))
run('rsync -avz --checksum --ignore-times /var/wwwstage/ /var/www') #--delete

fab static_generation
@taskdef static_generation():
execute(remote_generation)
local("wget --user=admin --password=rootme --recursive --page-requisites --
html-extension --convert-links --restrict-file-names=windows --domains
example.com --no-parent http://wwwstage.example.com/ -o dump.log || echo 'Looking
for 404 on wwwstage.example.com'")
local("cat dump.log | grep -B 2 '404 Not Found' | grep
'http://wwwstage.example.com/' || echo 'OK no 404 found...'")

fab upload_release_note:2.10.0
@task
@hosts('install@example.com')def upload_release_note(version):
release_note_file_name = "RELEASE_NOTE_%s.TXT" % version
with open(release_note_file_name,"w") as out_file:
notes = jira.Query().render(version=version)
out_file.write(notes.encode('ascii', 'ignore'))
out_file.close()
put(release_note_file_name, "/cygdrive/d/%s/" % version)

fab cleanup_nexus:10.0.2-RC2
@taskdef cleanup_nexus(version):
for module in [
"core-api",
"core-client-rest",
"core-manual",
"core-web"]:
local("curl -X DELETE -u user:rootme
http://nexus.example.com:8180/nexus/service/local/repositories/releases/content/
example/%s/%s/" % (module, version))

LOCK_FILE = "~/.lockfile.release.core.lock"class Lock():
def __enter__(self):
if os.access(os.path.expanduser(LOCK_FILE), os.F_OK):
pidfile = open(os.path.expanduser(LOCK_FILE), "r")
pidfile.seek(0)
old_pid = pidfile.readline()
print "There is an already a process running with pid: %s," % old_pid
sys.exit(1)
pidfile = open(os.path.expanduser(LOCK_FILE), "w")
pidfile.write("%s" % os.getpid())
pidfile.close
def __exit__(self, type, value, traceback):
os.remove(os.path.expanduser(LOCK_FILE))
Locks

fab send_email_candidate:2.12,me@...
@taskdef send_mail_candidate(version, *receivers):
sender = 'development@geniusbytes.com'
body = """From: Core Team <noreply@example.com>To: Development
<development@example.com>Subject: New Release CANDIDATE %(version)snNew Release
CANDIDATE %(version)savailable on: * smb://example.com/myproject/%(version)s
""" % dict(version=version)
try:
message = smtplib.SMTP('example.com')
message.sendmail(sender, receivers, body)
print "Successfully sent email"
except smtplib.SMTPException:
print "Error: unable to send email"

XML parser
@taskdef get_pom_version():
src=os.path.dirname(__file__)
pom_file=os.path.abspath(os.path.join(src, 'pom.xml'))
from xml.dom.minidom import parse
pom = parse(pom_file)
version = pom.getElementsByTagName("version")[1].firstChild.nodeValue
find = re.compile(r'^d+.d+.d+-([a-zA-Z-]+)d*-SNAPSHOT$').findall(version)
if not find:
abort(version + " is not a valid development version")
versions = re.compile(r'd+').findall(version)
if len(versions) is 3:
versions.append(1)
if len(versions) is not 4:
abort(version + " is not a valid development version")
versions.extend(find)
return versions

fab sed_poms_version:2.13
@taskdef sed_poms_version(new_version):
major, minor, patch, rev, rc = get_pom_version()
version = "%s.%s.%s-%s%s-SNAPSHOT" % (major, minor, patch, rc, rev)
local("sed -i '' 's@%s@%s@g' pom.xml */pom.xml" % (version, new_version))

fab create_manuals:2.10
ACTIVATE="source /home/installer/.sphinx_env/bin/activate"
@hosts('installer@example.com')def create_manuals(version):
with cd(DEPLOY_DIR + "/myproject"):
name = "manual-%s-doc" % version
run("wget
http://nexus.example.com:8180/nexus/service/local/repo_groups/public/content/com/
myproject/manual/%s/%s.zip" % (version, name))
with cd(name):
run(ACTIVATE + "&& make docs")
run("mv target/en/latex/MyProject*.pdf docs/" % version)
run("tar cjvf docs/MyCDDS-en-%s.tar.bz2 target/docs/" % version)
run("scp -r docs install@example.com:/cygdrive/d/%s/docs" % version)

fab create_installer:2.10 (innosetup)
@hosts('install@example.com')def create_installer(version):
name = "installer-%s-app" % version
run("wget
http://nexus.geniusbytes.com:8180/nexus/service/local/repo_groups/public/content/
myproject/installer/%s/%s.zip" % (version, name))
with cd(name):
run("tar xjvf /cygdrive/d/%(version)s/MyCDDS-en-%(version)s.tar.bz2" %
dict(version=version))
run(ANT)
run("mkdir -p /cygdrive/d/%s" % version)
run("cp build/*.exe /cygdrive/d/%s/" % version)
run("cp build-update/*.exe /cygdrive/d/%s/" % version)
run("rm -rf %s" % name)

fab release_core:master,2.10,2.11
@hosts('installer@ecample.com')def release_core(branch, version, next_version):
with cd(CORE_DEPLOY_DIR):
run('git clone ssh://installer@example.com/srv/git/myproject.git')
with cd(CORE_DEPLOY_DIR + "/myproject"):
run("git checkout %s" % branch)
run("mvn clean install")
run("mvn --batch-mode release:clean release:prepare -
DreleaseVersion=%s -DdevelopmentVersion=%s" % (version, next_version))
run("mvn release:perform")

fab release:master
def release(branch):
with Lock():
major, minor, patch, rev, rc = check_current_version(branch)
if 'RC' != rc:
abort("RC not found, not possible release a final version")
version = "%s.%s.%s" % (major, minor, patch)
next_version = "%s.%s.%s-RC%s-SNAPSHOT" % (major, minor, int(patch)+1, 1)
puts("preparing to release %s (next will be %s)" % (version,
next_version))
execute(release_core, branch, version, next_version)
execute(create_manuals, version)
execute(create_installer, version)
execute(upload_release_note, version)
execute(send_mail_final_release, version, 'me@ex.com', 'dev@ex.com')
local("git pull")
execute(labels.missing_translations, 'me@ex.com')

EC2 Testing with 200 micro server
EC2 + Fabric + Funkload
● EC2 use all ubuntu standard AMI
● Fabric as remote control, move files, aggregate.
● Funkload in order to stress an internet application. (not on
EC2)

Performance Testing Architecture
Target
Cloud
CTRL
Tester
Target
Target
FunkLoad
Fabric
(nmon+pefmon)
Fabric +
EC2
Fabric +
EC2

Testing phases
I. Prepare Monitoring
II.Prepare Cloud
1. Start Monitoring
2. Start Parallel Testing
3. Collecting Test Results
4. Collecting Perf Results
5. ReportingTarge
t
Cloud
CTRL
Tester
Targe
tTarge
t
FunkLoa
d
Fabric
(nmon+pefmon)
Fabric +
EC2
Fabric +
EC2

Testing Console
fab prepare_monitoring
fab prepare_cloud
fab start_monitoring
fab start_testing:ciccio,100,5000
fab collecting_test_results:ciccio
fab collecting_perf_results:ciccio
fab reporting:ciccio

DevOps with Fabric

More Related Content

What's hot (20)

Viewers also liked (19)

Similar to DevOps with Fabric (20)

More from Simone Federici (16)

Recently uploaded (20)

DevOps with Fabric