google seems to like this page best, but the real front page for this document is at AmazonEC2. Best to start there.

This code is a derivative of AmazonEC2_MPI_scripts

#!/usr/bin/env python # This software code is made available "AS IS" without warranties of any # kind. You may copy, display, modify and redistribute the software # code either by itself or as incorporated into your code; provided that # you do not remove any proprietary notices. Your use of this software # code is at your own risk and you waive any claim against Amazon Web # Services LLC or its affiliates with respect to your use of this software # code. (c) 2006 Amazon Web Services LLC or its affiliates. All rights # reserved. ''' ec2-mpi-config.py Before running, change the configuration strings in EC2config.py to match your AWS keys This script will: 1. find list of running or pending MPI compute node instances on EC2 2. select the first available MPI node as the master, and scp over the host files to all machines in the MPI cluster 3. if the master node is still pending, the script will sleep and try again until cancelled by the user The ssh variable StrictHostKeyChecking is set to "no". This is an evil hack to avoid the tedious adding of each compute node host when managing a large cluster... I'm assuming these EC2 nodes will only connect to eachother, please be careful. See http://www.securityfocus.com/infocus/1806 Created by Peter Skomoroch on 2007-04-09. Copyright (c) 2007 DataWrangling. All rights reserved. ''' import sys import EC2 import os import commands import socket from EC2config import * def remote(host, cmd='scp', user='root', src=None, dest=None, credential=None, silent=False, test=False): d = { 'cmd':cmd, 'user':user, 'src':src, 'dest':dest, 'host':host, } d['switches'] = '' if credential: d['switches'] = '-i %s' % credential if cmd == 'scp': template = '%(cmd)s %(switches)s -o "StrictHostKeyChecking no" %(src)s %(user)s@%(host)s:%(dest)s' else: template = 'ssh -o "StrictHostKeyChecking no" %(user)s@%(host)s "%(cmd)s" ' cmdline = template % d if not silent: print "n",cmdline,"nn" if not test: os.system(cmdline) conn = EC2.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) instance_response=conn.describe_instances() parsed_response=instance_response.parse() mpi_instances=[] mpi_hostnames=[] machine_state=[] for chunk in parsed_response: if chunk[0]=='INSTANCE': if chunk[-1]=='running' or chunk[-1]=='pending': if (chunk[2] == IMAGE_ID) or (chunk[2] == MASTER_IMAGE_ID) : mpi_instances.append(chunk[1]) mpi_hostnames.append(chunk[3]) machine_state.append(chunk[-1]) # list the nodes if len(mpi_instances) > 0: print "n---- MPI Cluster Details ----" print "Numer of nodes = %s" % len(mpi_instances) i=0 for machine in mpi_instances: print "Instance= %s hostname= %s state= %s" %(machine, mpi_hostnames[i], machine_state[i]) i+=1 # designate the first one as the master # Bug: But does not respect MASTER_IMAGE_ID master_node = mpi_hostnames[0] print "nThe master node is %s" % master_node # write out the hostnames to a mpd.hosts file # we might want to use ip addresses instead to save lookup time? print "nWriting out mpd.hosts file" hosts_file= "mpd.hosts" output=open(hosts_file,'w') for host in mpi_hostnames: print >> output, "%s" % host output.close() host_addresses = "hosts" h_output=open(host_addresses,'w') print >> h_output, "127.0.0.1 localhost localhost.localdomain" for host in mpi_hostnames: ip = socket.gethostbyname(host) print >> h_output, "%s %s" % (ip, host) h_output.close() # copy over the EC2 keys and host config to all machines in cluster (both /root and /home/lamuser) #prep client machine...TODO: add check to only append this if not in keys already import shutil homedir = os.path.expanduser( '~' ) shutil.copy(KEY_LOCATION, '%s/id_rsa.pub' % homedir) shutil.copy('%s/id_rsa.pub' % homedir, '%s/.ssh/id_rsa' % homedir) for host in mpi_hostnames: remote(host, src="~/id_rsa.pub", dest="~/id_rsa.pub", credential=KEY_LOCATION) remote(host, cmd="cp id_rsa.pub .ssh/id_rsa") remote(host, cmd="touch .ssh/authorized_keys") remote(host, cmd="cat id_rsa.pub >> .ssh/authorized_keys") remote(host, cmd="cp -r id_rsa.pub .ssh /home/lamuser/") remote(host, cmd="chown -R lamuser:lamuser /home/lamuser") remote(host, src=host_addresses, dest="/etc/") remote(host, src=hosts_file, dest="/etc/") remote(host, src=host_addresses, dest="/home/lamuser/") remote(host, src=hosts_file, dest="/home/lamuser/") #mpiblast specific remote(host, cmd="mkdir -p /mnt/dbs/local/") remote(host, cmd="chmod 777 /mnt/dbs/local/") remote(host, cmd="mkdir -p /mnt/dbs/mpiblastable") remote(host, cmd="chmod 777 /mnt/dbs/mpiblastable/") remote(host, cmd="chown -R lamuser:lamuser /home/lamuser/")