[cig-commits] r12418 - in cs/portal/trunk/northridge: . SeismoWebPortal SeismoWebPortal/templates/SeismoWebPortal
leif at geodynamics.org
leif at geodynamics.org
Tue Jul 15 18:03:41 PDT 2008
Author: leif
Date: 2008-07-15 18:03:40 -0700 (Tue, 15 Jul 2008)
New Revision: 12418
Added:
cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/cluster_queueWaitTime_form.html
Modified:
cs/portal/trunk/northridge/SeismoWebPortal/forms.py
cs/portal/trunk/northridge/SeismoWebPortal/management.py
cs/portal/trunk/northridge/SeismoWebPortal/mezzanine.py
cs/portal/trunk/northridge/SeismoWebPortal/models.py
cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/job_progress_form.html
cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_detail.html
cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_list.html
cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_status.html
cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/splash.html
cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/style.css
cs/portal/trunk/northridge/SeismoWebPortal/urls.py
cs/portal/trunk/northridge/SeismoWebPortal/views.py
cs/portal/trunk/northridge/setup.py
Log:
Implemented time estimation and usage accounting for SPECFEM. Added
multiple daemon, multiple cluster support.
Modified: cs/portal/trunk/northridge/SeismoWebPortal/forms.py
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/forms.py 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/forms.py 2008-07-16 01:03:40 UTC (rev 12418)
@@ -211,6 +211,29 @@
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# Clusters, Performance, Accounting
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+class ClusterQueueWaitTimeManipulator(forms.Manipulator):
+
+ def __init__(self, cluster):
+ self.cluster = cluster
+ self.fields = [
+ forms.IntegerField('queueWaitTime', is_required=True),
+ ]
+ return
+
+ def flatten_data(self):
+ return {'queueWaitTime': self.cluster.queueWaitTime}
+
+ def save(self, new_data):
+ self.cluster.queueWaitTime = new_data['queueWaitTime']
+ self.cluster.save()
+ return self.cluster
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Registration
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -350,22 +373,13 @@
def flatten_data(self):
new_data = {}
new_data.update(self.user.__dict__)
- try:
- userInfo = self.user.userinfo
- except models.UserInfo.DoesNotExist:
- pass
- else:
- new_data.update(userInfo.__dict__)
+ userInfo = self.user.userinfo
+ new_data.update(userInfo.__dict__)
return new_data
def save(self, new_data):
- # Create UserInfo if it doesn't exist.
user = self.user
- try:
- userInfo = user.userinfo
- except models.UserInfo.DoesNotExist:
- userInfo = models.UserInfo()
- user.userinfo = userInfo
+ userInfo = user.userinfo
# Save the new user.
user.first_name = new_data['first_name']
user.last_name = new_data['last_name']
Modified: cs/portal/trunk/northridge/SeismoWebPortal/management.py
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/management.py 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/management.py 2008-07-16 01:03:40 UTC (rev 12418)
@@ -99,6 +99,93 @@
return
+def randomName():
+ # Stolen from pyre.services.Pickler.
+
+ alphabet = list("0123456789abcdefghijklmnopqrstuvwxyz")
+
+ import random
+ random.shuffle(alphabet)
+ key = "".join(alphabet)[0:16]
+
+ return key
+
+
+def createClusters():
+ models.Cluster.objects.create(
+ name = "Lonestar",
+ url = "http://www.tacc.utexas.edu/resources/hpcsystems/#lonestar",
+ daemonCode = randomName(),
+ ring = 10,
+ order = 10,
+ suFactor = 1.935,
+ )
+
+ models.Cluster.objects.create(
+ name = "CITerra",
+ url = "http://citerra.caltech.edu/",
+ daemonCode = randomName(),
+ ring = 20,
+ order = 10,
+
+ # We compute CITerra's hypothetical TeraGrid SU conversion
+ # factor by comparing its performance to Lonestar for a given
+ # Specfem run.
+
+ # pangu Nex=240 rl=20.16462: 164.593911119 minutes * 216 procs = 35552.284801704001 cpu minutes
+ # - vs -
+ # lonestar Nex=240 rl=20.16462: 129.221072733 minutes * 216 procs = 27911.751710328001 cpu minutes
+ # => 27911.751710328001 * 1.935 / 35552.284801704001 = 1.5191496091102434
+ suFactor = 1.519,
+ )
+
+ return
+
+
+def createSpecfem3DGlobePerformance():
+ lonestar = models.Cluster.objects.get(name = "Lonestar")
+ ciTerra = models.Cluster.objects.get(name = "CITerra")
+ clusters = [lonestar, ciTerra]
+
+ meshes = [mesh.archivalObject() for mesh in models.Specfem3DGlobeMesh.builtInObjectList()]
+
+ perfData = {
+ # overhead timeFactor stationFactor
+ (lonestar, 6, 160): ( 6.0, 3.90919581078, 0.0122251748901 ),
+ (lonestar, 6, 240): ( 6.0, 6.40830686286, 0.0122251748901 ),
+ (ciTerra, 6, 160): ( 6.0, 5.09329515444, 0.00902156376639),
+ (ciTerra, 6, 240): ( 6.0, 8.16250993666, 0.00902156376639),
+ }
+
+ for cluster in clusters:
+ for mesh in meshes:
+ key = (cluster, mesh.nchunks, mesh.nex_xi())
+ data = perfData.get(key)
+ if data is None:
+ print "Warning: no Specfem3DGlobe performance data for '%s' on '%s'" % (mesh, cluster)
+ continue
+ print "Adding Specfem3DGlobe performance data for '%s' on '%s'" % (mesh, cluster)
+ overhead, timeFactor, stationFactor = data
+ models.Specfem3DGlobePerformance.objects.create(
+ cluster = cluster,
+ mesh = mesh,
+ overhead = overhead,
+ timeFactor = timeFactor,
+ stationFactor = stationFactor,
+ )
+
+ return
+
+
+def createAwards():
+ models.Award.objects.create(
+ user = None,
+ amount = 10000,
+ description = "Built-in allocation for all users.",
+ )
+ return
+
+
def importLegacyDatabase(legacyDB):
from pysqlite2 import dbapi2 as sqlite
@@ -187,6 +274,22 @@
return
+def createUserInfo():
+ from django.contrib.auth.models import User
+
+ for user in User.objects.all():
+ try:
+ userInfo = user.userinfo
+ except models.UserInfo.DoesNotExist:
+ print "Creating UserInfo for '%s'" % user
+ models.UserInfo.objects.create(
+ user = user,
+ institution = "CIG",
+ approved = True,
+ )
+ return
+
+
def createExamplesForUser(user):
from fformats import CMTSolutionFormat
from os.path import dirname, join
@@ -284,12 +387,20 @@
if (models.Station in created_models or
models.StationList in created_models):
createStationList()
+ if models.Cluster in created_models:
+ createClusters()
+ if models.Specfem3DGlobePerformance in created_models:
+ createSpecfem3DGlobePerformance()
+ if models.Award in created_models:
+ createAwards()
legacyDBList = os.environ.get('WEBPORTAL_LEGACY_DB')
if legacyDBList:
for legacyDB in legacyDBList.split(':'):
importLegacyDatabase(legacyDB)
+ createUserInfo()
+
if (flag or
models.Event in created_models or
models.Run in created_models):
Modified: cs/portal/trunk/northridge/SeismoWebPortal/mezzanine.py
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/mezzanine.py 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/mezzanine.py 2008-07-16 01:03:40 UTC (rev 12418)
@@ -285,12 +285,14 @@
'pages': paginator.pages,
'hits' : paginator.hits,
'root': request.root,
+ 'user': request.user,
})
else:
c = Context({
'%s_list' % template_object_name: l,
'is_paginated': False,
'root': request.root,
+ 'user': request.user,
})
if not allow_empty and len(l) == 0:
raise Http404
@@ -331,6 +333,7 @@
c = Context({
template_object_name: obj,
'root': request.root,
+ 'user': request.user,
})
for key, value in extra_context.items():
if callable(value):
Modified: cs/portal/trunk/northridge/SeismoWebPortal/models.py
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/models.py 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/models.py 2008-07-16 01:03:40 UTC (rev 12418)
@@ -5,6 +5,7 @@
from django.core import validators
import cmt
+import datetime
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -19,12 +20,15 @@
return ContentType.objects.get_for_model(cls)
@classmethod
+ def builtInObjectList(cls):
+ return [o.obj for o in BuiltIn.objects.filter(objType = cls.contentType())]
+
+ @classmethod
def userObjectList(cls, user):
- ctype = cls.contentType()
objList = []
- objList.extend([o.obj for o in BuiltIn.objects.filter(objType = ctype)])
+ objList.extend(cls.builtInObjectList())
builtInCount = len(objList)
- objList.extend([o.obj for o in Ownership.objects.filter(owner = user, objType = ctype)])
+ objList.extend([o.obj for o in Ownership.objects.filter(owner = user, objType = cls.contentType())])
return objList, builtInCount
@@ -228,8 +232,6 @@
return str(cmt.CMTSolution.createFromDBModel(self))
def saveSource(cls, event, cmtSolution):
- import datetime
-
when = datetime.datetime(cmtSolution.year,
cmtSolution.month,
cmtSolution.day,
@@ -393,7 +395,9 @@
# regional
return (256.0 / float(self.nex_xi())) * (self.angular_width_xi / 90.0) * 17.0
+ nproc = property(lambda self: self.nchunks * self.nproc_xi * self.nproc_eta)
+
class Specfem3DGlobeParameters(Model, EditableObject):
name = models.CharField(maxlength=100)
@@ -428,6 +432,35 @@
def get_model_id(self):
return model_types[self.model-1][0]
+ def runTime(self, run, cluster):
+ mesh = self.mesh.archivalObject()
+ perfSet = Specfem3DGlobePerformance.objects.filter(cluster = cluster, mesh = mesh)
+ if not perfSet.count():
+ return None
+ perf = perfSet.get()
+ time = (perf.overhead +
+ perf.timeFactor * run.record_length +
+ perf.stationFactor * run.stationList.station_set.count())
+ return time
+
+ def cost(self, run, cluster):
+ from math import ceil
+ time = self.runTime(run, cluster)
+ if time is None:
+ return None
+ return int(ceil((self.mesh.nproc * time / 60.0) * cluster.suFactor))
+
+ def estimatedCost(self, run):
+ estCost = None
+ for cluster in Cluster.objects.all():
+ c = self.cost(run, cluster)
+ if c is not None:
+ if estCost:
+ estCost = max(estCost, c)
+ else:
+ estCost = c
+ return estCost
+
code = 1
@@ -514,6 +547,16 @@
from math import ceil
return int(ceil(run.record_length * 60.0 / self.step))
+ def runTime(self, run, cluster):
+ return None # NYI
+
+ def cost(self, run, cluster):
+ return None # NYI
+
+ def estimatedCost(self, run):
+ return None # NYI
+
+
code = 2
@@ -550,6 +593,7 @@
finished = models.DateTimeField(null=True)
user = models.ForeignKey(User) # the user who submitted the request
+ cluster = models.ForeignKey('Cluster', null=True) # the cluster on which it ran
class Admin:
list_display = ('status', 'started', 'finished')
@@ -559,7 +603,12 @@
code = property(lambda self: self.parameters.code)
+ def cost(self):
+ if self.cluster:
+ return self.parameters.cost(self, self.cluster)
+ return self.parameters.estimatedCost(self)
+
class Run(Model):
event = models.ForeignKey(Event)
stationList = models.ForeignKey(StationList)
@@ -594,6 +643,14 @@
return 'purged'
status = property(_getStatus)
+ def _getCluster(self):
+ if self.isDraft:
+ return None
+ if self.archive:
+ return self.archive.cluster
+ return None
+ cluster = property(_getCluster)
+
def __str__(self):
return "Run %04d" % self.id
@@ -622,6 +679,7 @@
record_length = self.record_length,
status = "ready",
user = user,
+ cluster = Cluster.nextAvailableCluster(),
**self.archiveKeywords()
)
self._archive.save()
@@ -641,7 +699,62 @@
def isArchival(self):
return False
+ def queueWaitTime(self):
+ for clusters in Cluster.availableClusters(), Cluster.objects.all():
+ times = [cluster.queueWaitTime for cluster in clusters if cluster.queueWaitTime >= 0]
+ if times:
+ time = max(times)
+ if time:
+ return datetime.timedelta(seconds = 60 * int(time))
+ return None
+ def estimatedRunTime(self):
+ for clusters in Cluster.availableClusters(), Cluster.objects.all():
+ times = [self.parameters.runTime(self, cluster) for cluster in clusters]
+ if times:
+ time = max(times)
+ if time:
+ return datetime.timedelta(seconds = 60 * int(time))
+ return None
+
+ def estimatedTotalTime(self):
+ q = self.queueWaitTime()
+ r = self.estimatedRunTime()
+ if q is None or r is None:
+ return None
+ return q + r
+
+ def cost(self):
+ if self.archive:
+ return self.archive.cost()
+ return self.parameters.estimatedCost(self)
+
+ def projectedBalance(self):
+ # Get the owner of this run.
+ try:
+ ownership = Ownership.objects.get(
+ objType = self.contentType(),
+ objId = self.id,
+ )
+ except Ownership.DoesNotExist:
+ return None
+ avail = ownership.owner.userinfo.availableSUs()
+ cost = self.cost()
+ if cost is None:
+ # Unknown cost => unknown balance.
+ if avail == 0:
+ # If there are no SUs left, don't let the user run anything.
+ return -1
+ return None
+ return avail - cost
+
+ def breaksTheBank(self):
+ pb = self.projectedBalance()
+ if pb is None:
+ return False
+ return pb < 0
+
+
class Job(Model):
# each run may correspond to multiple jobs
run = models.ForeignKey(ArchivedRun)
@@ -686,6 +799,89 @@
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# Clusters, Performance, Accounting
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+class Cluster(Model):
+ name = models.CharField(maxlength=100)
+ url = models.URLField(verify_exists=False)
+ daemonCode = models.CharField(maxlength=100)
+ lastAccess = models.DateTimeField(null=True, blank=True)
+
+ reportedStatus = models.CharField(maxlength=100)
+ queueWaitTime = models.IntegerField(default=-1) # in minutes
+
+ # cluster selection algorithm
+ ring = models.IntegerField()
+ order = models.IntegerField()
+
+ # conversion factor from CPU hours to TeraGrid wide roaming SUs
+ # http://www.teragrid.org/userinfo/access/convert.php
+ suFactor = models.FloatField(max_digits=19, decimal_places=10)
+
+ def __str__(self): return self.name
+
+ def _getStatus(self):
+ now = datetime.datetime.now()
+ if (self.lastAccess is None or
+ now - self.lastAccess > datetime.timedelta(minutes=1)):
+ return 'offline'
+ return self.reportedStatus
+ status = property(_getStatus)
+
+ @classmethod
+ def availableClusters(cls):
+ clusters = []
+ ring = None
+ for cluster in cls.objects.all():
+ if cluster.status in ['offline', 'error']:
+ continue
+ clusters.append(cluster)
+ if ring is None or cluster.ring < ring:
+ ring = cluster.ring
+ # select only those clusters in the highest-priority ring
+ clusters = [cluster for cluster in clusters if cluster.ring == ring]
+ return clusters
+
+ @classmethod
+ def nextAvailableCluster(cls):
+ clusters = cls.availableClusters()
+ if not clusters:
+ return None
+ clusters.sort(key = lambda c: c.order)
+ lastCluster = None
+ for run in ArchivedRun.objects.all().order_by('-id'):
+ lastCluster = run.cluster
+ break
+ if lastCluster in clusters:
+ # Assign runs to clusters in a round-robin fashion.
+ while True:
+ c = clusters.pop(0)
+ clusters.append(c)
+ if c.order == lastCluster.order:
+ break
+ return clusters[0]
+
+
+class Specfem3DGlobePerformance(Model):
+ cluster = models.ForeignKey(Cluster)
+ mesh = models.ForeignKey(Specfem3DGlobeMesh)
+
+ overhead = models.FloatField(max_digits=19, decimal_places=10)
+ timeFactor = models.FloatField(max_digits=19, decimal_places=10)
+ stationFactor = models.FloatField(max_digits=19, decimal_places=10)
+
+
+class Award(Model):
+ user = models.ForeignKey(User, null=True, blank=True) # null = all users
+ amount = models.IntegerField() # in SUs
+ description = models.TextField(blank=True)
+ created = models.DateTimeField(auto_now_add=True, editable=False)
+ modified = models.DateTimeField(auto_now=True, editable=False)
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Registration
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -701,7 +897,6 @@
def __str__(self): return self.name
def hasExpired(self):
- import datetime
if self.expires is None:
return False
now = datetime.datetime.now()
@@ -739,7 +934,33 @@
list_display = ('username', 'first_name', 'last_name', 'invite', 'approved')
list_filter = ('approved',)
+ def totalSUs(self):
+ tally = 0
+ query = models.Q(user__isnull = True) | models.Q(user__exact = self.user)
+ for award in Award.objects.filter(query):
+ tally += award.amount
+ return tally
+ def usedSUs(self):
+ tally = 0
+ for run in ArchivedRun.objects.filter(user = self.user):
+ c = run.cost()
+ if c is not None:
+ tally += c
+ return tally
+
+ def availableSUs(self):
+ return max(self.totalSUs() - self.usedSUs(), 0)
+
+ def availableSUsPx154(self):
+ avail = float(self.availableSUs()) / float(self.totalSUs())
+ imageWidth = 154
+ offset = imageWidth - int(float(imageWidth) * avail)
+ if offset < 0:
+ offset = 0
+ return offset
+
+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Object Grouping
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Added: cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/cluster_queueWaitTime_form.html
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/cluster_queueWaitTime_form.html (rev 0)
+++ cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/cluster_queueWaitTime_form.html 2008-07-16 01:03:40 UTC (rev 12418)
@@ -0,0 +1,23 @@
+<html>
+ <head>
+ <title>Queue Wait Time</title>
+ </head>
+
+ <body>
+
+ <h1>Queue Wait Time</h1>
+
+ {% if form.has_errors %}
+ <p>{{ form.error_dict }}
+ {% endif %}
+
+ <form method="post" action="{{ action }}">
+
+ <p><label for="id_queueWaitTime">queue wait time</label> {{ form.queueWaitTime }}
+
+ <p><input type="submit" value="Save">
+
+ </form>
+
+ </body>
+</html>
Modified: cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/job_progress_form.html
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/job_progress_form.html 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/job_progress_form.html 2008-07-16 01:03:40 UTC (rev 12418)
@@ -13,7 +13,7 @@
<form method="post" action="{{ action }}">
- <p><label for="id_status">progress</label> {{ form.progress }}
+ <p><label for="id_progress">progress</label> {{ form.progress }}
<p><input type="submit" value="Save">
Modified: cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_detail.html
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_detail.html 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_detail.html 2008-07-16 01:03:40 UTC (rev 12418)
@@ -2,7 +2,11 @@
<h2><img src="{{root}}/images/rocket.gif" width=32 height=32> {{ object }}</h2>
{% ifequal object.status 'draft' %}
+{% if object.breaksTheBank %}
+<p class="message">You do not have enough SUs to perform this run. Please <a href="mailto:portal at geodynamics.org">contact us</a> to request more SUs.</p>
+{% else %}
<p class="message">This run has not been started. Verify that the parameters below are correct. Then click the "Start" button at the bottom of this page.</p>
+{% endif %}
{% endifequal %}
<dl class=parameters>
@@ -10,6 +14,10 @@
<dd>
<p>{{ object.status }}</p>
+ {% if object.cluster %}
+ <p>Cluster: <a href="{{object.cluster.url}}">{{ object.cluster.name }}</a></p>
+ {% endif %}
+
{% if not object.isDormant %}
<p>♦<i>This page was generated {% now "l, F jS, Y \a\t g:i:s a" %}
Click your web browser's “Refresh” or “Reload” button to get the current status.</i>
@@ -26,7 +34,16 @@
{% endifequal %}
{% ifnotequal object.status 'draft' %}
- {% if object.archive and object.archive.job_set.count %}
+ {% if object.archive %}
+
+ {% if not object.cluster %}
+ <p class="error">All supercomputing clusters are currently
+ unavailable. The site administrators have been notified of
+ this problem. This run will start as soon as a cluster becomes
+ available.</p>
+ {% endif %}
+
+ {% if object.archive.job_set.count %}
<table rules=cols>
<thead>
<tr>
@@ -60,9 +77,39 @@
</tbody>
</table>
{% endif %}
+ {% endif %}
{% endifnotequal %}
</dd>
+ {% if object.isDraft %}
+ <dt>estimated cost</dt>
+ <dd>
+ <table class="calc" cellspacing=20>
+ <tr><td>
+ <table class="calc">
+ <tr><td align=right>{{ user.userinfo.availableSUs }}</td><td>available SUs</td></tr>
+ <tr><td align=right>- {% if object.cost %}{{ object.cost }}{% else %}???{% endif %}</td><td>this run</td></tr>
+ <tr><td align=right>————</td><td></td></tr>
+ <tr><td align=right>{% ifnotequal object.projectedBalance None %}{% if object.breaksTheBank %}<span class="error">{{ object.projectedBalance }}</span>{% else %}{{ object.projectedBalance }}{% endif %}{% else %}???{% endifnotequal %}</td><td>projected balance</td></tr>
+ </table>
+ </td><td>
+ <table class="calc">
+ <tr><td align=right>{% if object.queueWaitTime %}{{ object.queueWaitTime }}{% else %}???{% endif %}</td><td>queue wait time</td></tr>
+ <tr><td align=right>+ {% if object.estimatedRunTime %}{{ object.estimatedRunTime }}{% else %}???{% endif %}</td><td>run time</td></tr>
+ <tr><td align=right>————</td><td></td></tr>
+ <tr><td align=right>{% if object.estimatedTotalTime %}{{ object.estimatedTotalTime }}{% else %}???{% endif %}</td><td>total time</td></tr>
+ </table>
+ </td></tr>
+ </table>
+ </dd>
+ {% else %}
+ <dt>cost</dt>
+ <dd>
+ <p>SUs: {% if object.cost %}{{ object.cost }}{% else %}unknown{% endif %}</p>
+ <p>Estimated run time: {% if object.estimatedRunTime %}{{ object.estimatedRunTime }}{% else %}unknown{% endif %}</p>
+ </dd>
+ {% endif %}
+
<dt>event</dt>
<dd>
<a href="{{root}}/?class=Event&object={{object.event.id}}">{{ object.event }}</a>
@@ -115,10 +162,13 @@
</dl>
{% ifequal object.status 'draft' %}
+{% if object.breaksTheBank %}
+{% else %}
<form method="post" action="{{root}}/?class=Run&object={{object.id}}&action=start">
<div class="tab30ex message">
<p>Click "Start" to start this run.</p>
<div><input class=submit type="submit" name="start" value="Start" /></div>
</div> <!-- tab30ex -->
</form>
+{% endif %}
{% endifequal %}
Modified: cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_list.html
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_list.html 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_list.html 2008-07-16 01:03:40 UTC (rev 12418)
@@ -1,6 +1,8 @@
<h2><img src="{{root}}/images/rocket.gif" width=32 height=32> Runs</h2>
+<center><p>{{ user.userinfo.availableSUs }} SUs available ~ <small>E</small> <img src="{{root}}/images/progress_box.gif" style="background: white url({{root}}/images/progress_bar.gif) top left no-repeat; background-position: -{{user.userinfo.availableSUsPx154}}px 0px;"> <small>F</small> ~ {{ user.userinfo.usedSUs }} SUs used</p></center>
+
{% if object_list %}
<table rules=cols class="clickable">
<thead>
@@ -9,6 +11,7 @@
<th>event</th>
<th>stations</th>
<th>parameters</th>
+ <th>SUs</th>
<th>started</th>
<th>finished</th>
<th>status</th>
@@ -22,6 +25,11 @@
<td>{{ object.event }}</td>
<td>{{ object.stationList }}</td>
<td>{{ object.parameters }}</td>
+ {% if object.cost %}
+ <td class=int>{{ object.cost }}</td>
+ {% else %}
+ <td class=notApplicable>-</td>
+ {% endif %}
{% if object.archive %}
<td>{{ object.archive.started|date }} {{ object.archive.started|time }}</td>
<td>{{ object.archive.finished|date }} {{ object.archive.finished|time }}</td>
Modified: cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_status.html
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_status.html 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/run_status.html 2008-07-16 01:03:40 UTC (rev 12418)
@@ -5,9 +5,8 @@
<body>
- <form action="{{action}}" method="POST" enctype="multipart/form-data">
+ <form method="post" action="{{action}}">
<p>{{ form.status }}
- <p>output: {{ form.output }} {{ form.output_file }}
<p><input type="submit" value="Save">
</form>
Modified: cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/splash.html
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/splash.html 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/splash.html 2008-07-16 01:03:40 UTC (rev 12418)
@@ -7,7 +7,7 @@
<h1><img src="{{root}}/images/cig.gif"> <img src="{{root}}/images/seismogram.gif"><br>CIG Seismology Web Portal</h1>
- <p>Version 3</p>
+ <p>Version 3.1.0</p>
<hr>
Modified: cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/style.css
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/style.css 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/templates/SeismoWebPortal/style.css 2008-07-16 01:03:40 UTC (rev 12418)
@@ -314,7 +314,12 @@
cursor: pointer;
}
+#content table.calc {
+ border: none;
+ width: auto;
+}
+
/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
forms
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
Modified: cs/portal/trunk/northridge/SeismoWebPortal/urls.py
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/urls.py 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/urls.py 2008-07-16 01:03:40 UTC (rev 12418)
@@ -20,21 +20,21 @@
(r'^(?P<pathname>style.css)$', 'SeismoWebPortal.views.directToTemplate', dict(template = 'SeismoWebPortal/style.css', mimetype='text/css')),
+ # cluster status
+ (r'^(?P<daemonCode>\w+)/qwt/$', 'SeismoWebPortal.views.daemonPost', dict(modelName='Cluster', action='queueWaitTime')),
+
# runs
- (r'^runs/list\.py$', 'django.views.generic.list_detail.object_list', dict(queryset = models.ArchivedRun.objects.all(),
- allow_empty = True,
- template_name='SeismoWebPortal/run_list.py.tmpl',
- mimetype='text/plain')),
- (r'^runs/(?P<object_id>\d+)/status/$', 'SeismoWebPortal.views.update_run_status'),
- (r'^runs/(?P<object_id>\d+)/(?P<filename>.*)$', 'SeismoWebPortal.views.downloadInputFile'),
+ (r'^(?P<daemonCode>\w+)/runs/list\.py$', 'SeismoWebPortal.views.runList'),
+ (r'^(?P<daemonCode>\w+)/runs/(?P<objectId>\d+)/status/$', 'SeismoWebPortal.views.updateRunStatus'),
+ (r'^(?P<daemonCode>\w+)/runs/(?P<objectId>\d+)/(?P<filename>.*)$', 'SeismoWebPortal.views.downloadInputFile'),
# jobs
- (r'^jobs/create/$', 'SeismoWebPortal.views.daemon_post', dict(modelName='Job', action='create')),
- (r'^jobs/(?P<object_id>\d+)/update/$', 'SeismoWebPortal.views.daemon_post', dict(modelName='Job', action='update')),
- (r'^jobs/(?P<object_id>\d+)/progress/$', 'SeismoWebPortal.views.daemon_post', dict(modelName='Job', action='progress')),
+ (r'^(?P<daemonCode>\w+)/jobs/create/$', 'SeismoWebPortal.views.daemonPost', dict(modelName='Job', action='create')),
+ (r'^(?P<daemonCode>\w+)/jobs/(?P<objectId>\d+)/update/$', 'SeismoWebPortal.views.daemonPost', dict(modelName='Job', action='update')),
+ (r'^(?P<daemonCode>\w+)/jobs/(?P<objectId>\d+)/progress/$', 'SeismoWebPortal.views.daemonPost', dict(modelName='Job', action='progress')),
# output files
- (r'^output/create/$', 'SeismoWebPortal.views.daemon_post', dict(modelName='OutputFile', action='create')),
+ (r'^(?P<daemonCode>\w+)/output/create/$', 'SeismoWebPortal.views.daemonPost', dict(modelName='OutputFile', action='create')),
# GMT
(r'^(?P<pathname>beachballs/.*)$', 'SeismoWebPortal.gmt.serve'),
Modified: cs/portal/trunk/northridge/SeismoWebPortal/views.py
===================================================================
--- cs/portal/trunk/northridge/SeismoWebPortal/views.py 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/SeismoWebPortal/views.py 2008-07-16 01:03:40 UTC (rev 12418)
@@ -13,12 +13,10 @@
import models
import mezzanine
+import datetime
import os, os.path
-OUTPUT_ROOT = os.path.join(settings.MEDIA_ROOT, 'SeismoWebPortal', 'output')
-
-
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# the main do-everything view
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -256,13 +254,42 @@
# daemon interface
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-def update_run_status(request, object_id):
+def daemonConnect(daemonCode):
+
+ cluster = get_object_or_404(models.Cluster, daemonCode = daemonCode)
+
+ oldStatus = cluster.status
+ cluster.lastAccess = datetime.datetime.now()
+ cluster.save()
+
+ # If this cluster just came online, send it any jobs that are
+ # waiting to be assigned to a cluster.
+ if oldStatus == 'offline':
+ for run in models.ArchivedRun.objects.filter(cluster__isnull = True):
+ run.cluster = cluster
+ run.save()
+
+ return cluster
+
+
+def runList(request, daemonCode):
+ from django.views.generic.list_detail import object_list
+ cluster = daemonConnect(daemonCode)
+ return object_list(request,
+ queryset = models.ArchivedRun.objects.filter(cluster = cluster),
+ allow_empty = True,
+ template_name = 'SeismoWebPortal/run_list.py.tmpl',
+ mimetype = 'text/plain')
+
+
+def updateRunStatus(request, daemonCode, objectId):
from forms import RunStatusManipulator
- import datetime
manipulator = RunStatusManipulator()
- run = get_object_or_404(models.ArchivedRun, id=object_id)
+ cluster = daemonConnect(daemonCode)
+ run = get_object_or_404(models.ArchivedRun, id=objectId)
+ assert run.cluster == cluster
if request.method == 'POST':
response = HttpResponse(mimetype='text/plain')
@@ -273,24 +300,13 @@
response.write(repr(errors))
else:
manipulator.do_html2python(new_data)
- output = new_data['output']
- if output:
- content = output['content']
- try:
- os.makedirs(OUTPUT_ROOT)
- except OSError: # Directory probably already exists.
- pass
- filename = os.path.join(OUTPUT_ROOT, object_id + '.tar.gz')
- stream = open(filename, 'wb')
- stream.write(content)
- stream.close()
run.status = new_data['status']
if run.status in ['done', 'error']:
run.finished = datetime.datetime.now()
if run.status == 'error':
- notify_admins_of_failed_run(run)
+ notifyAdminsOfFailedRun(run)
else:
- notify_user_of_successful_run(run, request)
+ notifyUserOfSuccessfulRun(run, request)
run.save()
response.write(repr('OK'))
return response
@@ -303,7 +319,7 @@
RequestContext(request, {}))
-def notify_admins_of_failed_run(run):
+def notifyAdminsOfFailedRun(run):
from django.core.mail import mail_admins
subject = 'run %d failed' % run.id
@@ -321,7 +337,7 @@
return
-def notify_user_of_successful_run(run, request):
+def notifyUserOfSuccessfulRun(run, request):
from django.core.mail import send_mail
server = request.META['SERVER_NAME']
@@ -363,9 +379,11 @@
return
-def daemon_post(request, modelName=None, action=None, object_id=None):
- from forms import JobProgressManipulator
+def daemonPost(request, daemonCode, modelName=None, action=None, objectId=None):
+ from forms import JobProgressManipulator, ClusterQueueWaitTimeManipulator
+ cluster = daemonConnect(daemonCode)
+
follow = None
if modelName == "Job":
ModelClass = models.Job
@@ -374,18 +392,25 @@
elif modelName == "OutputFile":
ModelClass = models.OutputFile
template = 'SeismoWebPortal/outputfile_form.html'
+ elif modelName == "Cluster":
+ ModelClass = models.Cluster
else:
raise Http404
if action == "create":
manipulator = ModelClass.AddManipulator(follow = follow)
elif action == "update":
- manipulator = ModelClass.ChangeManipulator(object_id, follow = follow)
+ manipulator = ModelClass.ChangeManipulator(objectId, follow = follow)
elif action == "progress":
if ModelClass is not models.Job:
raise Http404
- manipulator = JobProgressManipulator(object_id)
+ manipulator = JobProgressManipulator(objectId)
template = 'SeismoWebPortal/job_progress_form.html'
+ elif action == "queueWaitTime":
+ if ModelClass is not models.Cluster:
+ raise Http404
+ manipulator = ClusterQueueWaitTimeManipulator(cluster)
+ template = 'SeismoWebPortal/cluster_queueWaitTime_form.html'
else:
raise Http404
@@ -412,8 +437,10 @@
RequestContext(request, {}))
-def downloadInputFile(request, object_id, filename):
- run = get_object_or_404(models.ArchivedRun, id=object_id)
+def downloadInputFile(request, daemonCode, objectId, filename):
+ cluster = daemonConnect(daemonCode)
+ run = get_object_or_404(models.ArchivedRun, id=objectId)
+ assert run.cluster == cluster
if run.code == models.Specfem3DGlobeParameters.code:
index = {
@@ -612,7 +639,7 @@
if isNewUser:
request.session.delete_test_cookie()
createExamplesForUser(user)
- notify_managers_of_new_user(request, user)
+ notifyManagersOfNewUser(request, user)
user.message_set.create(message="Welcome to the CIG Seismology Web Portal!")
else:
user.message_set.create(message="Your contact information has been saved.")
@@ -635,7 +662,7 @@
), RequestContext(request, {}))
-def notify_managers_of_new_user(request, user):
+def notifyManagersOfNewUser(request, user):
from django.core.mail import mail_managers
userInfo = user.userinfo
Modified: cs/portal/trunk/northridge/setup.py
===================================================================
--- cs/portal/trunk/northridge/setup.py 2008-07-15 20:45:51 UTC (rev 12417)
+++ cs/portal/trunk/northridge/setup.py 2008-07-16 01:03:40 UTC (rev 12418)
@@ -3,7 +3,7 @@
setup(
name = 'SeismoWebPortal',
- version = '3.0.0',
+ version = '3.1.0',
url = 'http://www.geodynamics.org/',
author = 'Leif Strand',
author_email = 'leif at geodynamics.org',
More information about the cig-commits
mailing list