[cig-commits] r15853 - cs/plot-user-map/trunk
leif at geodynamics.org
leif at geodynamics.org
Tue Oct 20 15:53:44 PDT 2009
Author: leif
Date: 2009-10-20 15:53:44 -0700 (Tue, 20 Oct 2009)
New Revision: 15853
Added:
cs/plot-user-map/trunk/map-daemon.py
Log:
Created 'map-daemon.py' (derived from the legendary
'plot-user-map.py'). This script runs automatically at midnight from
Buildbot's crontab, generating every possible user map. It also adds
a "dot archive" feature, so that no dot is ever forgotten.
Copied: cs/plot-user-map/trunk/map-daemon.py (from rev 15851, cs/plot-user-map/trunk/plot-user-map.py)
===================================================================
--- cs/plot-user-map/trunk/map-daemon.py (rev 0)
+++ cs/plot-user-map/trunk/map-daemon.py 2009-10-20 22:53:44 UTC (rev 15853)
@@ -0,0 +1,237 @@
+#!/usr/bin/env python
+
+import os
+import tempfile
+
+from pysqlite2 import dbapi2 as sqlite
+from os.path import basename, join, isdir, exists
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# configuration
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+# data
+database = "/home/leif/projects/user-maps/geobytes-database"
+statsDir = "/home/leif/projects/user-maps/stats"
+archiveDir = "/home/leif/projects/user-maps/archive"
+
+# output
+outputDir = "/home/leif/public_html/maps"
+
+# software
+gmtRoot = "/home/leif/opt/gmt"
+netcdfRoot = "/home/leif/opt/netCDF"
+
+# misc.
+defaultDPI = 72
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# constants
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+queryFormat = """SELECT c.latitude, c.longitude, c.city, r.region, co.country
+FROM cities AS c
+JOIN subnets AS s ON c.cityid=s.cityid
+JOIN regions AS r ON c.regionid=r.regionid
+JOIN countries AS co ON c.countryid=co.countryid
+WHERE s.subnetaddress IN %s"""
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# code
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+def queryDatabase(ipList):
+
+ subnetList = []
+ for ip in ipList:
+ subnet = '.'.join(ip.split('.')[0:3])
+ subnetList.append(subnet)
+
+ subnets = '("' + '", "'.join(subnetList) + '")'
+ query = queryFormat % subnets
+
+ con = sqlite.connect(database)
+ con.text_factory = str # for conversion to UTF-8
+ cur = con.cursor()
+ cur.execute(query)
+
+ table = []
+ for row in cur:
+ table.append(row)
+
+ return table
+
+
+def plotMap(latLonList, dpi=72):
+
+ os.putenv("LD_LIBRARY_PATH", join(netcdfRoot, "lib"))
+
+ xyFile = "user-map.xy"
+
+ c = dict(
+ gmt_bin = join(gmtRoot, "bin"),
+ xyFile = xyFile,
+ psFile = "user-map.ps",
+ dpi = dpi,
+ gifFile = "user-map.gif",
+ )
+
+ xy = open(xyFile, 'w')
+ for lat, lon in latLonList:
+ print >>xy, lon, lat
+ xy.close()
+
+ os.system("%(gmt_bin)s/gmtset PAPER_MEDIA = letter+" % c);
+ os.system("%(gmt_bin)s/psbasemap -R-179/179/-60/70 -JM6i -Ba60/a30/wesn -P -K -V > %(psFile)s" % c);
+ os.system("%(gmt_bin)s/pscoast -R -JM -Dl -A10000 -G0/150/0 -S0/0/150 -K -O -V >> %(psFile)s" % c);
+ os.system("%(gmt_bin)s/psxy -R -JM -Sc0.17 -O -V -G255/255/0 -W1/0/0/0 %(xyFile)s >> %(psFile)s" % c);
+ os.system("convert -trim +repage -density %(dpi)d %(psFile)s %(gifFile)s" % c);
+
+
+def scanStats(rawStats, filter=None):
+
+ fileList = []
+
+ codeList = []
+ platformList = []
+ versionList = []
+ monthList = []
+
+ for name in rawStats:
+
+ tokens = name.split('_')
+ if len(tokens) == 3:
+ x, month, year = tokens
+ elif len(tokens) == 4:
+ # Seismic_CPML
+ x, y, month, year = tokens
+ x = x + '_' + y
+ else:
+ continue
+ if (len(month), len(year)) != (3, 4):
+ continue
+ month = "%s-%s" % (year, month)
+
+ code, platform, version = None, None, None
+ tokens = x.split('-')
+ if len(tokens) == 2:
+ code, version = tokens
+ elif len(tokens) == 3:
+ if tokens[1] == "Globe":
+ code = tokens[0] + tokens[1]
+ version = tokens[2]
+ else:
+ code, platform, version = tokens
+ elif len(tokens) == 4:
+ code, opSys, arch, version = tokens
+ if not opSys[0:3] in ["Lin", "Mac", "Win"]:
+ continue
+ platform = "%s-%s" % (opSys, arch)
+ else:
+ continue
+
+ if code is None or version is None:
+ continue
+ if code == "PyLith3d":
+ code = "PyLith"
+ if platform is None:
+ platform = "(None)"
+
+ if filter:
+ if not code in filter[0]:
+ continue
+ if len(filter) > 1:
+ if not platform in filter[1]:
+ continue
+ if not version in filter[2]:
+ continue
+ if not month in filter[3]:
+ continue
+
+ pathname = join(statsDir, name)
+ fileList.append(pathname)
+
+ codeList.append(code)
+ platformList.append(platform)
+ versionList.append(version)
+ monthList.append(month)
+
+ filterChoices = []
+ lists = [
+ ('codeList', codeList),
+ ('platformList', platformList),
+ ('versionList', versionList),
+ ('monthList', monthList),
+ ]
+
+ # sort and uniqify
+ for name, l in lists:
+ l.sort()
+ u = []
+ last = None
+ for x in l:
+ if x != last:
+ u.append(x)
+ last = x
+ filterChoices.append((name, u))
+
+ return fileList, filterChoices
+
+
+def main():
+ rawStats = os.listdir(statsDir)
+ fileList, lists = scanStats(rawStats)
+ codeList = lists[0][1]
+
+ for code in codeList:
+ fileList, lists = scanStats(rawStats, [[code]])
+
+ from socket import gethostbyname
+ ipList = []
+ errors = []
+ for pathname in fileList:
+ s = open(pathname, 'r')
+ for line in s:
+ entry = line.strip()
+ try:
+ entry = gethostbyname(entry)
+ except Exception, error:
+ errors.append((entry, str(error)))
+ continue
+ ipList.append(entry)
+
+ table = queryDatabase(ipList)
+
+ latLonList = []
+ for row in table:
+ latLonList.append((row[0], row[1]))
+
+ archive = join(archiveDir, code + ".py")
+ if exists(archive):
+ stream = open(archive, 'r')
+ data = stream.read()
+ stream.close()
+ archivedLatLonList = eval(data)
+ for item in archivedLatLonList:
+ if not item in latLonList:
+ latLonList.append(item)
+
+ plotMap(latLonList, defaultDPI)
+
+ output = join(outputDir, code + ".gif")
+ os.rename("user-map.gif", output)
+
+ # make sure no dot is ever forgotten
+ stream = open(archive, "w")
+ print >>stream, repr(latLonList)
+ stream.close()
+
+ return
+
+
+if __name__ == "__main__":
+ main()
Property changes on: cs/plot-user-map/trunk/map-daemon.py
___________________________________________________________________
Name: svn:executable
+
Name: svn:mergeinfo
+
More information about the CIG-COMMITS
mailing list