[cig-commits] r7904 - cs/stats/trunk
sue at geodynamics.org
sue at geodynamics.org
Wed Aug 29 12:24:18 PDT 2007
Author: sue
Date: 2007-08-29 12:24:18 -0700 (Wed, 29 Aug 2007)
New Revision: 7904
Added:
cs/stats/trunk/get_all.py
cs/stats/trunk/packages.py
Removed:
cs/stats/trunk/get_all_names
cs/stats/trunk/get_all_times
cs/stats/trunk/get_names
cs/stats/trunk/months
cs/stats/trunk/years
Log:
Replace most of the shell scripts with a python script
Added: cs/stats/trunk/get_all.py
===================================================================
--- cs/stats/trunk/get_all.py 2007-08-29 18:28:23 UTC (rev 7903)
+++ cs/stats/trunk/get_all.py 2007-08-29 19:24:18 UTC (rev 7904)
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+import gzip
+import re
+import os
+import string
+from sets import Set
+
+months=['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov',
+ 'Dec']
+years=['2006','2007']
+
+remove_regex=['bot','spider','Yahoo! Slurp','^131\.215','^127\.0\.0\.1']
+
+from packages import *
+
+# Initialize the hits for all of the packages and months
+hits={}
+for m in months:
+ for y in years:
+ for p in packages.itervalues():
+ hits[p+"_"+m+"_"+y]=Set()
+
+# Get all of the logs to scan
+logs=['/var/log/apache2/access_log']
+zope_logs=os.listdir("/var/lib/zope2.7/instance/plone/log")
+for log in zope_logs:
+ if re.search("Z2.log",log):
+ logs+=["/var/lib/zope2.7/instance/plone/log/"+log]
+
+# Scan through the logs, using gzip if needed
+for log in logs:
+ if log[-3:]==".gz":
+ f=gzip.GzipFile(log)
+ else:
+ f=open(log)
+ # Scan the log, line by line.
+ s=f.readline()
+ while(s):
+ # Remove robots, spiders, and local hits
+ valid=True
+ for pattern in remove_regex:
+ if re.search(pattern,s):
+ valid=False
+ continue
+ if valid:
+ for m in months:
+ for y in years:
+ if re.search(m+"/"+y,s):
+ for p in packages.iterkeys():
+ if re.search(p,s):
+ ttmp=packages[p]+"_"+m+"_"+y
+ hits[ttmp].add(string.split(s)[0])
+ s=f.readline()
+ f.close()
+
+# Write the raw ip's
+for h in hits.iterkeys():
+ f=open(h,"w")
+ for ip in hits[h]:
+ f.write(ip)
+ f.write("\n")
+ f.close()
+
Property changes on: cs/stats/trunk/get_all.py
___________________________________________________________________
Name: svn:executable
+ *
Deleted: cs/stats/trunk/get_all_names
===================================================================
--- cs/stats/trunk/get_all_names 2007-08-29 18:28:23 UTC (rev 7903)
+++ cs/stats/trunk/get_all_names 2007-08-29 19:24:18 UTC (rev 7904)
@@ -1,81 +0,0 @@
-./get_all_times MAG-1.0.0.tar.gz MAG-1.0.0
-./get_all_times MAG-1.0.1.tar.gz MAG-1.0.1
-./get_all_times MAG-1.0.2.tar.gz MAG-1.0.2
-./get_all_times mineos-1.0.0.tar.gz Mineos-1.0.0
-
-./get_all_times Gale-1_2_1.tar.gz Gale-1.2.1
-./get_all_times Gale-Linux32-1_2_1.tar.gz Gale-Linux32-1.2.1
-./get_all_times Gale-LinuxAMD64-1_2_1.tar.gz Gale-LinuxAMD64-1.2.1
-./get_all_times Gale-MacIntel-1_2_1.dmg Gale-MacIntel-1.2.1
-./get_all_times Gale-MacPPC-1_2_1.dmg Gale-MacPPC-1.2.1
-./get_all_times Gale-Win32-1_2_1.zip Gale-Win32-1.2.1
-./get_all_times Gale-1_2_0.tar.gz Gale-1.2.0
-./get_all_times Gale-Linux32-1_2_0.tar.gz Gale-Linux32-1.2.0
-./get_all_times Gale-LinuxAMD64-1_2_0.tar.gz Gale-LinuxAMD64-1.2.0
-./get_all_times Gale-MacIntel-1_2_0.dmg Gale-MacIntel-1.2.0
-./get_all_times Gale-MacPPC-1_2_0.dmg Gale-MacPPC-1.2.0
-./get_all_times Gale-Win32-1_2_0.zip Gale-Win32-1.2.0
-./get_all_times Gale-1_1_1.tar.gz Gale-1.1.1
-./get_all_times Gale-Linux32_1_1_1.tar.gz Gale-Linux32-1.1.1
-./get_all_times Gale-LinuxAMD64_1_1_1.tar.gz Gale-LinuxAMD64-1.1.1
-./get_all_times Gale-MacIntel-1_1_1.dmg Gale-MacIntel-1.1.1
-./get_all_times Gale-MacPPC-1_1_1.dmg Gale-MacPPC-1.1.1
-./get_all_times Gale-Win32_1_1_1.zip Gale-Win32-1.1.1
-./get_all_times Gale-1_1_0.tar.gz Gale-1.1.0
-./get_all_times Gale-Linux32_1_1_0.tar.gz Gale-Linux32-1.1.0
-./get_all_times Gale-LinuxAMD64_1_1_0.tar.gz Gale-LinuxAMD64-1.1.0
-./get_all_times Gale-MacIntel-1_1_0.dmg Gale-MacIntel-1.1.0
-./get_all_times Gale-MacPPC-1_1_0.dmg Gale-MacPPC-1.1.0
-./get_all_times Gale-Win32_1_1_0.zip Gale-Win32-1.1.0
-./get_all_times Gale-1.0.0.tar.gz Gale-1.0.0
-./get_all_times Gale-Linux-1.0.0.tar.gz Gale-Linux-1.0.0
-./get_all_times Gale-Mac-1_0_0.dmg Gale-Mac-1.0.0
-./get_all_times Gale-Win32-1_0_0.zip Gale-Win32-1.0.0
-./get_all_times Gale-0.9.0.tar.gz Gale-0.9.0
-./get_all_times Gale-0.2.0.tar.gz Gale-0.2.0
-./get_all_times Gale-0.1.0.tar.gz Gale-0.1.0
-./get_all_times Gale-Linux.tar.gz Gale-Linux-0.9
-./get_all_times Gale-Mac.dmg Gale-Mac-0.9
-./get_all_times Gale-Win32.zip Gale-Win32-0.9
-
-./get_all_times plasti-1.0.0.tar.gz Plasti-1.0.0
-
-./get_all_times lithomop3d-1.0.0.tar.gz LithoMop3d-1.0.0
-./get_all_times lithomop3d-0.7.2.tar.gz LithoMop3d-0.7.2
-
-./get_all_times pylith-1.0.1.tar.gz PyLith-1.0.1
-./get_all_times pylith-1.0.1-darwin-powerpc.tar.gz PyLith-Mac-PowerPC-1.0.1
-./get_all_times pylith-1.0.1-win-i686.exe PyLith-Win-1.0.1
-./get_all_times pylith-1.0.1-linux-i686.tar.gz PyLith-Linux-1.0.1
-./get_all_times pylith-1.0.0.tar.gz PyLith-1.0.0
-./get_all_times pylith-1.0.0-darwin-powerpc.tar.gz PyLith-Mac-PowerPC-1.0.0
-./get_all_times pylith-1.0.0-win-i686.exe PyLith-Win-1.0.0
-./get_all_times pylith-1.0.0-linux-i686.tar.gz PyLith-Linux-1.0.0
-./get_all_times pylith3d-0.8.3.tar.gz PyLith3d-0.8.3
-./get_all_times pylith3d-0.8.3-linux-i686.tar.gz PyLith-Linux-0.8.3
-./get_all_times pylith3d-0.8.3-darwin-powerpc.tar.gz PyLith-Mac-0.8.3
-./get_all_times pylith3d-0.8.3-win-i686.exe PyLith-Win32-0.8.3
-./get_all_times pylith3d-0.8.2.tar.gz PyLith3d-0.8.2
-./get_all_times pylith3d-0.8.1.tar.gz PyLith3d-0.8.1
-./get_all_times pylith3d-0.8.0.tar.gz PyLith3d-0.8.0
-./get_all_times PyLith-0.8p1.dmg PyLith-Mac-0.8p1
-./get_all_times setup.exe PyLith-Win32-0.8.0
-./get_all_times pylith-0.8.1-linux-x86.tar.gz PyLith-Linux-0.8.1
-
-./get_all_times Ellipsis3D-1.0.2.tar.gz Ellipsis3D-1.0.2
-./get_all_times ellipsis3D-1.0.1.tar.gz Ellipsis3D-1.0.1
-./get_all_times ellipsis3D-1.0.0.tar.gz Ellipsis3D-1.0.0
-
-./get_all_times cigma-0.9.0.tar.gz Cigma-0.9.0
-
-./get_all_times CitcomS-2.2.2.tar.gz CitcomS-2.2.2
-./get_all_times CitcomS-2.2.1.tar.gz CitcomS-2.2.1
-./get_all_times CitcomS-2.1.0.tar.gz CitcomS-2.1.0
-./get_all_times CitcomS-2.0.2.tar.gz CitcomS-2.0.2
-./get_all_times CitcomS-2.0.1.tar.gz CitcomS-2.0.1
-./get_all_times CitcomS-2.0.0.tar.gz CitcomS-2.0.0
-
-./get_all_times CitcomCU-1.0.0.tar.gz CitcomCU-1.0.0
-./get_all_times CitcomCU-1.0.1.tar.gz CitcomCU-1.0.1
-./get_all_times CitcomCU-1.0.2.tar.gz CitcomCU-1.0.2
-./get_all_times CitcomCU-1.0.2-br-inflow.tar.gz CitcomCU-1.0.2-br-inflow
Deleted: cs/stats/trunk/get_all_times
===================================================================
--- cs/stats/trunk/get_all_times 2007-08-29 18:28:23 UTC (rev 7903)
+++ cs/stats/trunk/get_all_times 2007-08-29 19:24:18 UTC (rev 7904)
@@ -1,13 +0,0 @@
-current_month=`date +%b`
-current_year=`date +%G`
-previous_month=`grep -B 1 $current_month months | head -n 1`
-previous_year=$current_year
-if test -z "$previous_month"; then
- previous_month="Dec"
- previous_year=`grep -B 1 $current_year years | head -n 1`
-fi
-
-echo $1 $2 $current_month $current_year $previous_month $previous_year
-
-./get_names ${1} ${current_month}/${current_year} > $2_${current_month}_${current_year}
-./get_names ${1} ${previous_month}/${previous_year} > $2_${previous_month}_${previous_year}
Deleted: cs/stats/trunk/get_names
===================================================================
--- cs/stats/trunk/get_names 2007-08-29 18:28:23 UTC (rev 7903)
+++ cs/stats/trunk/get_names 2007-08-29 19:24:18 UTC (rev 7904)
@@ -1 +0,0 @@
-zgrep -h $2 /var/lib/zope2.7/instance/plone/log/Z2.log* /var/log/apache2/access_log | grep $1 | grep -v bot | grep -v spider | grep -v "Yahoo! Slurp" | grep -v "^131\.215" | grep -v "^127.0.0.1" | awk '{print $1}' | sort | uniq
Deleted: cs/stats/trunk/months
===================================================================
--- cs/stats/trunk/months 2007-08-29 18:28:23 UTC (rev 7903)
+++ cs/stats/trunk/months 2007-08-29 19:24:18 UTC (rev 7904)
@@ -1,12 +0,0 @@
-Jan
-Feb
-Mar
-Apr
-May
-Jun
-Jul
-Aug
-Sep
-Oct
-Nov
-Dec
Added: cs/stats/trunk/packages.py
===================================================================
--- cs/stats/trunk/packages.py 2007-08-29 18:28:23 UTC (rev 7903)
+++ cs/stats/trunk/packages.py 2007-08-29 19:24:18 UTC (rev 7904)
@@ -0,0 +1,61 @@
+packages={
+ 'MAG-1.0.0.tar.gz':'MAG-1.0.0',
+ 'MAG-1.0.1.tar.gz':'MAG-1.0.1',
+ 'MAG-1.0.2.tar.gz':'MAG-1.0.2',
+ 'mineos-1.0.0.tar.gz':'Mineos-1.0.0',
+
+ 'Gale-1_2_0.tar.gz':'Gale-1.2.0',
+ 'Gale-Linux32-1_2_0.tar.gz':'Gale-Linux32-1.2.0',
+ 'Gale-LinuxAMD64-1_2_0.tar.gz':'Gale-LinuxAMD64-1.2.0',
+ 'Gale-MacIntel-1_2_0.dmg':'Gale-MacIntel-1.2.0',
+ 'Gale-MacPPC-1_2_0.dmg':'Gale-MacPPC-1.2.0',
+ 'Gale-Win32-1_2_0.zip':'Gale-Win32-1.2.0',
+ 'Gale-1_1_1.tar.gz':'Gale-1.1.1',
+ 'Gale-Linux32_1_1_1.tar.gz':'Gale-Linux32-1.1.1',
+ 'Gale-LinuxAMD64_1_1_1.tar.gz':'Gale-LinuxAMD64-1.1.1',
+ 'Gale-MacIntel-1_1_1.dmg':'Gale-MacIntel-1.1.1',
+ 'Gale-MacPPC-1_1_1.dmg':'Gale-MacPPC-1.1.1',
+ 'Gale-Win32_1_1_1.zip':'Gale-Win32-1.1.1',
+ 'Gale-1_1_0.tar.gz':'Gale-1.1.0',
+ 'Gale-Linux32_1_1_0.tar.gz':'Gale-Linux32-1.1.0',
+ 'Gale-LinuxAMD64_1_1_0.tar.gz':'Gale-LinuxAMD64-1.1.0',
+ 'Gale-MacIntel-1_1_0.dmg':'Gale-MacIntel-1.1.0',
+ 'Gale-MacPPC-1_1_0.dmg':'Gale-MacPPC-1.1.0',
+ 'Gale-Win32_1_1_0.zip':'Gale-Win32-1.1.0',
+ 'Gale-1.0.0.tar.gz':'Gale-1.0.0',
+ 'Gale-Linux-1.0.0.tar.gz':'Gale-Linux-1.0.0',
+ 'Gale-Mac-1_0_0.dmg':'Gale-Mac-1.0.0',
+ 'Gale-Win32-1_0_0.zip':'Gale-Win32-1.0.0',
+ 'Gale-0.9.0.tar.gz':'Gale-0.9.0',
+ 'Gale-0.2.0.tar.gz':'Gale-0.2.0',
+ 'Gale-0.1.0.tar.gz':'Gale-0.1.0',
+ 'Gale-Linux.tar.gz':'Gale-Linux-0.9',
+ 'Gale-Mac.dmg':'Gale-Mac-0.9',
+ 'Gale-Win32.zip':'Gale-Win32-0.9',
+
+ 'plasti-1.0.0.tar.gz':'Plasti-1.0.0',
+
+ 'lithomop3d-1.0.0.tar.gz':'LithoMop3d-1.0.0',
+ 'lithomop3d-0.7.2.tar.gz':'LithoMop3d-0.7.2',
+
+ 'pylith3d-0.8.2.tar.gz':'PyLith3d-0.8.2',
+ 'pylith3d-0.8.1.tar.gz':'PyLith3d-0.8.1',
+ 'pylith3d-0.8.0.tar.gz':'PyLith3d-0.8.0',
+ 'PyLith-0.8p1.dmg':'PyLith-Mac-0.8p1',
+ 'setup.exe':'PyLith-Win32-0.8.0',
+ 'pylith-0.8.1-linux-x86.tar.gz':'PyLith-Linux-0.8.1',
+
+ 'Ellipsis3D-1.0.2.tar.gz':'Ellipsis3D-1.0.2',
+ 'ellipsis3D-1.0.1.tar.gz':'Ellipsis3D-1.0.1',
+ 'ellipsis3D-1.0.0.tar.gz':'Ellipsis3D-1.0.0',
+
+ 'CitcomS-2.2.1.tar.gz':'CitcomS-2.2.1',
+ 'CitcomS-2.1.0.tar.gz':'CitcomS-2.1.0',
+ 'CitcomS-2.0.2.tar.gz':'CitcomS-2.0.2',
+ 'CitcomS-2.0.1.tar.gz':'CitcomS-2.0.1',
+ 'CitcomS-2.0.0.tar.gz':'CitcomS-2.0.0',
+
+ 'CitcomCU-1.0.0.tar.gz':'CitcomCU-1.0.0',
+ 'CitcomCU-1.0.1.tar.gz':'CitcomCU-1.0.1',
+ 'CitcomCU-1.0.2.tar.gz':'CitcomCU-1.0.2',
+ 'CitcomCU-1.0.2-br-inflow.tar.gz':'CitcomCU-1.0.2-br-inflow'}
Property changes on: cs/stats/trunk/packages.py
___________________________________________________________________
Name: svn:executable
+ *
Deleted: cs/stats/trunk/years
===================================================================
--- cs/stats/trunk/years 2007-08-29 18:28:23 UTC (rev 7903)
+++ cs/stats/trunk/years 2007-08-29 19:24:18 UTC (rev 7904)
@@ -1,2 +0,0 @@
-2006
-2007
More information about the cig-commits
mailing list