weekreport.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
  1. # nm.debian.org weekly report generation
  2. #
  3. # Copyright (C) 2012 Enrico Zini <enrico@debian.org>
  4. #
  5. # This program is free software: you can redistribute it and/or modify
  6. # it under the terms of the GNU Affero General Public License as
  7. # published by the Free Software Foundation, either version 3 of the
  8. # License, or (at your option) any later version.
  9. #
  10. # This program is distributed in the hope that it will be useful,
  11. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. # GNU Affero General Public License for more details.
  14. #
  15. # You should have received a copy of the GNU Affero General Public License
  16. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. from django.core.management.base import BaseCommand, CommandError
  18. from django.core.mail import send_mail
  19. import django.db
  20. from django.conf import settings
  21. from django.db import connection, transaction
  22. from django.contrib.sites.models import Site
  23. from django.db.models import Count, Min, Max
  24. from collections import defaultdict
  25. import optparse
  26. import sys
  27. import datetime
  28. import logging
  29. import json
  30. import os
  31. import os.path
  32. import gzip
  33. import re
  34. import time
  35. import codecs
  36. from cStringIO import StringIO
  37. from backend import models as bmodels
  38. from backend import const
  39. from backend import utils
  40. log = logging.getLogger(__name__)
  41. # AM Inactivity threshold in days
  42. INACTIVE_AM_PERIOD = 30
  43. # AM_HOLD Inactivity threshold in days
  44. INACTIVE_AMHOLD_PERIOD = 180
  45. # Days one needs to have been DD in order to become AM
  46. NEW_AM_THRESHOLD = 180
  47. class Reporter(object):
  48. def __init__(self, since=None, until=None, twidth=72, **kw):
  49. if until is None:
  50. until = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0)
  51. if since is None:
  52. since = until - datetime.timedelta(days=7)
  53. self.since = since
  54. self.until = until
  55. self.twidth = twidth
  56. def print_proclist(self, out, procs, print_manager=True):
  57. """Format and print a list of processes to `out`. If `print_manager`
  58. is True, print a column with the AM login. The `procs` list needs to
  59. be annotated with a `last_log` property used to display the date."""
  60. print >>out
  61. col_uid = 0
  62. if print_manager:
  63. for p in procs:
  64. l = len(p.manager.person.uid)
  65. if l > col_uid:
  66. col_uid = l
  67. for p in procs:
  68. if print_manager:
  69. print >>out, \
  70. str(p.last_log.date()).rjust(12), \
  71. p.manager.person.uid.ljust(col_uid), \
  72. "%s <%s>" % (p.person.fullname, p.person.lookup_key)
  73. else:
  74. print >>out, \
  75. str(p.last_log.date()).rjust(12), \
  76. "%s <%s>" % (p.person.fullname, p.person.lookup_key)
  77. print >>out
  78. def subject(self):
  79. if (self.until - self.since).days == 7:
  80. return "NM report for week ending %s" % str(self.until.date())
  81. else:
  82. return "NM report from %s to %s" % (self.since.date(), self.until.date())
  83. def rep00_period(self, out, **opts):
  84. if (self.until - self.since).days == 7:
  85. print >>out, "For week ending %s." % str(self.until.date())
  86. else:
  87. print >>out, "From %s to %s." % (self.since.date(), self.until.date())
  88. def rep01_summary(self, out, **opts):
  89. "Weekly Summary Statistics"
  90. # Processes that started
  91. # We reuse last_log as that's what print_proclist expects
  92. new_procs = bmodels.Process.objects.filter(is_active=True) \
  93. .annotate(
  94. last_log=Min("log__logdate")) \
  95. .filter(last_log__gte=self.since)
  96. counts = defaultdict(list)
  97. for p in new_procs:
  98. counts[p.applying_for].append(p)
  99. for k, processes in sorted(counts.iteritems(), key=lambda x:const.SEQ_STATUS.get(x[0], 0)):
  100. print >>out, "%d more people applied to become a %s:" % (len(counts[k]), const.ALL_STATUS_DESCS.get(k, "(unknown)"))
  101. self.print_proclist(out, processes, False)
  102. # Processes that ended
  103. new_procs = bmodels.Process.objects.filter(progress=const.PROGRESS_DONE) \
  104. .annotate(
  105. last_log=Max("log__logdate")) \
  106. .filter(last_log__gte=self.since)
  107. counts = defaultdict(list)
  108. for p in new_procs:
  109. counts[p.applying_for].append(p)
  110. for k, processes in sorted(counts.iteritems(), key=lambda x:const.SEQ_STATUS.get(x[0], 0)):
  111. print >>out, "%d people became a %s:" % (len(counts[k]), const.ALL_STATUS_DESCS.get(k, "(unknown)"))
  112. self.print_proclist(out, processes, False)
  113. def rep02_newams(self, out, **opts):
  114. "New AM candidates"
  115. min_date = self.since - datetime.timedelta(days=NEW_AM_THRESHOLD)
  116. max_date = self.until - datetime.timedelta(days=NEW_AM_THRESHOLD)
  117. new_procs = bmodels.Process.objects.filter(progress=const.PROGRESS_DONE,
  118. applying_for__in=[const.STATUS_DD_U, const.STATUS_DD_NU]) \
  119. .annotate(
  120. ended=Max("log__logdate")) \
  121. .filter(ended__gte=min_date, ended__lte=max_date) \
  122. .order_by("ended")
  123. count = new_procs.count()
  124. if count:
  125. print >>out, "%d DDs are now %d days old and can decide to become AMs: ;)" % (
  126. count, NEW_AM_THRESHOLD)
  127. print >>out
  128. for p in new_procs:
  129. print >>out, " %s <%s>" % (p.person.fullname, p.person.uid)
  130. def rep03_amchecks(self, out, **opts):
  131. "AM checks"
  132. # Inactive AM processes
  133. procs = bmodels.Process.objects.filter(is_active=True, progress=const.PROGRESS_AM) \
  134. .annotate(
  135. last_log=Max("log__logdate")) \
  136. .filter(last_log__lte=self.until - datetime.timedelta(days=INACTIVE_AM_PERIOD)) \
  137. .order_by("last_log")
  138. count = procs.count()
  139. if count > 0:
  140. print >>out, "%d processes have had no apparent activity in the last %d days:" % (
  141. count, INACTIVE_AM_PERIOD)
  142. self.print_proclist(out, procs)
  143. # Inactive AM_HOLD processes
  144. procs = bmodels.Process.objects.filter(is_active=True, progress=const.PROGRESS_AM_HOLD) \
  145. .annotate(
  146. last_log=Max("log__logdate")) \
  147. .filter(last_log__lte=self.until - datetime.timedelta(days=INACTIVE_AMHOLD_PERIOD)) \
  148. .order_by("last_log")
  149. count = procs.count()
  150. if count > 0:
  151. print >>out, "%d processes have been on hold for longer than %d days:" % (
  152. count, INACTIVE_AMHOLD_PERIOD)
  153. self.print_proclist(out, procs)
  154. # $sql = "SELECT forename, surname, email FROM applicant WHERE newmaint IS NOT NULL AND newmaint BETWEEN NOW() - interval '6 months 1 week' AND NOW() - interval '6 months' ORDER BY newmaint DESC";
  155. # $sth = $dbh->prepare($sql);
  156. # $sth->execute();
  157. # $sth->bind_columns(\$firstname, \$surname, \$email);
  158. # if ($sth->rows > 0) {
  159. # print $header;
  160. # print "The following DDs are now 6 months old and can decide to become AMs: ;)\n";
  161. # while($sth->fetch()) {
  162. # print "$firstname $surname <$email>\n";
  163. # }
  164. # }
  165. def run(self, out, **opts):
  166. """
  167. Run all weekly report functions
  168. """
  169. title = "Weekly Report on Debian New Members"
  170. print >>out, title.center(self.twidth)
  171. print >>out, ("=" * len(title)).center(self.twidth)
  172. import inspect
  173. for name, meth in sorted(inspect.getmembers(self, predicate=inspect.ismethod)):
  174. if not name.startswith("rep"): continue
  175. log.info("running %s", name)
  176. # Compute output for this method
  177. mout = StringIO()
  178. meth(codecs.getwriter("utf8")(mout), **opts)
  179. # Skip it if it had no output
  180. if not mout.getvalue():
  181. log.info("skipping %s as it had no output", name)
  182. continue
  183. # Else output it, with title and stuff
  184. print >>out
  185. if meth.__doc__:
  186. title = meth.__doc__.strip().split("\n")[0].strip()
  187. print >>out, title
  188. print >>out, "=" * len(title)
  189. out.write(mout.getvalue())
  190. print >>out
  191. re_date = re.compile("^\d+-\d+-\d+$")
  192. re_datetime = re.compile("^\d+-\d+-\d+ \d+:\d+:\d+$")
  193. def get_date(s):
  194. import rfc822
  195. if re_date.match(s):
  196. try:
  197. return datetime.datetime.strptime(s, "%Y-%m-%d")
  198. except ValueError:
  199. date = rfc822.parsedate(s)
  200. elif re_datetime.match(s):
  201. try:
  202. return datetime.datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
  203. except ValueError:
  204. date = rfc822.parsedate(s)
  205. else:
  206. date = rfc822.parsedate(s)
  207. if date is None:
  208. return None
  209. return datetime.datetime(*date)
  210. class Command(BaseCommand):
  211. help = 'Daily maintenance of the nm.debian.org database'
  212. option_list = BaseCommand.option_list + (
  213. optparse.make_option("--quiet", action="store_true", default=None, help="Disable progress reporting"),
  214. optparse.make_option("--since", action="store", default=None, help="Start of report period (default: a week before the end)"),
  215. optparse.make_option("--until", action="store", default=None, help="End of report period (default: midnight this morning)"),
  216. optparse.make_option("--email", action="store", default=None, help="Email address to send the report to (default: print to stdout)"),
  217. )
  218. def handle(self, *fnames, **opts):
  219. FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
  220. if opts["quiet"]:
  221. logging.basicConfig(level=logging.WARNING, stream=sys.stderr, format=FORMAT)
  222. else:
  223. logging.basicConfig(level=logging.INFO, stream=sys.stderr, format=FORMAT)
  224. if opts["since"] is not None:
  225. opts["since"] = get_date(opts["since"])
  226. if opts["until"] is not None:
  227. opts["until"] = get_date(opts["until"])
  228. reporter = Reporter(**opts)
  229. if opts["email"]:
  230. mout = StringIO()
  231. reporter.run(mout, **opts)
  232. send_mail(
  233. reporter.subject(),
  234. mout.getvalue(),
  235. "NM Front Desk <nm@debian.org>",
  236. [opts["email"]])
  237. else:
  238. reporter.run(sys.stdout, **opts)