spam-stat.el 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668
  1. ;;; spam-stat.el --- detecting spam based on statistics
  2. ;; Copyright (C) 2002-2012 Free Software Foundation, Inc.
  3. ;; Author: Alex Schroeder <alex@gnu.org>
  4. ;; Keywords: network
  5. ;; URL: http://www.emacswiki.org/cgi-bin/wiki.pl?SpamStat
  6. ;; This file is part of GNU Emacs.
  7. ;; GNU Emacs is free software: you can redistribute it and/or modify
  8. ;; it under the terms of the GNU General Public License as published by
  9. ;; the Free Software Foundation, either version 3 of the License, or
  10. ;; (at your option) any later version.
  11. ;; GNU Emacs is distributed in the hope that it will be useful,
  12. ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. ;; GNU General Public License for more details.
  15. ;; You should have received a copy of the GNU General Public License
  16. ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
  17. ;;; Commentary:
  18. ;; This implements spam analysis according to Paul Graham in "A Plan
  19. ;; for Spam". The basis for all this is a statistical distribution of
  20. ;; words for your spam and non-spam mails. We need this information
  21. ;; in a hash-table so that the analysis can use the information when
  22. ;; looking at your mails. Therefore, before you begin, you need tons
  23. ;; of mails (Graham uses 4000 non-spam and 4000 spam mails for his
  24. ;; experiments).
  25. ;;
  26. ;; The main interface to using spam-stat, are the following functions:
  27. ;;
  28. ;; `spam-stat-buffer-is-spam' -- called in a buffer, that buffer is
  29. ;; considered to be a new spam mail; use this for new mail that has
  30. ;; not been processed before
  31. ;;
  32. ;; `spam-stat-buffer-is-non-spam' -- called in a buffer, that buffer
  33. ;; is considered to be a new non-spam mail; use this for new mail that
  34. ;; has not been processed before
  35. ;;
  36. ;; `spam-stat-buffer-change-to-spam' -- called in a buffer, that
  37. ;; buffer is no longer considered to be normal mail but spam; use this
  38. ;; to change the status of a mail that has already been processed as
  39. ;; non-spam
  40. ;;
  41. ;; `spam-stat-buffer-change-to-non-spam' -- called in a buffer, that
  42. ;; buffer is no longer considered to be spam but normal mail; use this
  43. ;; to change the status of a mail that has already been processed as
  44. ;; spam
  45. ;;
  46. ;; `spam-stat-save' -- save the hash table to the file; the filename
  47. ;; used is stored in the variable `spam-stat-file'
  48. ;;
  49. ;; `spam-stat-load' -- load the hash table from a file; the filename
  50. ;; used is stored in the variable `spam-stat-file'
  51. ;;
  52. ;; `spam-stat-score-word' -- return the spam score for a word
  53. ;;
  54. ;; `spam-stat-score-buffer' -- return the spam score for a buffer
  55. ;;
  56. ;; `spam-stat-split-fancy' -- for fancy mail splitting; add
  57. ;; the rule (: spam-stat-split-fancy) to `nnmail-split-fancy'
  58. ;;
  59. ;; This requires the following in your ~/.gnus file:
  60. ;;
  61. ;; (require 'spam-stat)
  62. ;; (spam-stat-load)
  63. ;;; Testing:
  64. ;; Typical test will involve calls to the following functions:
  65. ;;
  66. ;; Reset: (spam-stat-reset)
  67. ;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
  68. ;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
  69. ;; Save table: (spam-stat-save)
  70. ;; File size: (nth 7 (file-attributes spam-stat-file))
  71. ;; Number of words: (hash-table-count spam-stat)
  72. ;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
  73. ;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
  74. ;; Reduce table size: (spam-stat-reduce-size)
  75. ;; Save table: (spam-stat-save)
  76. ;; File size: (nth 7 (file-attributes spam-stat-file))
  77. ;; Number of words: (hash-table-count spam-stat)
  78. ;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
  79. ;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
  80. ;;; Dictionary Creation:
  81. ;; Typically, you will filter away mailing lists etc. using specific
  82. ;; rules in `nnmail-split-fancy'. Somewhere among these rules, you
  83. ;; will filter spam. Here is how you would create your dictionary:
  84. ;; Reset: (spam-stat-reset)
  85. ;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
  86. ;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
  87. ;; Repeat for any other non-spam group you need...
  88. ;; Reduce table size: (spam-stat-reduce-size)
  89. ;; Save table: (spam-stat-save)
  90. ;;; Todo:
  91. ;; Speed it up. Integrate with Gnus such that it uses spam and expiry
  92. ;; marks to call the appropriate functions when leaving the summary
  93. ;; buffer and saves the hash table when leaving Gnus. More testing:
  94. ;; More mails, disabling SpamAssassin, double checking algorithm, find
  95. ;; improved algorithm.
  96. ;;; Thanks:
  97. ;; Ted Zlatanov <tzz@lifelogs.com>
  98. ;; Jesper Harder <harder@myrealbox.com>
  99. ;; Dan Schmidt <dfan@dfan.org>
  100. ;; Lasse Rasinen <lrasinen@iki.fi>
  101. ;; Milan Zamazal <pdm@zamazal.org>
  102. ;;; Code:
  103. (require 'mail-parse)
  104. (defvar gnus-original-article-buffer)
  105. (defgroup spam-stat nil
  106. "Statistical spam detection for Emacs.
  107. Use the functions to build a dictionary of words and their statistical
  108. distribution in spam and non-spam mails. Then use a function to determine
  109. whether a buffer contains spam or not."
  110. :version "22.1"
  111. :group 'gnus)
  112. (defcustom spam-stat-file "~/.spam-stat.el"
  113. "File used to save and load the dictionary.
  114. See `spam-stat-to-hash-table' for the format of the file."
  115. :type 'file
  116. :group 'spam-stat)
  117. (defcustom spam-stat-unknown-word-score 0.2
  118. "The score to use for unknown words.
  119. Also used for words that don't appear often enough."
  120. :type 'number
  121. :group 'spam-stat)
  122. (defcustom spam-stat-max-word-length 15
  123. "Only words shorter than this will be considered."
  124. :type 'integer
  125. :group 'spam-stat)
  126. (defcustom spam-stat-max-buffer-length 10240
  127. "Only the beginning of buffers will be analyzed.
  128. This variable says how many characters this will be."
  129. :type 'integer
  130. :group 'spam-stat)
  131. (defcustom spam-stat-split-fancy-spam-group "mail.spam"
  132. "Name of the group where spam should be stored.
  133. If `spam-stat-split-fancy' is used in fancy splitting rules. Has
  134. no effect when spam-stat is invoked through spam.el."
  135. :type 'string
  136. :group 'spam-stat)
  137. (defcustom spam-stat-split-fancy-spam-threshold 0.9
  138. "Spam score threshold in spam-stat-split-fancy."
  139. :type 'number
  140. :group 'spam-stat)
  141. (defcustom spam-stat-washing-hook nil
  142. "Hook applied to each message before analysis."
  143. :type 'hook
  144. :group 'spam-stat)
  145. (defcustom spam-stat-score-buffer-user-functions nil
  146. "List of additional scoring functions.
  147. Called one by one on the buffer.
  148. If all of these functions return non-nil answers, these numerical
  149. answers are added to the computed spam stat score on the buffer. If
  150. you defun such functions, make sure they don't return the buffer in a
  151. narrowed state or such: use, for example, `save-excursion'. Each of
  152. your functions is also passed the initial spam-stat score which might
  153. aid in your scoring.
  154. Also be careful when defining such functions. If they take a long
  155. time, they will slow down your mail splitting. Thus, if the buffer is
  156. large, don't forget to use smaller regions, by wrapping your work in,
  157. say, `with-spam-stat-max-buffer-size'."
  158. :type '(repeat sexp)
  159. :group 'spam-stat)
  160. (defcustom spam-stat-process-directory-age 90
  161. "Max. age of files to be processed in directory, in days.
  162. When using `spam-stat-process-spam-directory' or
  163. `spam-stat-process-non-spam-directory', only files that have
  164. been touched in this many days will be considered. Without
  165. this filter, re-training spam-stat with several thousand messages
  166. will start to take a very long time."
  167. :type 'number
  168. :group 'spam-stat)
  169. (defvar spam-stat-last-saved-at nil
  170. "Time stamp of last change of spam-stat-file on this run")
  171. (defvar spam-stat-syntax-table
  172. (let ((table (copy-syntax-table text-mode-syntax-table)))
  173. (modify-syntax-entry ?- "w" table)
  174. (modify-syntax-entry ?_ "w" table)
  175. (modify-syntax-entry ?. "w" table)
  176. (modify-syntax-entry ?! "w" table)
  177. (modify-syntax-entry ?? "w" table)
  178. (modify-syntax-entry ?+ "w" table)
  179. table)
  180. "Syntax table used when processing mails for statistical analysis.
  181. The important part is which characters are word constituents.")
  182. (defvar spam-stat-dirty nil
  183. "Whether the spam-stat database needs saving.")
  184. (defvar spam-stat-buffer nil
  185. "Buffer to use for scoring while splitting.
  186. This is set by hooking into Gnus.")
  187. (defvar spam-stat-buffer-name " *spam stat buffer*"
  188. "Name of the `spam-stat-buffer'.")
  189. (defvar spam-stat-coding-system
  190. (if (mm-coding-system-p 'emacs-mule) 'emacs-mule 'raw-text)
  191. "Coding system used for `spam-stat-file'.")
  192. ;; Hooking into Gnus
  193. (defun spam-stat-store-current-buffer ()
  194. "Store a copy of the current buffer in `spam-stat-buffer'."
  195. (let ((buf (current-buffer)))
  196. (with-current-buffer (get-buffer-create spam-stat-buffer-name)
  197. (erase-buffer)
  198. (insert-buffer-substring buf)
  199. (setq spam-stat-buffer (current-buffer)))))
  200. (defun spam-stat-store-gnus-article-buffer ()
  201. "Store a copy of the current article in `spam-stat-buffer'.
  202. This uses `gnus-article-buffer'."
  203. (with-current-buffer gnus-original-article-buffer
  204. (spam-stat-store-current-buffer)))
  205. ;; Data -- not using defstruct in order to save space and time
  206. (defvar spam-stat (make-hash-table :test 'equal)
  207. "Hash table used to store the statistics.
  208. Use `spam-stat-load' to load the file.
  209. Every word is used as a key in this table. The value is a vector.
  210. Use `spam-stat-ngood', `spam-stat-nbad', `spam-stat-good',
  211. `spam-stat-bad', and `spam-stat-score' to access this vector.")
  212. (defvar spam-stat-ngood 0
  213. "The number of good mails in the dictionary.")
  214. (defvar spam-stat-nbad 0
  215. "The number of bad mails in the dictionary.")
  216. (defvar spam-stat-error-holder nil
  217. "A holder for condition-case errors while scoring buffers.")
  218. (defsubst spam-stat-good (entry)
  219. "Return the number of times this word belongs to good mails."
  220. (aref entry 0))
  221. (defsubst spam-stat-bad (entry)
  222. "Return the number of times this word belongs to bad mails."
  223. (aref entry 1))
  224. (defsubst spam-stat-score (entry)
  225. "Set the score of this word."
  226. (if entry
  227. (aref entry 2)
  228. spam-stat-unknown-word-score))
  229. (defsubst spam-stat-set-good (entry value)
  230. "Set the number of times this word belongs to good mails."
  231. (aset entry 0 value))
  232. (defsubst spam-stat-set-bad (entry value)
  233. "Set the number of times this word belongs to bad mails."
  234. (aset entry 1 value))
  235. (defsubst spam-stat-set-score (entry value)
  236. "Set the score of this word."
  237. (aset entry 2 value))
  238. (defsubst spam-stat-make-entry (good bad)
  239. "Return a vector with the given properties."
  240. (let ((entry (vector good bad nil)))
  241. (spam-stat-set-score entry (spam-stat-compute-score entry))
  242. entry))
  243. ;; Computing
  244. (defun spam-stat-compute-score (entry)
  245. "Compute the score of this word. 1.0 means spam."
  246. ;; promote all numbers to floats for the divisions
  247. (let* ((g (* 2.0 (spam-stat-good entry)))
  248. (b (float (spam-stat-bad entry))))
  249. (cond ((< (+ g b) 5)
  250. .2)
  251. ((= 0 spam-stat-ngood)
  252. .99)
  253. ((= 0 spam-stat-nbad)
  254. .01)
  255. (t
  256. (max .01
  257. (min .99 (/ (/ b spam-stat-nbad)
  258. (+ (/ g spam-stat-ngood)
  259. (/ b spam-stat-nbad)))))))))
  260. ;; Parsing
  261. (defmacro with-spam-stat-max-buffer-size (&rest body)
  262. "Narrow the buffer down to the first 4k characters, then evaluate BODY."
  263. `(save-restriction
  264. (when (> (- (point-max)
  265. (point-min))
  266. spam-stat-max-buffer-length)
  267. (narrow-to-region (point-min)
  268. (+ (point-min) spam-stat-max-buffer-length)))
  269. ,@body))
  270. (defun spam-stat-buffer-words ()
  271. "Return a hash table of words and number of occurrences in the buffer."
  272. (run-hooks 'spam-stat-washing-hook)
  273. (with-spam-stat-max-buffer-size
  274. (with-syntax-table spam-stat-syntax-table
  275. (goto-char (point-min))
  276. (let ((result (make-hash-table :test 'equal))
  277. word count)
  278. (while (re-search-forward "\\w+" nil t)
  279. (setq word (match-string-no-properties 0)
  280. count (1+ (gethash word result 0)))
  281. (when (< (length word) spam-stat-max-word-length)
  282. (puthash word count result)))
  283. result))))
  284. (defun spam-stat-buffer-is-spam ()
  285. "Consider current buffer to be a new spam mail."
  286. (setq spam-stat-nbad (1+ spam-stat-nbad))
  287. (maphash
  288. (lambda (word count)
  289. (let ((entry (gethash word spam-stat)))
  290. (if entry
  291. (spam-stat-set-bad entry (+ count (spam-stat-bad entry)))
  292. (setq entry (spam-stat-make-entry 0 count)))
  293. (spam-stat-set-score entry (spam-stat-compute-score entry))
  294. (puthash word entry spam-stat)))
  295. (spam-stat-buffer-words))
  296. (setq spam-stat-dirty t))
  297. (defun spam-stat-buffer-is-non-spam ()
  298. "Consider current buffer to be a new non-spam mail."
  299. (setq spam-stat-ngood (1+ spam-stat-ngood))
  300. (maphash
  301. (lambda (word count)
  302. (let ((entry (gethash word spam-stat)))
  303. (if entry
  304. (spam-stat-set-good entry (+ count (spam-stat-good entry)))
  305. (setq entry (spam-stat-make-entry count 0)))
  306. (spam-stat-set-score entry (spam-stat-compute-score entry))
  307. (puthash word entry spam-stat)))
  308. (spam-stat-buffer-words))
  309. (setq spam-stat-dirty t))
  310. (autoload 'gnus-message "gnus-util")
  311. (defun spam-stat-buffer-change-to-spam ()
  312. "Consider current buffer no longer normal mail but spam."
  313. (setq spam-stat-nbad (1+ spam-stat-nbad)
  314. spam-stat-ngood (1- spam-stat-ngood))
  315. (maphash
  316. (lambda (word count)
  317. (let ((entry (gethash word spam-stat)))
  318. (if (not entry)
  319. (gnus-message 8 "This buffer has unknown words in it")
  320. (spam-stat-set-good entry (- (spam-stat-good entry) count))
  321. (spam-stat-set-bad entry (+ (spam-stat-bad entry) count))
  322. (spam-stat-set-score entry (spam-stat-compute-score entry))
  323. (puthash word entry spam-stat))))
  324. (spam-stat-buffer-words))
  325. (setq spam-stat-dirty t))
  326. (defun spam-stat-buffer-change-to-non-spam ()
  327. "Consider current buffer no longer spam but normal mail."
  328. (setq spam-stat-nbad (1- spam-stat-nbad)
  329. spam-stat-ngood (1+ spam-stat-ngood))
  330. (maphash
  331. (lambda (word count)
  332. (let ((entry (gethash word spam-stat)))
  333. (if (not entry)
  334. (gnus-message 8 "This buffer has unknown words in it")
  335. (spam-stat-set-good entry (+ (spam-stat-good entry) count))
  336. (spam-stat-set-bad entry (- (spam-stat-bad entry) count))
  337. (spam-stat-set-score entry (spam-stat-compute-score entry))
  338. (puthash word entry spam-stat))))
  339. (spam-stat-buffer-words))
  340. (setq spam-stat-dirty t))
  341. ;; Saving and Loading
  342. (defun spam-stat-save (&optional force)
  343. "Save the `spam-stat' hash table as lisp file.
  344. With a prefix argument save unconditionally."
  345. (interactive "P")
  346. (when (or force spam-stat-dirty)
  347. (let ((coding-system-for-write spam-stat-coding-system))
  348. (with-temp-file spam-stat-file
  349. (let ((standard-output (current-buffer))
  350. (font-lock-maximum-size 0))
  351. (insert (format ";-*- coding: %s; -*-\n" spam-stat-coding-system))
  352. (insert (format "(setq spam-stat-ngood %d spam-stat-nbad %d
  353. spam-stat (spam-stat-to-hash-table '(" spam-stat-ngood spam-stat-nbad))
  354. (maphash (lambda (word entry)
  355. (prin1 (list word
  356. (spam-stat-good entry)
  357. (spam-stat-bad entry))))
  358. spam-stat)
  359. (insert ")))"))))
  360. (message "Saved %s." spam-stat-file)
  361. (setq spam-stat-dirty nil
  362. spam-stat-last-saved-at (nth 5 (file-attributes spam-stat-file)))))
  363. (defun spam-stat-load ()
  364. "Read the `spam-stat' hash table from disk."
  365. ;; TODO: maybe we should warn the user if spam-stat-dirty is t?
  366. (let ((coding-system-for-read spam-stat-coding-system))
  367. (cond (spam-stat-dirty (message "Spam stat not loaded: spam-stat-dirty t"))
  368. ((or (not (boundp 'spam-stat-last-saved-at))
  369. (null spam-stat-last-saved-at)
  370. (not (equal spam-stat-last-saved-at
  371. (nth 5 (file-attributes spam-stat-file)))))
  372. (progn
  373. (load-file spam-stat-file)
  374. (setq spam-stat-dirty nil
  375. spam-stat-last-saved-at
  376. (nth 5 (file-attributes spam-stat-file)))))
  377. (t (message "Spam stat file not loaded: no change in disk.")))))
  378. (defun spam-stat-to-hash-table (entries)
  379. "Turn list ENTRIES into a hash table and store as `spam-stat'.
  380. Every element in ENTRIES has the form \(WORD GOOD BAD) where WORD is
  381. the word string, NGOOD is the number of good mails it has appeared in,
  382. NBAD is the number of bad mails it has appeared in, GOOD is the number
  383. of times it appeared in good mails, and BAD is the number of times it
  384. has appeared in bad mails."
  385. (let ((table (make-hash-table :size (length entries)
  386. :test 'equal)))
  387. (mapc (lambda (l)
  388. (puthash (car l)
  389. (spam-stat-make-entry (nth 1 l) (nth 2 l))
  390. table))
  391. entries)
  392. table))
  393. (defun spam-stat-reset ()
  394. "Reset `spam-stat' to an empty hash-table.
  395. This deletes all the statistics."
  396. (interactive)
  397. (setq spam-stat (make-hash-table :test 'equal)
  398. spam-stat-ngood 0
  399. spam-stat-nbad 0)
  400. (setq spam-stat-dirty t))
  401. ;; Scoring buffers
  402. (defvar spam-stat-score-data nil
  403. "Raw data used in the last run of `spam-stat-score-buffer'.")
  404. (defsubst spam-stat-score-word (word)
  405. "Return score for WORD.
  406. The default score for unknown words is stored in
  407. `spam-stat-unknown-word-score'."
  408. (spam-stat-score (gethash word spam-stat)))
  409. (defun spam-stat-buffer-words-with-scores ()
  410. "Process current buffer, return the 15 most conspicuous words.
  411. These are the words whose spam-stat differs the most from 0.5.
  412. The list returned contains elements of the form \(WORD SCORE DIFF),
  413. where DIFF is the difference between SCORE and 0.5."
  414. (let (result word score)
  415. (maphash (lambda (word ignore)
  416. (setq score (spam-stat-score-word word)
  417. result (cons (list word score (abs (- score 0.5)))
  418. result)))
  419. (spam-stat-buffer-words))
  420. (setq result (sort result (lambda (a b) (< (nth 2 b) (nth 2 a)))))
  421. (setcdr (nthcdr 14 result) nil)
  422. result))
  423. (defun spam-stat-score-buffer ()
  424. "Return a score describing the spam-probability for this buffer.
  425. Add user supplied modifications if supplied."
  426. (interactive) ; helps in debugging.
  427. (setq spam-stat-score-data (spam-stat-buffer-words-with-scores))
  428. (let* ((probs (mapcar 'cadr spam-stat-score-data))
  429. (prod (apply #'* probs))
  430. (score0
  431. (/ prod (+ prod (apply #'* (mapcar #'(lambda (x) (- 1 x))
  432. probs)))))
  433. (score1s
  434. (condition-case
  435. spam-stat-error-holder
  436. (spam-stat-score-buffer-user score0)
  437. (error nil)))
  438. (ans
  439. (if score1s (+ score0 score1s) score0)))
  440. (when (interactive-p)
  441. (message "%S" ans))
  442. ans))
  443. (defun spam-stat-score-buffer-user (&rest args)
  444. (let* ((scores
  445. (mapcar
  446. (lambda (fn)
  447. (apply fn args))
  448. spam-stat-score-buffer-user-functions)))
  449. (if (memq nil scores) nil
  450. (apply #'+ scores))))
  451. (defun spam-stat-split-fancy ()
  452. "Return the name of the spam group if the current mail is spam.
  453. Use this function on `nnmail-split-fancy'. If you are interested in
  454. the raw data used for the last run of `spam-stat-score-buffer',
  455. check the variable `spam-stat-score-data'."
  456. (condition-case spam-stat-error-holder
  457. (progn
  458. (set-buffer spam-stat-buffer)
  459. (goto-char (point-min))
  460. (when (> (spam-stat-score-buffer) spam-stat-split-fancy-spam-threshold)
  461. (when (boundp 'nnmail-split-trace)
  462. (mapc (lambda (entry)
  463. (push entry nnmail-split-trace))
  464. spam-stat-score-data))
  465. spam-stat-split-fancy-spam-group))
  466. (error (message "Error in spam-stat-split-fancy: %S" spam-stat-error-holder)
  467. nil)))
  468. ;; Testing
  469. (defun spam-stat-strip-xref ()
  470. "Strip the Xref header."
  471. (save-restriction
  472. (mail-narrow-to-head)
  473. (when (re-search-forward "^Xref:.*\n" nil t)
  474. (delete-region (match-beginning 0) (match-end 0)))))
  475. (autoload 'time-to-number-of-days "time-date")
  476. (defun spam-stat-process-directory (dir func)
  477. "Process all the regular files in directory DIR using function FUNC."
  478. (let* ((files (directory-files dir t "^[^.]"))
  479. (max (/ (length files) 100.0))
  480. (count 0))
  481. (with-temp-buffer
  482. (dolist (f files)
  483. (when (and (file-readable-p f)
  484. (file-regular-p f)
  485. (> (nth 7 (file-attributes f)) 0)
  486. (< (time-to-number-of-days (time-since (nth 5 (file-attributes f))))
  487. spam-stat-process-directory-age))
  488. (setq count (1+ count))
  489. (message "Reading %s: %.2f%%" dir (/ count max))
  490. (insert-file-contents-literally f)
  491. (spam-stat-strip-xref)
  492. (funcall func)
  493. (erase-buffer))))))
  494. (defun spam-stat-process-spam-directory (dir)
  495. "Process all the regular files in directory DIR as spam."
  496. (interactive "D")
  497. (spam-stat-process-directory dir 'spam-stat-buffer-is-spam))
  498. (defun spam-stat-process-non-spam-directory (dir)
  499. "Process all the regular files in directory DIR as non-spam."
  500. (interactive "D")
  501. (spam-stat-process-directory dir 'spam-stat-buffer-is-non-spam))
  502. (defun spam-stat-count ()
  503. "Return size of `spam-stat'."
  504. (interactive)
  505. (hash-table-count spam-stat))
  506. (defun spam-stat-test-directory (dir &optional verbose)
  507. "Test all the regular files in directory DIR for spam.
  508. If the result is 1.0, then all files are considered spam.
  509. If the result is 0.0, non of the files is considered spam.
  510. You can use this to determine error rates.
  511. If VERBOSE is non-nil display names of files detected as spam or
  512. non-spam in a temporary buffer. If it is the symbol `ham',
  513. display non-spam files; otherwise display spam files."
  514. (interactive "DDirectory: ")
  515. (let* ((files (directory-files dir t "^[^.]"))
  516. display-files
  517. buffer-score
  518. (total (length files))
  519. (score 0.0); float
  520. (max (/ total 100.0)); float
  521. (count 0))
  522. (with-temp-buffer
  523. (dolist (f files)
  524. (when (and (file-readable-p f)
  525. (file-regular-p f)
  526. (> (nth 7 (file-attributes f)) 0))
  527. (setq count (1+ count))
  528. (message "Reading %.2f%%, score %.2f"
  529. (/ count max) (/ score count))
  530. (insert-file-contents-literally f)
  531. (setq buffer-score (spam-stat-score-buffer))
  532. (when (> buffer-score 0.9)
  533. (setq score (1+ score)))
  534. (when verbose
  535. (if (> buffer-score 0.9)
  536. (unless (eq verbose 'ham) (push f display-files))
  537. (when (eq verbose 'ham) (push f display-files))))
  538. (erase-buffer))))
  539. (when display-files
  540. (with-output-to-temp-buffer "*spam-stat results*"
  541. (dolist (file display-files)
  542. (princ file)
  543. (terpri))))
  544. (message "Final score: %d / %d = %f" score total (/ score total))))
  545. ;; Shrinking the dictionary
  546. (defun spam-stat-reduce-size (&optional count)
  547. "Reduce the size of `spam-stat'.
  548. This removes all words that occur less than COUNT from the dictionary.
  549. COUNT defaults to 5"
  550. (interactive)
  551. (setq count (or count 5))
  552. (maphash (lambda (key entry)
  553. (when (< (+ (spam-stat-good entry)
  554. (spam-stat-bad entry))
  555. count)
  556. (remhash key spam-stat)))
  557. spam-stat)
  558. (setq spam-stat-dirty t))
  559. (defun spam-stat-install-hooks-function ()
  560. "Install the spam-stat function hooks."
  561. (interactive)
  562. (add-hook 'nnmail-prepare-incoming-message-hook
  563. 'spam-stat-store-current-buffer)
  564. (add-hook 'gnus-select-article-hook
  565. 'spam-stat-store-gnus-article-buffer))
  566. (defun spam-stat-unload-hook ()
  567. "Uninstall the spam-stat function hooks."
  568. (interactive)
  569. (remove-hook 'nnmail-prepare-incoming-message-hook
  570. 'spam-stat-store-current-buffer)
  571. (remove-hook 'gnus-select-article-hook
  572. 'spam-stat-store-gnus-article-buffer))
  573. (add-hook 'spam-stat-unload-hook 'spam-stat-unload-hook)
  574. (provide 'spam-stat)
  575. ;;; spam-stat.el ends here