Bots/wikipedia-crawler.scm

;; (use-modules (ice-9 receive)
;; 	     (web client)
;; 	     (web request)
;; 	     (web http)
;; 	     (web uri))

;; (http-get "https://en.wikipedia.org/w/api.php?action=query&titles=Main%20Page&prop=revisions&rvprop=content&format=xml")
;;  (http-get (parse-header 'content-location  "https://en.wikipedia.org/w/api.php?action=query&titles=Linux&prop=revisions&rvprop=content&format=xml"))

;; (parameterize ((current-http-proxy #f))  (http-get "https://en.wikipedia.org/w/api.php?action=query&titles=Linux&prop=revisions&rvprop=content&format=xml"))

;; (define a (open-output-file "uno.txt"))
;; scheme@(guile-user)> (display "?" a)
;; scheme@(guile-user)> (newline a)
;; scheme@(guile-user)> (close-output-port a)
;; scheme@(guile-user)> (newline a)
;; scheme@(guile-user)> (close-output-port a)
;; scheme@(guile-user)> (define a (open-output-file "uno.txt"))
;; scheme@(guile-user)> (display (parameterize ((current-http-proxy #f))  (http-get "https://en.wikipedia.org/w/api.php?action=query&titles=Linux&prop=revisions&rvprop=content&format=xml" #:keep-alive? #t)) a)
;https://en.wikipedia.org/w/api.php?format=xml&action=query&prop=extracts&exintro=&explaintext=&titles=Stack%20Overflow

(use-modules (ice-9 receive)
	     (web client)
	     (sxml simple))
 

;; ;;; Using receive.
;; (receive (head body)
;;      (http-get "http://gnu.org/")
;;      (begin
;;        ;; Display HTTP Response
;;        (newline)
;;        (display "RESPONSE:")
;;        (newline)
;;        (display head)
;;        ;; Display HTTP Response body
;;        (newline)
;;        (display "BODY:")
;;        (newline)
;;        (display body)
;;        (newline)))

;; ;texto casi completo
;; (define b (car (cdr (cdr (xml->sxml a)))))
;; ;titulo del documento!!
;; (define c (car (cdr (cdr (cdr b)))))
;; ;not yet
;; (define d (caddr c))
;; ;ya casi!
;; (define g (cadr d))
;; ;;; mejor aun!
;; ;https://en.wikipedia.org/w/api.php?format=xml&action=query&prop=extracts&exintro=&explaintext=&titles=Stack%20Overflow
;; (substring (texto (xml->sxml nautilus)) (string-contains (texto (xml->sxml nautilus))  "'''''Nautilus'''''"))

;; (define titulo (cadr (car (cdr (car (cdr g))))))

;; (define m (car (cdr (cdr g))))

(use-modules (ice-9 receive)
	     (web client)
	     (sxml simple))


(define (last lis)
  (cond [(and (not (pair? (car lis)))
	      (null? (cdr lis))) (car lis)]
	[(not (null? (cdr lis))) (last (cdr lis))]
	[else (last (car lis))]))
(define texto last)
;;; recive un texto "Stack%20Overflow"
;; y extrae las primeras 0-200 letras con substring
; al llamar http-get en (query text)
;;; lo convierte de xml a sxml
;;; y me da texto, es decir el (last list)
;;; wikipedia xml te da el ultimo como xml
(define (search text)
  (substring
   (texto (xml->sxml
	   (receive (head body)
	       (http-get (query text))
	     body)))
   0
   300))

;; como llamar a wikipedia
;;;; agregar para que convierta espacios a %20
(define (query text)
  (format #f "https://en.wikipedia.org/w/api.php?format=xml&action=query&prop=extracts&exintro=&explaintext=&titles=~a" text))


(define (wikipedia text)
  (string-append (search text)
		 "... More at: "
		 (link-w text)))

;;;; agregar para que convierta espacios a guion bajo _
(define (link-w text)
  (format #f "https://en.wikipedia.org/wiki/~a" text))