woof-CE-libre/woof-code/support/fetch.bac

'
' Small utility to get all BaCon programs over HTTP from the BaCon website
'
' Dec 2010 - PvE.
' Mar 2011: added size check - PvE
' Jul 2013: various improvements, network handling more robust - PvE
'-----------------------------------------------------------------------------------

CONST Sep$ = CHR$(13) & NL$ & CHR$(13) & NL$

' Define website
website$ = "www.basic-converter.org"

' Get the list with files
OPEN website$ & ":80" FOR NETWORK AS mynet
	SEND "GET /files.txt HTTP/1.1\r\nHost: " & website$ & Sep$ TO mynet

	REPEAT
		RECEIVE dat$ FROM mynet
		IF LEN(dat$) > 0 THEN total$ = total$ & dat$
	UNTIL ISFALSE(WAIT(mynet, 500))
CLOSE NETWORK mynet

' Split the text into separate filenames
FOR file$ IN MID$(total$, INSTR(total$, Sep$)+4) STEP NL$

	PRINT "Fetching ", file$, "... ";
	attempt = 0

	REPEAT
		' We create a new connection for each individual file
		OPEN website$ & ":80" FOR NETWORK AS mynet

		' Get the current file
		SEND "GET /" & file$ & " HTTP/1.1\r\nHost: " & website$ & Sep$ TO mynet
		total$ = ""

		' Prevent endless waiting for server
		IF NOT(WAIT(mynet, 250)) THEN
			COLOR FG TO CYAN
			PRINT "Timeout on server! Retrying... ";
			SLEEP 500
			COLOR RESET
			CLOSE NETWORK mynet
			INCR attempt
			IF attempt = 4 THEN
				COLOR FG TO RED
				PRINT "Could not download ", file$, " - skipping."
				SLEEP 500
				COLOR RESET
				BREAK
			END IF
			CONTINUE
		ELSE
			' So there is data
			RECEIVE dat$ FROM mynet

			' Get the filesize from the HTTP header
			dat$ = MID$(dat$, INSTR(dat$, "Content-Length:")+15)
			length = VAL(LEFT$(dat$, INSTR(dat$, NL$)))

			' As long as there is data, get it
			WHILE WAIT(mynet, 500) AND LEN(dat$) > 0
				total$ = total$ & dat$
				RECEIVE dat$ FROM mynet
			WEND

			total$ = total$ & dat$

			' Write to file
			OPEN file$ FOR WRITING AS baconfile
				WRITELN MID$(total$, INSTR(total$, Sep$)+4) TO baconfile
			CLOSE FILE baconfile

			' Closing the connection again
			CLOSE NETWORK mynet
		END IF

		' If the size of the file is not correct
		IF FILELEN(file$) <> length+1 THEN

			' Keep track of attempts
			INCR attempt

			SELECT attempt
				CASE 2;
				CASE 3
					COLOR FG TO CYAN
					PRINT "Failed! Retrying... ";
					SLEEP 500
					COLOR RESET
				CASE 4
					COLOR FG TO RED
					PRINT "Could not download ", file$, " - skipping."
					SLEEP 500
					COLOR RESET
					IF FILEEXISTS(file$) THEN DELETE FILE file$
					BREAK
			END SELECT
		END IF

	' Make sure filelength equals size mentioned in HTTP header +1 for last newline
	UNTIL LEN(total$) > 0 AND FILELEN(file$) = length+1

	IF attempt < 4 THEN
		COLOR FG TO GREEN
		PRINT "done."
		COLOR RESET
	END IF
	
	' Give the remote server some time
	SLEEP 250
NEXT