various/crawler/crawl.sh

#!/bin/bash
# Crawl at a thread rate

threads=4
urls="urls.txt"
output="db.txt"

x=1
while IFS= read -r line
do
    echo "Off crawler number ${x}!"
    php crawl.php $line $output.$x > webcrawl.log &
    ((x=x+1))
done < $urls