User Tools

Site Tools


zz_deprecated:infojobs_spider

This is an old revision of the document!


jobs_mailer.sh
#!/bin/bash
 
TMPFILE="/tmp/jobs_today_$(date +%s).html"
SENDFILE="/tmp/jobs_save_today_$(date +%s).html"
OFFERMARK="table_results_offer"
INFOTAGS="prefijoPuesto prefijoPoblacion prefijoEmpresa"
#AVOIDTHIS="Sant Cugat Del Vall|Sabadell"
AVOIDTHIS="monguiland"
 
get_data()
{
        #wget -O- -q $SEARCHURL | grep -A 6 "<div class=\"cell-text\">$(date +%d/%m)</div>" > $TMPFILE
        local AUXOFFERID=""
        local AUXCOMPANY=""
        local AUXDESC=""
        local AUXURL=""
        wget -O- -q $SEARCHURL | grep -A 15 "<div class=\"cell-text\">$(date +%d/%m)</div>" | egrep "${OFFERMARK}" | awk -F\' '{print $4}' | sed "s,',,g" | head -100 | while read AUXOFFERID
        do
                if [ $(sqlite3 $DBFILE "SELECT count(OFFERID) from offers where OFFERID like '$AUXOFFERID' ;") -eq 0 ]
                then
                        sqlite3 $DBFILE "INSERT INTO offers (id, OFFERID, is_sent) VALUES(NULL, '$AUXOFFERID', 1)  ;"
                fi
        done
}
 
createdb()
{       
        if ! [ -f $DBFILE ]
        then    
                sqlite3 $DBFILE "create table offers (id INTEGER PRIMARY KEY, OFFERID text, is_sent boolean) ;"
        fi
}
 
mailer()
{
        local let UNSENTJOBS=$(sqlite3 $DBFILE "SELECT count(id) from offers where is_sent=1 ;")
        local SENDME=""
        if [ ${UNSENTJOBS} -gt 10 ]
        then
                > $SENDFILE
                sqlite3 $DBFILE "SELECT OFFERID from offers where is_sent=1 ;" | while read AUXOFFERID ; do
                        echo "http://www.infojobs.net/barcelona/linux/of-i${AUXOFFERID}                       " >> $SENDFILE
                        #wget -O- -q "http://www.infojobs.net/barcelona/linux/of-i${AUXOFFERID}" | egrep "${INFOTAGS// /|}"  | awk 'BEGIN{FS = "[<>\"]"} {print $9}' >> $SENDFILE
                        SENDME=$(wget -O- -q "http://www.infojobs.net/barcelona/linux/of-i${AUXOFFERID}" | egrep "${INFOTAGS// /|}"  | awk 'BEGIN{FS = "[<>\"]"} {print $9}')
                        if [ ! "$(echo "${SENDME}" | egrep "${AVOIDTHIS}")" ]; then
                                echo "${SENDME}" >> $SENDFILE
                        else
                                sqlite3 $DBFILE "UPDATE offers set is_sent=0 where OFFERID LIKE '${AUXOFFERID}' ;"
                        fi
                done
                sqlite3 $DBFILE "UPDATE offers set is_sent=0 where is_sent=1 ;"
                cat $SENDFILE | mail -s "Jobs $(date +%Y/%m/%d_%H:%M)" $MAIL
        fi
}
clean_all()
{
        for i in $SENDFILE $TMPFILE
        do
                if [ -f $i ]
                then
                        rm -f $i
                fi
        done
}
for i in $(dirname $0)/*.conf
#for i in $(dirname $0)/jholgado-v3.conf
do
        . $i
        createdb
        get_data
        mailer
        clean_all
done
zz_deprecated/infojobs_spider.1330008539.txt.gz ยท Last modified: 2012/02/23 14:49 (external edit)