mvnic/dhiraaguscrap/scrap.sh

#/bin/bash
START_NUMBER=$1
NUMBER=$START_NUMBER
END_NUMBER=$2
FILENAME=${1}-${2}.json
while [ "$START_NUMBER" -le "$END_NUMBER" ];
do
  echo Scraping: $NUMBER # print
  CONTENT=$(curl -si https://app-production.dhiraagu.com.mv/io/v1/info/subscribers/$NUMBER/dir > .tmpcache) # curl output with http headers and data
  DATA=$(cat .tmpcache | tail -n1) # get the json data from last line
  HTTP_STATUS=$(cat .tmpcache | head -n1 | awk '{print $2}') # get the http code from first line

        if [ "$HTTP_STATUS" = "200" ]
        then
                if [ "$DATA" = "null" ]
                then
                        echo $NUMBER | tee -a norecords
                else
                        echo $DATA | tee -a $FILENAME
#                       printf "\n" | tee -a $FILENAME
                fi

        START_NUMBER=$(echo $[$START_NUMBER+1])
#        printf -v NUMBER "%06d" $START_NUMBER
        sleep 3.5

        else
                echo delaying
                sleep 70
        fi
done