diff --git a/dhiraaguscrap/scrap.sh b/dhiraaguscrap/scrap.sh new file mode 100644 index 00000000..41f82bd6 --- /dev/null +++ b/dhiraaguscrap/scrap.sh @@ -0,0 +1,31 @@ +#/bin/bash +START_NID=$1 +NID=$START_NID +END_NID=$2 +FILENAME=${1}-${2}.json +while [ "$START_NID" -le "$END_NID" ]; +do + echo Scraping: $NID # print + CONTENT=$(curl -sihttps://app-production.dhiraagu.com.mv/io/v1/info/subscribers/$NID/dir > .cache) # curl output with http headers and data + DATA=$(cat .cache | tail -n1) # get the json data from last line + HTTP_STATUS=$(cat .cache | head -n1 | awk '{print $2}') # get the http code from first line + + if [ "$HTTP_STATUS" = "200" ] + then + if [ "$DATA" = "null" ] + then + echo $NID | tee -a norecords + else + echo $DATA | tee -a $FILENAME +# printf "\n" | tee -a $FILENAME + fi + + START_NID=$(echo $[$START_NID+1]) + printf -v NID "%06d" $START_NID + sleep 3.5 + + else + echo delaying + sleep 70 + fi +done