mvnic/scrap.sh

#/bin/bash
START_NID=$1
NID=$START_NID
END_NID=$2
FILENAME=A${1}-A${2}.json
mkdir -p null
while [ "$START_NID" -le "$END_NID" ];
do
  echo Scraping: $NID # print
  CONTENT=$(curl -si https://fikureggedhirun.com/api/fetch?nic=A$NID > .cache) # curl output with http headers and data
  DATA=$(cat .cache | tail -n1) # get the json data from last line
  HTTP_STATUS=$(cat .cache | head -n1 | awk '{print $2}') # get the http code from first line

 	if [ "$HTTP_STATUS" = "200" ]
 	then
		if [ "$DATA" = "null" ]
		then
			echo A$NID > null/A$NID
		else
		  echo $DATA | tee -a $FILENAME
		  printf "\n" | tee -a $FILENAME
		  START_NID=$(echo $[$START_NID+1])
		  printf -v NID "%06d" $START_NID
		  sleep 1.5
		fi
	else
	 	echo delaying
	 	sleep 1000
	fi
done