mvnic/scrap.sh

#/bin/bash
START_NID=$1
NID=$START_NID
END_NID=$2
FILENAME=A${1}-A${2}.json
while [ "$START_NID" -le "$END_NID" ];
do
  echo Scraping: $NID # print
  CONTENT=$(curl -si https://fikureggedhirun.com/api/fetch?nic=A$NID > .cache) # curl output with http headers and data
  DATA=$(cat .cache | tail -n1) # get the json data from last line
  HTTP_STATUS=$(cat .cache | head -n1 | awk '{print $2}') # get the http code from first line

 	if [ "$HTTP_STATUS" = "200" ]
 	then
		if [ "$DATA" = "null" ]
		then
			echo A$NID | tee -a norecords
		else
			echo $DATA | tee -a $FILENAME
			printf "\n" | tee -a $FILENAME
		fi

	START_NID=$(echo $[$START_NID+1])
	printf -v NID "%06d" $START_NID
	sleep 3.5

	else
	 	echo delaying
	 	sleep 1000
	fi
done