diff --git a/README.md b/README.md deleted file mode 100644 index f5b06357..00000000 --- a/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# mvnic - -scrapped json data from mdp \ No newline at end of file diff --git a/scrap.sh b/scrap.sh new file mode 100755 index 00000000..41ef3915 --- /dev/null +++ b/scrap.sh @@ -0,0 +1,27 @@ +#/bin/bash +START_NID=$1 +END_NID=$2 +FILENAME=A${1}-A${2}.json +while true; +do + echo Scraping: $START_NID + CONTENT=$(curl -si https://fikureggedhirun.com/api/fetch?nic=A$START_NID > .cache) + DATA=$(cat .cache | tail -n1) + HTTP_STATUS=$(cat .cache | head -n1 | awk '{print $2}') + + if [ "$HTTP_STATUS" = "200" ] + then + echo $DATA | tee -a $FILENAME + printf "\n" | tee -a $FILENAME + START_NID=$(echo $[$START_NID+1]) + sleep 5 + else + echo delaying + sleep 1000 + fi + + if [ "$END_NID" = "$START_NID" ] + then + exit 1 + fi +done diff --git a/split.sh b/split.sh new file mode 100755 index 00000000..4c4ce572 --- /dev/null +++ b/split.sh @@ -0,0 +1,21 @@ +#!/bin/bash + + +LINE_NO=1 +NUMBER_OF_LINES=$(cat $1 | wc -l) +mkdir split -p +while true +do + +LINE=$(cat $1 | head -n$LINE_NO | tail -n1) +FILENAME=$(echo $LINE | jq -r ".nic").json +echo $LINE | tee split/$FILENAME + +if [ "$NUMBER_OF_LINES" = "$LINE_NO" ] +then + exit +fi +LINE_NO=$(echo $[$LINE_NO+1]) + + +done