init

README.md

@@ -1,3 +0,0 @@
-# mvnic
-
-scrapped json data from mdp

scrap.sh

@@ -0,0 +1,27 @@
+#/bin/bash
+START_NID=$1
+END_NID=$2
+FILENAME=A${1}-A${2}.json
+while true;
+do
+  echo Scraping: $START_NID
+  CONTENT=$(curl -si https://fikureggedhirun.com/api/fetch?nic=A$START_NID > .cache)
+  DATA=$(cat .cache | tail -n1)
+  HTTP_STATUS=$(cat .cache | head -n1 | awk '{print $2}')
+
+ 	if [ "$HTTP_STATUS" = "200" ]
+ 	then
+		  echo $DATA | tee -a $FILENAME
+		  printf "\n" | tee -a $FILENAME
+		  START_NID=$(echo $[$START_NID+1])
+		  sleep 5
+	else
+	 	echo delaying
+	 	sleep 1000
+	fi
+
+	if [ "$END_NID" = "$START_NID" ]
+	then
+		exit 1
+	fi
+done

split.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+
+LINE_NO=1
+NUMBER_OF_LINES=$(cat $1 | wc -l)
+mkdir split -p
+while true
+do
+
+LINE=$(cat $1 | head -n$LINE_NO | tail -n1)
+FILENAME=$(echo $LINE | jq -r ".nic").json
+echo $LINE | tee split/$FILENAME
+
+if [ "$NUMBER_OF_LINES" = "$LINE_NO"   ]
+then
+	exit
+fi
+LINE_NO=$(echo $[$LINE_NO+1])
+
+
+done