86 lines
3.4 KiB
Bash
86 lines
3.4 KiB
Bash
#!/bin/bash
|
|
|
|
# API, API KEY, BOT TOKEN, CHATIDS
|
|
#SCRAPPER_CONTROL_API_KEY=<from env>
|
|
#TG_BOT_TOKEN=<from env>
|
|
#CHAT_ID=<from env>
|
|
SCRAP_URL="https://app-production.dhiraagu.com.mv/io/v1/info/subscribers"
|
|
SCRAPPER_CONTROL_API_URL="https://scraper-control.awfulshit.xyz/api/dhiraagu-number"
|
|
TELEGRAM_API_URL=
|
|
|
|
sleep=$((RANDOM % 21)); echo Sleeping.. $sleep; sleep $sleep
|
|
|
|
# REPEAT AFTER FILE UPLOAD
|
|
while true; do
|
|
# LOOKING FOR FREE BITCHES
|
|
GET_FREE_BATCH=$(curl --request GET --url "$SCRAPPER_CONTROL_API_URL"/next-batch --header "Authorization: Bearer $SCRAPPER_CONTROL_API_KEY" --header "Content-Type: application/json")
|
|
BATCH_NUMBER=$(echo $GET_FREE_BATCH | jq -r '.batch_number')
|
|
echo "First availible batch: $BATCH_NUMBER."
|
|
|
|
# CLAIMING A FREE BITCH
|
|
CLAIM_FREE_BATCH=$(curl --request POST --url "$SCRAPPER_CONTROL_API_URL"/next-batch/$BATCH_NUMBER --header "Authorization: Bearer $SCRAPPER_CONTROL_API_KEY" --header "Content-Type: application/json")
|
|
BATCH_START=$(echo $GET_FREE_BATCH | jq -r '.batch_start')
|
|
BATCH_END=$(echo $GET_FREE_BATCH | jq -r '.batch_end')
|
|
|
|
# CANCEL THE BITCH ON STOP
|
|
function CANCEL {
|
|
echo "SCRIPT STOP REQUESTED, POSTING A CANCEL REQUEST TO SCRAPPER_CONTROL_API_URL"
|
|
CANCEL_BATCH=$(curl --request POST --url "$SCRAPPER_CONTROL_API_URL"/next-batch/$BATCH_NUMBER/cancel --header "Authorization: Bearer $SCRAPPER_CONTROL_API_KEY" --header "Content-Type: application/json")
|
|
echo "BATCH CANCELLED. STOPPING SCRIPT."
|
|
rm $FILENAME
|
|
exit
|
|
}
|
|
|
|
|
|
# SCRAP THE FREE BITCH
|
|
START_NUM=$BATCH_START
|
|
NUM=$START_NUM
|
|
END_NUM=$((BATCH_END+1))
|
|
FILENAME=${BATCH_START}-${BATCH_END}.json
|
|
|
|
# CHECK IF RANFE IS ASCENDING OR DESCENDING
|
|
if [ "$START_NUM" -le "$END_NUM" ]; then
|
|
step=1
|
|
else
|
|
step=-1
|
|
fi
|
|
|
|
# UPDATE THE LOOP DEPENDING ON ABOVE STEP
|
|
while [ "$START_NUM" -ne "$END_NUM" ]; do
|
|
echo "Scraping: $NUM" # LOG SCRAPPING NUMBER
|
|
curl -si --socks5 127.0.0.1:9050 "$SCRAP_URL/$NUM/dir" > .cache
|
|
DATA=$(tail -n1 .cache) # GET JSON DATA
|
|
HTTP_STATUS=$(head -n1 .cache | awk '{print $2}') # GET HEADER
|
|
trap CANCEL SIGINT
|
|
|
|
if [ "$HTTP_STATUS" = "200" ]; then # IF DATA IS AVAILIBLE
|
|
echo "$DATA" | tee -a "$FILENAME"
|
|
printf "\n" | tee -a "$FILENAME"
|
|
START_NUM=$(expr "$START_NUM" + "$step")
|
|
printf -v NUM "%06d" "$START_NUM"
|
|
else
|
|
echo "";echo "COULD NOT GET DATA, GETTING NEW IP.";
|
|
#systemctl restart tor
|
|
#service tor restart
|
|
#echo -e 'AUTHENTICATE ""\r\nSIGNAL NEWNYM\r\nQUIT' | nc -U /var/run/tor/control
|
|
#printf 'AUTHENTICATE ""\r\nSIGNAL NEWNYM\r\nQUIT' | nc localhost 9051
|
|
printf 'AUTHENTICATE ""\r\nSIGNAL NEWNYM\r\nQUIT' | nc -w 1 localhost 9051
|
|
echo ""
|
|
sleep 1
|
|
fi
|
|
done
|
|
|
|
|
|
|
|
# COMPLETED SCRAPPING BITCH
|
|
BATCH_COMPLETE=$(curl --request POST --url "$SCRAPPER_CONTROL_API_URL"/next-batch/$BATCH_NUMBER/finish --header "Authorization: Bearer $SCRAPPER_CONTROL_API_KEY" --header "Content-Type: application/json")
|
|
BATCH_NUMBER=$(echo $BATCH_COMPLETE | jq -r '.batch.batch_number')
|
|
COMPLETED_BY=$(echo $BATCH_COMPLETE | jq -r '.batch.completed_by.name')
|
|
CAPTION=$(echo -e "$FILENAME COMPLETED.\nBATCH $BATCH_NUMBER COMPLETED BY $COMPLETED_BY." | tr '[:lower:]' '[:upper:]')
|
|
echo $CAPTION
|
|
|
|
# UPLOAD SCRAPPED BITCH FILE TO TELEGRAM
|
|
curl -s -F chat_id=$CHAT_ID -F caption="$CAPTION" -F document=@$FILENAME "https://api.telegram.org/bot$TG_BOT_TOKEN/sendDocument"
|
|
rm -v $FILENAME
|
|
done
|