# Function for parsing _config.yml function yaml() {
hashdot=$(gem list hash_dot); if ! [ "$hashdot" != "" ]; then sudo gem install "hash_dot" ; fi if [ -f $1 ];then cmd=" Hash.use_dot_syntax = true; hash = YAML.load(File.read('$1'));"; if [ "$2" != "" ] ;then cmd="$cmd puts hash.$2;" else cmd="$cmd puts hash;" fi ruby -r yaml -r hash_dot <<< $cmd; fi
}
# Function for slugify string slugify () {
echo "$1" | sed -e 's/’/ /g' | sed -e 's/./ /g' | sed -e 's/[[:punct:]]//g' | sed -e 's/[^[:alnum:]]/-/g' | tr -s '-' | tr A-Z a-z | sed -e 'y/āáǎàēéěèêīíǐìōóǒòôūúǔùûǖǘǚǜĀÁǍÀĒÉĚÈÊĪÍǏÌŌÓǑÒÔŪÚǓÙǕǗǙǛ/aaaaeeeeeiiiiooooouuuuuuuuuuaaaaeeeeiiiiooooouuuuuuuu/'
}
# Declare array separator IFS='£' COLUMNS=1
# Variables for colorize console log red=`tput setaf 1` green=`tput setaf 2` reset=`tput sgr0` bold=$(tput bold)
# Define website array length with jq length=$(jq '. | length' _data/websites.json )
# Create audit array audite=() for ((idx=0; idx<$length; idx++)); do
audit+=$(cat _data/websites.json | jq --arg idx "$idx" '.[$idx|tonumber].name' | tr -d '"' ) audit+=£
done
echo “${green}Select the audit to launch :${reset}” # Menu for command line selection select choice in $audit Exit; do echo $i
for ((idx=0; idx<$length; idx++)); do value=$(cat _data/websites.json | jq --arg idx "$idx" '.[$idx|tonumber].name' | tr -d '"' | sed -e 's/^[ \t]*//') if [[ $choice == $value ]]; then websitetested=$choice continue fi done
### Last done is on the end of this file ###
# Defining config datapath=“_data”
# Test if there is a parameter if [ -z “$websitetested” ]; then
echo "${green}No Website for test is declared - Add a parameter in the commend line${reset}" exit
fi
echo “${green}Main variables${reset}” echo “Audit name: $websitetested” echo “Data path : $datapath” echo websitetestedslug=$(slugify “$websitetested”)
# Defining config (need jq installed) urlstested=($(cat _data/websites.json | jq -r –arg websitetested “$websitetested” '.[] | select(.name==$websitetested) | .pages | .[].url')) pagestested=($(cat _data/websites.json | jq -r –arg websitetested “$websitetested” '.[] | select(.name==$websitetested) | .pages | .[].name'))
length2=$(jq –arg websitetested “$websitetested” '.[] | select(.name==$websitetested) | .pages | length' _data/websites.json )
urlstested=() for ((idx=0; idx<$length2; idx++)); do
urlstested+=($(cat _data/websites.json | jq -r --arg websitetested "$websitetested" --arg idx "$idx" '.[] | select(.name==$websitetested) | .pages | .[$idx|tonumber].url' | tr -d '"' | sed -e 's/^[ \t]*//')) urlstested+=£
done
pagestested=() for ((idx=0; idx<$length2; idx++)); do
pagestested+=($(slugify "$(cat _data/websites.json | jq -r --arg websitetested "$websitetested" --arg idx "$idx" '.[] | select(.name==$websitetested) | .pages | .[$idx|tonumber].name' | tr -d '"' | sed -e 's/^[ \t]*//')")) pagestested+=£
done
# Test if there is a parameter if [ -z “$urlstested” ]; then
echo "${green}No audit test find with this name - try another name${reset}" exit
fi
# Create variables echo “${green}All site pages${reset}” for ((idx=0; idx<${#pagestested}; idx++)); do
echo "$idx" - "${pagestested[idx]//£/}" eval namepage$idx=${pagestested[idx]//£/}
done
echo “${green}All site URL${reset}” for ((idx=0; idx<${#urlstested}; idx++)); do
echo "$idx" - "${urlstested[idx]//£/}" eval urlpage$idx=${urlstested[idx]//£/}
done
# Create folder namespace in _data if [ ! -d $datapath/$websitetestedslug ]; then
mkdir -p $datapath/$websitetestedslug;
fi
if [ ! -d $datapath/$websitetestedslug/$namepage0 ]; then
mkdir -p $datapath/$websitetestedslug/$namepage0;
fi
# Specifics pages tests for ((idx=0; idx<${#urlstested}; idx++)); do
echo "${red}-----------------------------------${reset}" urlpage="urlpage${idx}" namepage="namepage${idx}" domain="${!urlpage#http://}" domain="${domain#https://}" domain=${domain%%/*} hostname="${domain#www.}" date=$(date '+%Y-%m-%d') date="2020-01-23" echo "TEST FOR:" echo "${green}URL:${reset} ${!urlpage}" printf "${green}%-15s | %-15s | %-20s | %-20s \n${reset}" "DATE" "PAGE" "DOMAIN" "HOSTNAME" printf "${red}%-15s | %-15s | %-20s | %-20s \n\n${reset}" "${date}" "${!namepage}" "${domain}" "${hostname}" #echo "${green}url:${reset} ${!urlpage}" "- ${green}namepage:${reset} ${!namepage}" "- ${green}domain:${reset} ${domain}" "- ${green}hostname:${reset} ${hostname}" if [ ! -d $datapath/$websitetestedslug/${!namepage} ]; then mkdir -p $datapath/$websitetestedslug/${!namepage}; fi for VARIABLE in axe gpagespeed lighthouse stylestats wallace pa11y observatory htmlvalidator greenweb cssstats do if [ ! -d $datapath/$websitetestedslug/${!namepage}/$VARIABLE ]; then mkdir -p $datapath/$websitetestedslug/${!namepage}/$VARIABLE; fi done echo "${bold}TESTS LAUNCHED:${reset}" ## AXE if [ ! -f $datapath/$websitetestedslug/${!namepage}/axe/${date}.json ]; then echo "${red}axe${reset} '${!urlpage}' --save $datapath/$websitetestedslug/${!namepage}/axe/${date}.json" axe ${!urlpage} --no-sandbox --save $datapath/$websitetestedslug/${!namepage}/axe/${date}.json else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> axe${reset} '${!urlpage}' --save $datapath/$websitetestedslug/${!namepage}/axe/${date}.json" fi ## PA11Y if [ ! -f $datapath/$websitetestedslug/${!namepage}/pa11y/${date}.json ]; then echo "${red}pa11y-ci${reset} '${!urlpage}' -j > $datapath/$websitetestedslug/${!namepage}/pa11yci/${date}.json" pa11y-ci ${!urlpage} -j > $datapath/$websitetestedslug/${!namepage}/pa11y/${date}.json else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> pa11y-ci${reset} '${!urlpage}' -j > $datapath/$websitetestedslug/${!namepage}/pa11yci/${date}.json" fi ## GPAGESPEED if [ ! -f $datapath/$websitetestedslug/${!namepage}/gpagespeed/${date}.json ]; then echo "${red}gpagespeed${reset} '${!urlpage}' > $datapath/$websitetestedslug/${!namepage}/gpagespeed/${date}.json" gpagespeed ${!urlpage} > $datapath/$websitetestedslug/${!namepage}/gpagespeed/${date}.json else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> gpagespeed${reset} '${!urlpage}' > $datapath/$websitetestedslug/${!namepage}/gpagespeed/${date}.json" fi ## LIGHTHOUSE if [ ! -f $datapath/$websitetestedslug/${!namepage}/lighthouse/${date}.json ]; then echo "${red}lighthouse${reset} '${!urlpage}' --chrome-flags="--headless" --output json --output-path $datapath/$websitetestedslug/${!namepage}/lighthouse/${date}.json" lighthouse ${!urlpage} --chrome-flags="--headless" --quiet --output json --output-path $datapath/$websitetestedslug/${!namepage}/lighthouse/${date}.json else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> lighthouse${reset} '${!urlpage}' --chrome-flags="--headless" --output json --output-path $datapath/$websitetestedslug/${!namepage}/lighthouse/${date}.json" fi ## ExtractCSS if [ ! -f download/${!namepage}-${date}.css ]; then echo "${red}curl${reset} -o download/${!namepage}-${date}.css https://extract-css.now.sh/${domain}" curl -m 8 -o download/${!namepage}-${date}.css https://extract-css.now.sh/${domain} find download -size 0 -delete sed -i '' '/@import/d' $datapath/$websitetestedslug/${!namepage}/stylestats/${date}.json echo "${red}cleancss${reset} --inline remote,${domain} --format beautify -o download/${!namepage}-${date}.css download/${!namepage}-${date}.css" cleancss --inline remote,${domain} --format beautify -o download/${!namepage}-${date}.css download/${!namepage}-${date}.css else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> curl${reset} -o download/${!namepage}-${date}.css https://extract-css.now.sh/${domain}" fi ## GETCSS if [ ! -f download/${!namepage}-${date}.css ]; then echo "${red}getcss${reset} ${!urlpage} > download/${!namepage}-${date}.css" getcss ${!urlpage} > download/${!namepage}-${date}.css sed -i '' '/@import/d' $datapath/$websitetestedslug/${!namepage}/stylestats/${date}.json echo "${red}cleancss${reset} --inline remote,${domain} --format beautify -o download/${!namepage}-${date}.css download/${!namepage}-${date}.css" cleancss --inline remote,${domain} --format beautify -o download/${!namepage}-${date}.css download/${!namepage}-${date}.css else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> getcss${reset} ${!urlpage} > download/${!namepage}-${date}.css" fi ## STYLESTATS if [ ! -f $datapath/$websitetestedslug/${!namepage}/stylestats/${date}.json ]; then echo "${red}stylestats${reset} download/${!namepage}-${date}.css -f json > $datapath/$websitetestedslug/${!namepage}/stylestats/${date}.json" stylestats download/${!namepage}-${date}.css -f json > $datapath/$websitetestedslug/${!namepage}/stylestats/${date}.json sed -i '' '/[ERROR]/d' $datapath/$websitetestedslug/${!namepage}/stylestats/${date}.json sed -i '' -e ':a' -e 'N' -e '$!ba' -e 's/,\n ]/\'$'\n]/g' $datapath/$websitetestedslug/${!namepage}/stylestats/${date}.json else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> stylestats${reset} download/${!namepage}-${date}.css -f json > $datapath/$websitetestedslug/${!namepage}/stylestats/${date}.json" fi ## WALLACE if [ ! -f $datapath/$websitetestedslug/${!namepage}/wallace/${date}.json ]; then echo "${red}wallace${reset} ${!urlpage} -f json > $datapath/$websitetestedslug/${!namepage}/wallace/${date}.json" wallace ${!urlpage} -f json > $datapath/$websitetestedslug/${!namepage}/wallace/${date}.json find download -size 0 -delete else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> wallace${reset} '${!urlpage}' -f json > $datapath/$websitetestedslug/${!namepage}/wallace/${date}.json" fi ## CSSSTATS if [ ! -f $datapath/$websitetestedslug/${!namepage}/cssstats/${date}.json ]; then echo "${red}curl${reset} -m 5 -o $datapath/$websitetestedslug/${!namepage}/cssstats/${date}.json https://${red}cssstats.com${reset}/api/stats\?url\=${hostname}" curl -m 30 -o $datapath/$websitetestedslug/${!namepage}/cssstats/${date}.json https://cssstats.com/api/stats\?url\=${hostname} else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> curl${reset} -m 5 -o $datapath/$websitetestedslug/${!namepage}/cssstats/${date}.json https://cssstats.com/api/stats\?url\=${hostname}" curl -m 30 -o $datapath/$websitetestedslug/${!namepage}/cssstats/${date}.json https://cssstats.com/api/stats\?url\=${hostname} fi ## OBSERVATORY if [ ! -f $datapath/$websitetestedslug/${!namepage}/observatory/${date}.json ]; then echo "${red}observatory${reset} ${hostname} > $datapath/$websitetestedslug/${!namepage}/observatory/${date}.json" observatory ${hostname} > $datapath/$websitetestedslug/${!namepage}/observatory/${date}.json sed -i '' '/^observatory/d' $datapath/$websitetestedslug/${!namepage}/observatory/${date}.json find download -size 0 -delete else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> observatory${reset} ${hostname} > $datapath/$websitetestedslug/${!namepage}/observatory/${date}.json" sed -i '' '/^observatory/d' $datapath/$websitetestedslug/${!namepage}/observatory/${date}.json fi ## HTML-VALIDATOR if [ ! -f $datapath/$websitetestedslug/${!namepage}/htmlvalidator/${date}.json ]; then echo "${red}html-validator${reset} ${!urlpage} --format=json a --verbose > $datapath/$websitetestedslug/${!namepage}/sitevalidator/${date}.json" #html-validator ${!urlpage} --format=json a --verbose > $datapath/$websitetestedslug/${!namepage}/htmlvalidator/${date}.json else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> html-validator${reset} ${!urlpage} --format=json a --verbose > $datapath/$websitetestedslug/${!namepage}/sitevalidator/${date}.json" fi ## GREENWEB if [ ! -f $datapath/$websitetestedslug/${!namepage}/greenweb/${date}.json ]; then echo "${red}curl${reset} http://api.thegreenwebfoundation.org/greencheck/${hostname} > $datapath/$websitetestedslug/${!namepage}/greenweb/${date}.json" curl -m 5 http://api.thegreenwebfoundation.org/greencheck/${hostname} > $datapath/$websitetestedslug/${!namepage}/greenweb/${date}.json else echo "${green}Test already launched, try this command for a new manual test:${reset}" echo "${red}=> curl${reset} http://api.thegreenwebfoundation.org/greencheck/${hostname} > $datapath/$websitetestedslug/${!namepage}/greenweb/${date}.json" fi ## SITE-VALIDATOR ## if [ ! -f $$datapath/$websitetestedslug/${!namepage}/sitevalidator/${date} ]; then ## echo "${red}site-validator${reset} '${!urlpage}' --output json > $datapath/$websitetestedslug/${!namepage}/sitevalidator/${date}.json" ## site-validator ${!urlpage} --output $datapath/$websitetestedslug/${!namepage}/sitevalidator/${date} ## else ## echo "${green}Test already launched, try this command for a new manual test:${reset}" ## echo "${red}=> site-validator${reset} '${!urlpage}' --output json > $datapath/$websitetestedslug/${!namepage}/sitevalidator/${date}.json" ## fi ## SEOCLI ## if [ ! -f $datapath/$websitetestedslug/${!namepage}/seocli/${date}.json ]; then ## echo "${red}./${reset}vendor/bin/seocli -u '${!urlpage}' -d 3 -f json > $datapath/$websitetestedslug/${!namepage}/seocli/${date}.json" ## ./vendor/bin/seocli -u ${!urlpage} -d 3 -f json > $datapath/$websitetestedslug/${!namepage}/seocli/${date}.json ## else ## echo "${green}Test already launched, try this command for a new manual test:${reset}" ## echo "${red}=> ./${reset}vendor/bin/seocli -u '${!urlpage}' -d 3 -f json > $datapath/$websitetestedslug/${!namepage}/seocli/${date}.json" ## fi ## YELLOWLABTOOLS ## if [ ! -f $datapath/$websitetestedslug/${!namepage}/yellowlabtools/${date}.json ]; then ## echo "${red}yellowlabtools${reset} '${!urlpage}' > $datapath/$websitetestedslug/${!namepage}/yellowlabtools/${date}.json" ## # yellowlabtools ${!urlpage} > $datapath/$websitetestedslug/${!namepage}/yellowlabtools/${date}.json ## else ## echo "${green}Test already launched, try this command for a new manual test:${reset}" ## echo "${red}=> yellowlabtools${reset} '${!urlpage}' > $datapath/$websitetestedslug/${!namepage}/yellowlabtools/${date}.json" ## fi
done
echo “${red}Deleting Empty files${reset}” find . -empty -type d -delete find . -size 0 -delete ##sed -i '' -e ':a' -e 'N' -e '$!ba' -e 's/,n ]/'$'n]/g'
# General site tests # seobutler.com/badass-seo-automate-screaming-frog/ # open “/Applications/Screaming Frog SEO Spider.app” –args –crawl $urlpage0 # curl www.googleapis.com/pagespeedonline/v5/runPagespeed?url=https://www.leforestier-immobilier.com/fr&key=AIzaSyATvQKCg0r2tKjt9OhEHXuqlpVMjHby730 > aa.json exit done done