# HG changeset patch # User edhoprima@gmail.com # Date 1260851986 0 # Node ID 5d3a0645b5049f79590b9f8a4690601ac484bf9a # Parent 8b1f6f6b6a3b25f10022f767d292030814bcdc00 - Restructured some things. - Removed -- from commands. Replaced with safe_path command. - Updated help. - All global variables should be initally set at init_globals command. diff -r 8b1f6f6b6a3b -r 5d3a0645b504 moefetch.sh --- a/moefetch.sh Mon Dec 14 09:57:35 2009 +0000 +++ b/moefetch.sh Tue Dec 15 04:39:46 2009 +0000 @@ -43,18 +43,39 @@ ### WILL BE FOR 0.3 # useless welcome message. Also version -Msg_Welcome() { - MOEFETCHVERSION="0.3-beta2" - echo "moefetch ${MOEFETCHVERSION} +msg_welcome() { + echo "moefetch ${_version} Copyright (c) 2009 edogawaconan " } -get_md5() { cat -- "$1" | openssl dgst -md5; } -get_basename() { basename /"$1"; } +# Sanitize path. Totally safe. Usage: cmd "$(safe_path "${filename}")" +safe_path() +{ + # It all depends on the first character. + __start=$(printf "%s" "$*" | cut -c 1) + __path= + case "${__start}" in + .|/) __path="$*";; # . and / is safe. No change. + *) __path="./$*";; # Anything else must be prefixed with ./ + esac + printf "%s" "${__path}" # Return. +} + +# Checks md5. OpenSSL should be available on anything usable. +get_md5() { cat "$(safe_path "${1}")" | openssl dgst -md5; } + +# Safely get basename. +get_basename() { basename "$(safe_path "${1}")"; } + +# Safely get filename (basename without the extension). get_filename() { get_basename "${1%.*}"; } -get_cleantags() { printf "%s " "$@" | sed -e 's/\&/%26/g;s/=/%3D/g'; } -Is_NotMD5() { get_filename "$1" | sed -e 's/\([0-9a-f]\{32\}\)//g'; } + +# Transformation for tag url. +get_cleantags() { printf "%s " "$*" | sed -e 's/\&/%26/g;s/=/%3D/g'; } + +# Returns something if not an md5 value. +is_not_md5() { get_filename "$1" | sed -e 's/\([0-9a-f]\{32\}\)//g'; } # fatal error handler @@ -73,15 +94,25 @@ # help message Err_Help() { - echo "moefetch.sh COMMAND [-s SITE_URL] TAGS + echo "moefetch.sh COMMAND [-n] [-p PASSWORD] [-s SITE_URL] [-u USERNAME] TAGS COMMAND: -(quick)fetch: do a complete update. Add prefix quick to skip file checking -check: get list of new files, clean up local folder and print total new files + (quick)fetch: + Do a complete update. Add prefix quick to skip file checking + check: + Get list of new files, clean up local folder and print total new files --s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE} - -TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme" +OPTIONS: + -n: + Skip checking repository directory. + -p PASSWORD: + Specifies password for login. + -s SITE_URL: + Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE}. + -u USERNAME: + Specifies username for login. + TAGS: + Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme." exit 2 } @@ -89,11 +120,11 @@ Generate_Link() { echo " Fetching XML file" - tempnum=1000 - iternum=1 + __tempnum=1000 + __iternum=1 > "${TEMP_PREFIX}-list" - while [ "${tempnum}" -ge 1000 ]; do - __url="http://${SITE}/post/index.xml?tags=$(get_cleantags "${TAGS}")&offset=0&limit=1000&page=${iternum}" + while [ "${__tempnum}" -ge 1000 ]; do + __url="http://${SITE}/post/index.xml?tags=$(get_cleantags "${TAGS}")&offset=0&limit=1000&page=${__iternum}" [ ${_use_login} -eq 1 ] && __url="${__url}&login=${LOGIN_USER}&password_hash=${LOGIN_PASS}" wget "${__url}" -O "${TEMP_PREFIX}-xml" -e continue=off || Err_Fatal "Failed download catalog file" printf "Processing XML file... " @@ -106,10 +137,10 @@ EOF - tempnum=$(echo $(wc -l < "${TEMP_PREFIX}-templist")) - iternum=$((iternum + 1)) + __tempnum=$(echo $(wc -l < "${TEMP_PREFIX}-templist")) + __iternum=$((__iternum + 1)) cat "${TEMP_PREFIX}-templist" >> "${TEMP_PREFIX}-list" - echo "${tempnum} file(s) available" + echo "${__tempnum} file(s) available" done numfiles=$(echo $(wc -l < "${TEMP_PREFIX}-list")) echo "${numfiles} file(s) available on server" @@ -117,12 +148,12 @@ } -Progress_Init() { +progress_init() { _last="-" printf "${_last}" } -Progress_Anim() { +progress_anim() { case "${_last}" in /) _last="-";; -) _last=\\;; @@ -132,7 +163,7 @@ printf "\b${_last}" } -Progress_Done() { printf "\bdone\n"; } +progress_done() { printf "\bdone\n"; } # getting rid of ls (as per suggestion) Count_Files() { @@ -178,7 +209,7 @@ Cleanup_Repository() { # current dir: ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR} printf "Cleaning up repository folder... " - Progress_Init + progress_init trash_dir="${BASE_DIR}/trash/${trash_dir}/$(date -u "+${SITE_DIR}-${TARGET_DIR}-%Y%m%d-%H.%M")" trashes="These files have been moved to ${trash_dir}:" has_trash= @@ -192,17 +223,17 @@ for trash in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* do is_trash= - if [ -d "${trash}" ] || [ -n "$(Is_NotMD5 "${trash}")" ] || [ -z "$(grep "$(get_basename "${trash}")" "${TEMP_PREFIX}-list")" ]; then + if [ -d "${trash}" ] || [ -n "$(is_not_md5 "${trash}")" ] || [ -z "$(grep "$(get_basename "${trash}")" "${TEMP_PREFIX}-list")" ]; then is_trash=1 has_trash=1 mv -f -- "${trash}" "${trash_dir}" || Err_Impossible trashes="${trashes} $(get_basename "${trash}")" fi - Progress_Anim + progress_anim done rmdir "${trash_dir}" 2>/dev/null - Progress_Done + progress_done [ -n "${has_trash}" ] && echo "${trashes}" } @@ -211,55 +242,55 @@ if [ ! -n "${ISNEW}" ]; then [ -z "${NOCLEAN}" ] && Cleanup_Repository printf "Checking for errors... " - Progress_Init - files_error="These files do not match its md5:" - files_notdanbooru="These files are not checked:" - has_err_filename= - has_err_md5= + progress_init + __files_error="These files do not match its md5:" + __files_notdanbooru="These files are not checked:" + __has_err_filename= + __has_err_md5= > "${TEMP_PREFIX}-error" > "${TEMP_PREFIX}-ok" for file in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* do if [ "${file}" != "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/*" ]; then - if [ -n "$(Is_NotMD5 "${file}")" ] || [ -d "${file}" ]; then - files_notdanbooru="${files_notdanbooru} + if [ -n "$(is_not_md5 "${file}")" ] || [ -d "${file}" ]; then + __files_notdanbooru="${__files_notdanbooru} $(get_basename "${file}")" - has_err_filename=1 + __has_err_filename=1 else if [ "$(get_md5 "${file}")" = "$(get_filename "${file}")" ]; then echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-ok" else rm "${file}" || Err_Fatal "Error removing ${file}" echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-error" - files_error="${files_error} + __files_error="${__files_error} $(get_basename "${file}")" - has_err_md5=1 + __has_err_md5=1 fi fi fi - Progress_Anim + progress_anim done - Progress_Done - if [ ! -n "${has_err_md5}" ] && [ ! -n "${has_err_filename}" ]; then + progress_done + if [ ! -n "${__has_err_md5}" ] && [ ! -n "${__has_err_filename}" ]; then echo "All files OK" else - if [ -n "${has_err_md5}" ]; then - echo "${files_error}" + if [ -n "${__has_err_md5}" ]; then + echo "${__files_error}" echo "$(echo $(wc -l < "${TEMP_PREFIX}-error")) file(s) removed" fi - [ -n "${has_err_filename}" ] && echo "${files_notdanbooru}" + [ -n "${__has_err_filename}" ] && echo "${__files_notdanbooru}" fi echo "$(echo $(wc -l < "${TEMP_PREFIX}-ok")) file(s) available locally" printf "Generating list of new files... " - Progress_Init + progress_init cp -f "${TEMP_PREFIX}-list" "${TEMP_PREFIX}-templist" while read -r is_ok; do grep -v "${is_ok}" "${TEMP_PREFIX}-templist" > "${TEMP_PREFIX}-newlist" cp -f "${TEMP_PREFIX}-newlist" "${TEMP_PREFIX}-templist" || Err_Impossible - Progress_Anim + progress_anim done < "${TEMP_PREFIX}-ok" - Progress_Done + progress_done echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded" else if [ -n "${ISQUICK}" ]; then @@ -297,8 +328,6 @@ # misc variables ISQUICK= ISNEW= - # variable to check whether a login is used or not - _use_login=0 # minimum number of arguments: 2 (command and tag). If less than two, exit and print help message [ $# -lt 2 ] && Err_Help @@ -348,10 +377,20 @@ TEMP_PREFIX="${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}" } +# global variables goes here +init_globals() +{ + _version="0.3-beta3" # version of this script + _use_login=0 # variable to check whether a login is used or not +} + main() { + #initialize global variables + init_globals + #print welcome message + msg_welcome # initialization - Msg_Welcome init "$@" Check_Tools Check_Folders