Mercurial > ec-dotfiles
diff moefetch.sh @ 225:265a9ca47a19
- Replaced md5(sum) with openssl. Less platform dependent because the tool is same across platforms
- Preparation for name-safe file operations.
- Sanity against sed
- Correctly encoded url (hopefully)
- getopts replacing fail hackjob with while and shift
- support for login (provided with openssl)
author | edhoprima |
---|---|
date | Fri, 13 Nov 2009 18:47:02 +0000 |
parents | 0ac1805621d4 |
children | f8be4a3d3b4a |
line wrap: on
line diff
--- a/moefetch.sh Sat Jul 18 09:10:42 2009 +0000 +++ b/moefetch.sh Fri Nov 13 18:47:02 2009 +0000 @@ -18,15 +18,11 @@ # USE WITH CARE # # what it does: fetch every picture that has the specified TAGS. -# requirement: wget, libxslt, md5sum (or md5) +# requirement: wget, libxslt, openssl -# program additional paths for: cut, sed, wc, MD5(sum), wget, xsltproc, grep +# program additional paths for: cut, sed, wc, openssl, wget, xsltproc, grep ADDITIONAL_PATH= -# custom md5 path with arguments, expected output: <32digit md5><space(s)><filename> -# Leave empty for "md5sum" (Linux, Solaris), "md5 -r" (*BSD) -MD5= - # default server address. Danbooru only! I do not take responsibility of stupidity. DEFAULT_SITE="moe.imouto.org" @@ -47,12 +43,19 @@ # useless welcome message. Also version Msg_Welcome() { - MOEFETCHVERSION="0.3-beta1" + MOEFETCHVERSION="0.3-beta2" echo "moefetch ${MOEFETCHVERSION} Copyright (c) 2009 edogawaconan <me@myconan.net> " } +get_md5() { cat -- "$1" | openssl dgst -md5; } +get_basename() { basename -- "$1"; } +get_filename() { get_basename "${1%.*}"; } +get_cleantags() { printf "%s " "$@" | sed -e 's/\&/%26/g;s/=/%3D/g'; } +Is_NotMD5() { get_filename "$1" | sed -e 's/\([0-9a-f]\{32\}\)//g'; } + + # fatal error handler Err_Fatal() { echo " @@ -89,7 +92,7 @@ iternum=1 > "${TEMP_PREFIX}-list" while [ "${tempnum}" -ge 1000 ]; do - wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=1000&page=${_i}" -O "${TEMP_PREFIX}-xml" -e continue=off + wget "http://${SITE}/post/index.xml?tags=$(get_cleantags "${TAGS}")&offset=0&limit=1000&page=${iternum}&login=${LOGIN_USER}&password_hash=${LOGIN_PASS}" -O "${TEMP_PREFIX}-xml" -e continue=off printf "Processing XML file... " # xslt evilry xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-templist" @@ -110,9 +113,6 @@ [ "${numfiles}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site." } -Is_NotMD5() { - echo "${*}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g' -} Progress_Init() { _last="-" @@ -145,17 +145,7 @@ # check tools availability Check_Tools() { # verify all programs required do indeed exist - #MD5 - if [ -z "${MD5}" ]; then - case "$(uname)" in - *BSD) MD5="md5 -r";; - Linux|SunOS) MD5="md5sum";; - *) Fatal_Err "No known md5 tool for this platform. Please specify manually";; - esac - fi - md5_command=$(echo ${MD5} | cut -d' ' -f1) - # basic tools - commands="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${md5_command}" + commands="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date openssl" for cmd in ${commands} do [ "$(command -v "${cmd}")" ] || Err_Fatal "${cmd} doesn't exist in ${PATH}" @@ -199,12 +189,12 @@ for trash in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* do is_trash= - if [ -d "${trash}" ] || [ -n "$(Is_NotMD5 "$(basename "${trash}")")" ] || [ -z "$(grep "$(basename "${trash}")" "${TEMP_PREFIX}-list")" ]; then + if [ -d "${trash}" ] || [ -n "$(Is_NotMD5 "${trash}")" ] || [ -z "$(grep "$(get_basename "${trash}")" "${TEMP_PREFIX}-list")" ]; then is_trash=1 has_trash=1 - mv -f "${trash}" "${trash_dir}" || Err_Impossible + mv -f -- "${trash}" "${trash_dir}" || Err_Impossible trashes="${trashes} - $(basename "${trash}")" + $(get_basename "${trash}")" fi Progress_Anim done @@ -228,18 +218,18 @@ for file in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* do if [ "${file}" != "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/*" ]; then - if [ -n "$(Is_NotMD5 "$(basename "${file}")")" ] || [ -d "${file}" ]; then + if [ -n "$(Is_NotMD5 "${file}")" ] || [ -d "${file}" ]; then files_notdanbooru="${files_notdanbooru} - $(basename "${file}")" + $(get_basename "${file}")" has_err_filename=1 else - if [ "$(${MD5} "${file}" | cut -d ' ' -f 1)" = "$(basename "${file}" | cut -d '.' -f 1)" ]; then - echo "$(basename "${file}")" >> "${TEMP_PREFIX}-ok" + if [ "$(get_md5 "${file}")" = "$(get_filename "${file}")" ]; then + echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-ok" else rm "${file}" || Err_Fatal "Error removing ${file}" - echo "$(basename "${file}")" >> "${TEMP_PREFIX}-error" + echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-error" files_error="${files_error} - $(basename "${file}")" + $(get_basename "${file}")" has_err_md5=1 fi fi @@ -312,25 +302,19 @@ shift SITE= TAGS= - while [ "${1}" ]; do - case "$1" in - -s|--site) - shift - SITE="$1" - ;; - -nc|--noclean) - NOCLEAN=1 - ;; - *) - if [ "${TAGS}" ]; then - TAGS="$1 ${TAGS}" - else - TAGS="$1" - fi - ;; + x=1 + while getopts "s:(site)n(noclean)u:(user)p:(password)" opt + do + case "$opt" in + s) SITE="$OPTARG";; + n) NOCLEAN=1;; + p) LOGIN_PASS=$(printf "%s" "$OPTARG" | openssl dgst -sha1);; + u) LOGIN_USER="$OPTARG";; esac - shift + x=${OPTIND} done + shift $(($x-1)) + TAGS="$@" [ -n "${SITE}" ] || SITE=${DEFAULT_SITE} [ -n "${TAGS}" ] || Err_Fatal "No tag specified" # Get base folder - default, current folder or fallback to ${HOME}