Mercurial > ec-dotfiles

changeset 225:265a9ca47a19

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Replaced md5(sum) with openssl. Less platform dependent because the tool is same across platforms - Preparation for name-safe file operations. - Sanity against sed - Correctly encoded url (hopefully) - getopts replacing fail hackjob with while and shift - support for login (provided with openssl)
author edhoprima
date Fri, 13 Nov 2009 18:47:02 +0000
parents 0ac1805621d4
children f8be4a3d3b4a
files moefetch.sh
diffstat 1 files changed, 32 insertions(+), 48 deletions(-) [+]
line wrap: on
line diff
--- a/moefetch.sh	Sat Jul 18 09:10:42 2009 +0000
+++ b/moefetch.sh	Fri Nov 13 18:47:02 2009 +0000
@@ -18,15 +18,11 @@
 # USE WITH CARE
 #
 # what it does: fetch every picture that has the specified TAGS.
-# requirement: wget, libxslt, md5sum (or md5)
+# requirement: wget, libxslt, openssl
 
-# program additional paths for: cut, sed, wc, MD5(sum), wget, xsltproc, grep
+# program additional paths for: cut, sed, wc, openssl, wget, xsltproc, grep
 ADDITIONAL_PATH=
 
-# custom md5 path with arguments, expected output: <32digit md5><space(s)><filename>
-# Leave empty for "md5sum" (Linux, Solaris), "md5 -r" (*BSD)
-MD5=
-
 # default server address. Danbooru only! I do not take responsibility of stupidity.
 DEFAULT_SITE="moe.imouto.org"
 
@@ -47,12 +43,19 @@
 
 # useless welcome message. Also version
 Msg_Welcome() {
-	MOEFETCHVERSION="0.3-beta1"
+	MOEFETCHVERSION="0.3-beta2"
 	echo "moefetch ${MOEFETCHVERSION}
 Copyright (c) 2009 edogawaconan <me@myconan.net>
 "
 }
 
+get_md5() { cat -- "$1" | openssl dgst -md5; }
+get_basename() { basename -- "$1"; }
+get_filename() { get_basename "${1%.*}"; }
+get_cleantags() { printf "%s " "$@" | sed -e 's/\&/%26/g;s/=/%3D/g'; }
+Is_NotMD5() { get_filename "$1" | sed -e 's/\([0-9a-f]\{32\}\)//g'; }
+
+
 # fatal error handler
 Err_Fatal() {
 	echo "
@@ -89,7 +92,7 @@
 	iternum=1
 	> "${TEMP_PREFIX}-list"
 	while [ "${tempnum}" -ge 1000 ]; do
-		wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=1000&page=${_i}" -O "${TEMP_PREFIX}-xml" -e continue=off
+		wget "http://${SITE}/post/index.xml?tags=$(get_cleantags "${TAGS}")&offset=0&limit=1000&page=${iternum}&login=${LOGIN_USER}&password_hash=${LOGIN_PASS}" -O "${TEMP_PREFIX}-xml" -e continue=off
 		printf "Processing XML file... "
 		# xslt evilry
 		xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-templist"
@@ -110,9 +113,6 @@
 	[ "${numfiles}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site."
 }
 
-Is_NotMD5() {
-	echo "${*}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g'
-}
 
 Progress_Init() {
 	_last="-"
@@ -145,17 +145,7 @@
 # check tools availability
 Check_Tools() {
 	# verify all programs required do indeed exist
-	#MD5
-	if [ -z "${MD5}" ]; then
-		case "$(uname)" in
-			*BSD) MD5="md5 -r";;
-			Linux|SunOS) MD5="md5sum";;
-			*) Fatal_Err "No known md5 tool for this platform. Please specify manually";;
-		esac
-	fi
-	md5_command=$(echo ${MD5} | cut -d' ' -f1)
-	# basic tools
-	commands="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${md5_command}"
+	commands="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date openssl"
 	for cmd in ${commands}
 	do
 		 [ "$(command -v "${cmd}")" ] || Err_Fatal "${cmd} doesn't exist in ${PATH}"
@@ -199,12 +189,12 @@
 	for trash in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"*
 	do
 		is_trash=
-		if [ -d "${trash}" ] || [ -n "$(Is_NotMD5 "$(basename "${trash}")")" ] || [ -z "$(grep "$(basename "${trash}")" "${TEMP_PREFIX}-list")" ]; then
+		if [ -d "${trash}" ] || [ -n "$(Is_NotMD5 "${trash}")" ] || [ -z "$(grep "$(get_basename "${trash}")" "${TEMP_PREFIX}-list")" ]; then
 			is_trash=1
 			has_trash=1
-			mv -f "${trash}" "${trash_dir}" || Err_Impossible
+			mv -f -- "${trash}" "${trash_dir}" || Err_Impossible
 			trashes="${trashes}
-  $(basename "${trash}")"
+  $(get_basename "${trash}")"
 		fi
 		Progress_Anim
 	done
@@ -228,18 +218,18 @@
 		for file in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"*
 		do
 			if [ "${file}" != "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/*" ]; then
-				if [ -n "$(Is_NotMD5 "$(basename "${file}")")" ] || [ -d "${file}" ]; then
+				if [ -n "$(Is_NotMD5 "${file}")" ] || [ -d "${file}" ]; then
 					files_notdanbooru="${files_notdanbooru}
-  $(basename "${file}")"
+  $(get_basename "${file}")"
 					has_err_filename=1
 				else
-					if [ "$(${MD5} "${file}" | cut -d ' ' -f 1)" = "$(basename "${file}" | cut -d '.' -f 1)" ]; then
-						echo "$(basename "${file}")" >> "${TEMP_PREFIX}-ok"
+					if [ "$(get_md5 "${file}")" = "$(get_filename "${file}")" ]; then
+						echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-ok"
 					else
 						rm "${file}" || Err_Fatal "Error removing ${file}"
-						echo "$(basename "${file}")" >> "${TEMP_PREFIX}-error"
+						echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-error"
 						files_error="${files_error}
-  $(basename "${file}")"
+  $(get_basename "${file}")"
 						has_err_md5=1
 					fi
 				fi
@@ -312,25 +302,19 @@
 	shift
 	SITE=
 	TAGS=
-	while [ "${1}" ]; do
-		case "$1" in
-			-s|--site)
-				shift
-				SITE="$1"
-			;;
-			-nc|--noclean)
-				NOCLEAN=1
-			;;
-			*)
-				if [ "${TAGS}" ]; then
-					TAGS="$1 ${TAGS}"
-				else
-					TAGS="$1"
-				fi
-			;;
+	x=1
+	while getopts "s:(site)n(noclean)u:(user)p:(password)" opt
+	do
+		case "$opt" in
+			s) SITE="$OPTARG";;
+			n) NOCLEAN=1;;
+			p) LOGIN_PASS=$(printf "%s" "$OPTARG" | openssl dgst -sha1);;
+			u) LOGIN_USER="$OPTARG";;
 		esac
-		shift
+		x=${OPTIND}
 	done
+	shift $(($x-1))
+	TAGS="$@"
 	[ -n "${SITE}" ] || SITE=${DEFAULT_SITE}
 	[ -n "${TAGS}" ] || Err_Fatal "No tag specified"
 	# Get base folder - default, current folder or fallback to ${HOME}