Mercurial > ec-dotfiles

changeset 214:a6624fb9b317

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
major cleanup. tweaking. untested
author edhoprima@gmail.com <edhoprima@gmail.com>
date Thu, 02 Jul 2009 19:10:36 +0000
parents dd95cf01602c
children 710082ce6788
files moefetch.sh
diffstat 1 files changed, 55 insertions(+), 48 deletions(-) [+]
line wrap: on
line diff
--- a/moefetch.sh	Tue Jun 30 09:23:02 2009 +0000
+++ b/moefetch.sh	Thu Jul 02 19:10:36 2009 +0000
@@ -51,23 +51,20 @@
 # useless welcome message. Also version
 Msg_Welcome() {
 	MOEFETCHVERSION="0.2.1"
-	cat <<EOF
-moefetch ${MOEFETCHVERSION}
-Copyright (c) 2009 edogawaconan <me@myconan.net>
-
-EOF
+	echo "moefetch ${MOEFETCHVERSION}
+Copyright (c) 2009 edogawaconan <me@myconan.net>"
 }
 
 # fatal error handler
 Err_Fatal() {
-	printf "\nFatal error: ${1}\n"
+	echo "
+Fatal error: ${1}"
 	exit 1
 }
 
 # help message
 Err_Help() {
-	cat <<EOF
-moefetch.sh COMMAND [-s SITE_URL] TAGS
+	echo "moefetch.sh COMMAND [-s SITE_URL] TAGS
 
 COMMAND: 
 (quick)fetch: do a complete update. Add prefix quick to skip file checking
@@ -75,21 +72,20 @@
 
 -s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE}
 
-TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme
-	
-EOF
+TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme"
 	exit 2
 }
 
 # generate link by transforming xml
 Generate_Link() {
-	printf "\nFetching xml file\n"
-	TEMPNUM=1001
+	echo "
+Fetching xml file"
+	TEMPNUM=1000
 	_i=1
 	> "${TEMP_PREFIX}-list"
 	while [ "${TEMPNUM}" -ge 1000 ]; do
 		wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=1000&page=${_i}" -O "${TEMP_PREFIX}-xml" -e continue=off
-		printf "Processing XML file..."
+		printf "Processing XML file... "
 		# xslt evilry
 		> "${TEMP_PREFIX}-templist"
 		xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-templist" 2>/dev/null
@@ -103,11 +99,15 @@
 		TEMPNUM=$(echo $(wc -l < "${TEMP_PREFIX}-templist"))
 		_i=$((_i+1))
 		cat "${TEMP_PREFIX}-templist" >> "${TEMP_PREFIX}-list"
+		echo "${TEMPNUM} file(s) available"
 	done
 	NUMFILES=$(echo $(wc -l < "${TEMP_PREFIX}-list"))
+	echo "${NUMFILES} file(s) available on server"
 	[ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site"
-	echo " ${NUMFILES} file(s) available on server"
-	#output file: ${TARGET_DIR}-list
+}
+
+Is_NotMD5() {
+	echo "${*}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g'
 }
 
 Progress_Init() {
@@ -179,13 +179,11 @@
 
 # Do some cleanup
 Cleanup_Repository() {
-
-	# THE FILES
-	
 	# current dir: ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}
 	printf "Cleaning up repository folder... "
 	Progress_Init
 	TRASH_DIR=$(date -u "+${SITE_DIR}-${TARGET_DIR}-%Y%m%d-%H.%M")
+	TRASHES=
 	mkdir -p "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Unable to create trash folder"
 	for TRASH in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"*
 	do
@@ -193,71 +191,80 @@
 		if [ -d "${TRASH}" ]; then
 			ISTRASH=1
 		else
-			if [ "$(echo "${TRASH}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then
+			if [ "$(Is_NotMD5 "$(basename "${TRASH}")")" ]; then
 				ISTRASH=1
 			else
-				[ "$(cat "${TEMP_PREFIX}-list" | sed -e "${SED_GET_FILENAME}" | grep $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}"))" ] || ISTRASH=1
+				[ "$(grep "$(basename "${TRASH}")" "${TEMP_PREFIX}-list")" ] || ISTRASH=1
 			fi
 		fi
 		if [ "${ISTRASH}" ]; then
 			mv -f "${TRASH}" "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Error deleting files"
-			printf "\bMoved $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}") to ${BASE_DIR}/trash/${TRASH_DIR}\n${_last}"
+			TRASHES="${TRASHES}
+$(basename "${TRASH}")"
 		fi
 		Progress_Anim
 	done
 	rmdir "${BASE_DIR}/trash/${TRASH_DIR}" 2>/dev/null
 	Progress_Done
+	echo "These files have been moved to ${BASE_DIR}/trash/${TRASH_DIR}:
+${TRASHES}"
 }
 
 # check files correctness
 Check_Files() {
-	if [ ! "${ISNEW}" ]; then
-		[ "${NOCLEAN}" ] || Cleanup_Repository
+	if test ! -n "${ISNEW}"; then
+		test -z "${NOCLEAN}" && Cleanup_Repository
 		printf "Checking for errors... "
 		Progress_Init
+		files_error="These files do not match its md5:"
+		files_notdanbooru="These files are not checked:"
+		has_err_filename=
+		has_err_md5=
 		> "${TEMP_PREFIX}-error"
-		for FILE in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"*
+		> "${TEMP_PREFIX}-ok"
+		for file in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"*
 		do
-			if [ "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then
-				printf "\bNot a valid danbooru file: $(echo ${FILE} | sed -e "${SED_GET_FILENAME}")\n${_last}"
+			if test -n "$(Is_NotMD5 "$(basename "${file}")")" || test -d "${file}"; then
+				files_notdanbooru="${files_notdanbooru}
+$(basename "${file}")"
+				has_err_filename=1
 			else
-				if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME}" | cut -d '.' -f1)" ]
-				then
-					echo "${FILE}" >> "${TEMP_PREFIX}-error"
-					printf "\bError: $(echo "${FILE}" | sed -e "${SED_GET_FILENAME}")\n${_last}"
+				if test "$(${MD5} "${file}" | cut -d ' ' -f1 -)" = "$(basename "${file}" | cut -d '.' -f1)"; then
+					echo "$(basename "${file}")" >> "${TEMP_PREFIX}-ok"
+				else
+					rm "${file}" || Err_Fatal "Error removing ${file}"
+					echo "$(basename "${file}")" >> "${TEMP_PREFIX}-error"
+					files_error="${files_error}
+$(basename "${file}")"
+					has_err_md5=1
 				fi
-				Progress_Anim
 			fi
+			Progress_Anim
 		done
 		Progress_Done
-		TOTAL_ERROR=$(echo $(wc -l < "${TEMP_PREFIX}-error"))
-		if [ "${TOTAL_ERROR}" -eq 0 ]; then
+		if test ! -n "${has_err_md5}" && test ! -n "${has_err_filename}"; then
 			echo "All files OK"
 		else
-			printf "${TOTAL_ERROR} file(s) broken: removing..."
-			cat "${TEMP_PREFIX}-error" | xargs rm
-			echo " ${TOTAL_ERROR} file(s) removed"
+			if test ! -n "${has_err_md5}"; then
+				echo "${files_error}"
+				echo "$(echo $(wc -l < "${TEMP_PREFIX}-error")) file(s) removed"
+			fi
+			test -n "${has_err_filename}" && echo "${files_notdanbooru}" 
 		fi
-		echo "$(Count_Files "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}") file(s) available locally"
+		echo "$(echo $(wc -l < "${TEMP_PREFIX}-ok")) file(s) available locally"
 
 		printf "Generating list of new files... "
 		Progress_Init
-		# THE FILES
-		#ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok"
-		#
-		find "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${TEMP_PREFIX}-error" - | sed -e "${SED_GET_FILENAME}" > "${TEMP_PREFIX}-ok"
 		cat "${TEMP_PREFIX}-list" > "${TEMP_PREFIX}-templist"
-		while read -r IS_OK; do
-			cat "${TEMP_PREFIX}-templist" | grep -v "${IS_OK}" > "${TEMP_PREFIX}-newlist"
+		while read -r is_ok; do
+			grep -v "${is_ok}" "${TEMP_PREFIX}-templist" > "${TEMP_PREFIX}-newlist"
 			cat "${TEMP_PREFIX}-newlist" > "${TEMP_PREFIX}-templist"
 			Progress_Anim
 		done < "${TEMP_PREFIX}-ok"
 		Progress_Done
-		#cat "${TEMP_PREFIX}-list" | egrep -vf "${TEMP_PREFIX}-ok" > "${TEMP_PREFIX}-newlist"
-		echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded"
-		
+		echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded"		
 	else
-		if [ "${ISQUICK}" ]; then
+		if test -n "${ISQUICK}"; then
 			echo "Quick mode selected. Skipping check"
 		else
 			echo "Empty local repository"