# HG changeset patch
# User edhoprima@gmail.com <edhoprima@gmail.com>
# Date 1246272580 0
# Node ID ac6533a8fb51c61be3c079977ef7c462c11c7ef9
# Parent  47efb7a2342584a7571c1cac13887c6306153c08
- Documentation
- Cleanup

diff -r 47efb7a23425 -r ac6533a8fb51 Script_Explanation
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Script_Explanation	Mon Jun 29 10:49:40 2009 +0000
@@ -0,0 +1,97 @@
+Variables:
+
+Outer:
+- ADDITIONAL_PATH
+- MD5
+- DEFAULT_SITE
+- BASE_DIR
+- MOEFETCHVERSION
+
+Functions:
+
+- Msg_Welcome
+	- Welcome message (MOEFETCHVERSION)
+- Err_Help
+- Err_Fatal
+- Generate_Link
+	- chdir to ${BASE_DIR}/temp
+	- fetch xml with wget
+	- xsltproc the xml
+- Check_Tools
+	- Check if MD5 is empty - if empty: check os
+		- *BSD: md -r
+		- Linux/SunOS: md5sum
+		- Anything else: Err_Fatal
+	- Get md5 command (MD5_COMMAND)
+	- Check availability of needed tools
+		- cut 
+		- sed 
+		- wc 
+		- wget 
+		- xsltproc 
+		- xargs 
+		- rm 
+		- mkdir 
+		- chown 
+		- comm 
+		- grep 
+		- date 
+		- MD5_COMMAND
+	- Check for grep usability 
+	- TODO: greplace grep -f with POSIX compatible
+- Check_Folders
+	- Check BASE_DIR ownership
+	- Check: existance (create if not exist) and ownership (apply globally writable permission) of BASE_DIR/:
+		- temp
+		- trash
+		- deleted
+		- SITE_DIR/TARGET_DIR
+	- Check if "BASE_DIR/SITE_DIR/TARGET_DIR" is empty: if empty, ISNEW=1
+	- Create temporary files: BASE_DIR/temp/${SITE_DIR}-${TARGET_DIR}-:
+		- error
+		- ok
+		- list
+		- newlist
+- Cleanup_Repository
+	- TRASH_DIR: ${BASE_DIR}/trash/${SITE_DIR}-${TARGET_DIR}-%Y%m%d-%H.%M
+	- create trash folder
+	- check if file in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/ fulfills [a-f0-9]{32}\..* or is a folder
+	- move to trash if is trash
+	- check if file is contained in ${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list
+	- if is not, move to trash
+- Check_Files
+	- if not ISNEW (empty target folder):
+		- if not NOCLEAN (not skipping cleanup:
+			- Call Clean_Repository
+		- chdir to target folder
+		- TODO: chdir-free operation
+		- empty ${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-err
+		- check file in current directory (${BASE_DIR}/${SITE_DIR}/${TARGET_DIR})
+			- skip if not correct file ([a-f0-9]{32}\..*)
+			- put into ${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error for every error files
+		- remove the files
+		- chdir to temp folder
+		- list ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}, compare with error, exclude the errors, put into ${SITE_DIR}-${TARGET_DIR}-ok
+		- get list of new files - compare with ${SITE_DIR}-${TARGET_DIR}-list
+		- TODO: remove ls, grep -f dependencies
+	- if ISQUICK: skip check
+	- if not ISQUICK: print 'empty repository'
+	- copy ${SITE_DIR}-${TARGET_DIR}-list to ${SITE_DIR}-${TARGET_DIR}-newlist
+- Fetch_Images
+	- chdir to ${BASE_DIR}/temp
+	- check if ${SITE_DIR}-${TARGET_DIR}-newlist is empty -> stop
+	- chdir to ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}
+	- start wget: wget -e continue=on -bi "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-newlist" -o "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}.log"
+- Init
+	- Add path (PATH)
+	- Check command (fetch/check/quickfetch/* - JOB)
+	- Check site (-s <site> or default - SITE)
+	- Check if not clean folder (-nc/no clean - NOCLEAN)
+	- Get tags (TAGS)
+	- Check site - if SITE empty then set default (SITE=DEFAULT_SITE)
+	- TODO: Validate SITE
+	- Check tag - if TAGS empty then Err_Fatal
+	- Get BASE_DIR: default to PWD - fallback to HOME
+	- Validate BASE_DIR: must absolute path
+	- Get TARGET_DIR: escape TAGS (replace / with _)
+	- Get SITE_DIR: escape SITE (remove ending /, replace / with _)
diff -r 47efb7a23425 -r ac6533a8fb51 moefetch.sh
--- a/moefetch.sh	Sun Jun 28 17:20:46 2009 +0000
+++ b/moefetch.sh	Mon Jun 29 10:49:40 2009 +0000
@@ -34,13 +34,23 @@
 # Structure is ${BASE_DIR}/<TAGS>
 # Absolute path only.
 # Leave empty to use whatever folder you're running this at
-BASE_DIR=""
+BASE_DIR=
 
 # not user modifiable from here
 
+
+### TODO:
+###  - replace `...` with $(..)
+###  - sanity validator
+###  - unified repository to save bandwidth
+###  - bug stomping
+###  - sanity checking
+###  - replace grep -vf with POSIX compatible command
+### WILL BE FOR 0.3
+
 # useless welcome message. Also version
 Msg_Welcome() {
-	MOEFETCHVERSION="0.1-beta2"
+	MOEFETCHVERSION="0.2-beta1"
 	cat <<EOF
 moefetch ${MOEFETCHVERSION}
 Copyright (c) 2009 edogawaconan <me@myconan.net>
@@ -68,7 +78,7 @@
 TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme
 	
 EOF
-	exit 0
+	exit 2
 }
 
 # generate link by transforming xml
@@ -87,7 +97,7 @@
 </xsl:template>
 </xsl:stylesheet>
 EOF
-	NUMFILES=`echo \`wc -l < "${SITE_DIR}-${TARGET_DIR}-list" \``
+	NUMFILES=$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-list"))
 	[ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site"
 	echo "${NUMFILES} file(s) available on server"
 	#output file: ${TARGET_DIR}-list
@@ -98,25 +108,24 @@
 	# verify all programs required do indeed exist
 	#MD5
 	if [ ! "${MD5}" ]; then
-		case `uname` in
+		case $(uname) in
 			*BSD) MD5="md5 -r";;
 			Linux|SunOS) MD5="md5sum";;
 			*) Fatal_Err "No known md5 tool for this platform. Please specify manually"
 		esac
 	fi
-	MD5_COMMAND=`echo ${MD5} | cut -d' ' -f1`
+	MD5_COMMAND=$(echo ${MD5} | cut -d' ' -f1)
 	# basic tools
 	COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}"
 	for COMMAND in ${COMMANDS}
 	do
-		COMMAND_CHECK=`command -v "${COMMAND}"`
-		 [ "${COMMAND_CHECK}" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}"
+		 [ "$(command -v "${COMMAND}")" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}"
 	done
 
 	# grep checking
 	# originally created for workaround on solaris
 	#if [ `uname` = "SunOS" ]; then
-	FAIL=""
+	FAIL=
 	echo "blah" > superrandomtestfile
 	echo "blah" > superrandomtestfile.2
 	grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1
@@ -136,7 +145,7 @@
 			chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen"
 		fi
 	done
-	[ `echo \`ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l\`` -eq 0 ] && ISNEW=1
+	[ "$(echo $(ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l))" -eq 0 ] && ISNEW=1
 	# let's move to workdir
 	cd "${BASE_DIR}/temp"
 	for i in error ok list newlist; do
@@ -160,7 +169,7 @@
 		if [ -d "${TRASH}" ]; then
 			ISTRASH=1
 		else
-			if [ "$(echo "${FILE}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}.*\)//g' | grep -v ^$)" ]; then
+			if [ "$(echo "${TRASH}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then
 				ISTRASH=1
 			else
 				[ "$(cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list" | sed -e 's/.*\/\([^\/]*\)/\1/g' | grep $(echo "${TRASH}" | sed -e 's/.*\/\([^\/]*\)/\1/g'))" ] || ISTRASH=1
@@ -184,10 +193,11 @@
 		printf "" > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"
 		for FILE in *
 		do
-			if [ "`echo \"${FILE}\" | sed -e 's/\([0-9a-f]\{32\}.*\)//g' | grep -v ^$`" ]; then
+			if [ "$(echo "${FILE}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then
+				echo
 				echo "Not a valid danbooru file: ${FILE}"
 			else
-				if [ "`${MD5} "${FILE}" | cut -d ' ' -f1 -`" != "`echo "${FILE}" | cut -d '.' -f1`" ]
+				if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | cut -d '.' -f1)" ]
 				then
 					echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"
 					echo
@@ -197,7 +207,7 @@
 			fi
 		done
 		echo
-		TOTAL_ERROR=`echo \`wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"\``
+		TOTAL_ERROR=$(echo $(wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"))
 		echo "${TOTAL_ERROR} file(s) error"
 		echo "Removing error files"
 		if [ "${TOTAL_ERROR}" -eq 0 ]; then
@@ -206,10 +216,10 @@
 			cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm
 			echo "${TOTAL_ERROR} file(s) removed"
 		fi
-		echo "`echo \`ls | wc -l\`` file(s) available locally"
+		echo "$(echo $(ls | wc -l)) file(s) available locally"
 
 		# current dir: ${BASE_DIR}/temp
-		cd ${BASE_DIR}/temp
+		cd "${BASE_DIR}/temp"
 
 		echo "Generating list of new files..."
 		# THE FILES
@@ -217,7 +227,7 @@
 		#
 		ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok"
 		cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist"
-		echo "`echo \`wc -l < \"${SITE_DIR}-${TARGET_DIR}-newlist\"\`` file(s) to be downloaded"
+		echo "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist")) file(s) to be downloaded"
 		
 	else
 		if [ "${ISQUICK}" ]; then
@@ -233,7 +243,7 @@
 # start downloading the images
 Fetch_Images() {
 	cd "${BASE_DIR}/temp"
-	if [ `echo \`wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"\`` -eq 0 ]; then
+	if [ "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"))" -eq 0 ]; then
 		echo "No new file"
 	else
 		echo "Starting wget"
@@ -289,12 +299,12 @@
 	# Get base folder - default, current folder or fallback to ${HOME}
 	[ "${BASE_DIR}"	] || BASE_DIR="${PWD}"
 	[ "${BASE_DIR}" ] || BASE_DIR="{$HOME}"
-	[ "`echo \"${BASE_DIR}\" | cut -c1 | grep \/`" ] || BASE_DIR="/${BASE_DIR}"	
+	[ "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" ] || BASE_DIR="/${BASE_DIR}"	
 
 	echo "Tags: ${TAGS}"
 	# slash is not wanted for folder name
-	TARGET_DIR="`echo "${TAGS}" | sed -e 's/\//_/g'`"
-	SITE_DIR="`echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g'`"
+	TARGET_DIR=$(echo "${TAGS}" | sed -e 's/\//_/g')
+	SITE_DIR=$(echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g')
 }
 
 # initialization