comparison moefetch.sh @ 193:ac6533a8fb51

- Documentation - Cleanup
author edhoprima@gmail.com <edhoprima@gmail.com>
date Mon, 29 Jun 2009 10:49:40 +0000
parents 47efb7a23425
children a8da49424468
comparison
equal deleted inserted replaced
192:47efb7a23425 193:ac6533a8fb51
32 32
33 # base directory. make sure it's writeable. I do not take responsibility if you don't own the folder and files as no check is done for this one. 33 # base directory. make sure it's writeable. I do not take responsibility if you don't own the folder and files as no check is done for this one.
34 # Structure is ${BASE_DIR}/<TAGS> 34 # Structure is ${BASE_DIR}/<TAGS>
35 # Absolute path only. 35 # Absolute path only.
36 # Leave empty to use whatever folder you're running this at 36 # Leave empty to use whatever folder you're running this at
37 BASE_DIR="" 37 BASE_DIR=
38 38
39 # not user modifiable from here 39 # not user modifiable from here
40
41
42 ### TODO:
43 ### - replace `...` with $(..)
44 ### - sanity validator
45 ### - unified repository to save bandwidth
46 ### - bug stomping
47 ### - sanity checking
48 ### - replace grep -vf with POSIX compatible command
49 ### WILL BE FOR 0.3
40 50
41 # useless welcome message. Also version 51 # useless welcome message. Also version
42 Msg_Welcome() { 52 Msg_Welcome() {
43 MOEFETCHVERSION="0.1-beta2" 53 MOEFETCHVERSION="0.2-beta1"
44 cat <<EOF 54 cat <<EOF
45 moefetch ${MOEFETCHVERSION} 55 moefetch ${MOEFETCHVERSION}
46 Copyright (c) 2009 edogawaconan <me@myconan.net> 56 Copyright (c) 2009 edogawaconan <me@myconan.net>
47 57
48 EOF 58 EOF
66 -s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE} 76 -s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE}
67 77
68 TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme 78 TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme
69 79
70 EOF 80 EOF
71 exit 0 81 exit 2
72 } 82 }
73 83
74 # generate link by transforming xml 84 # generate link by transforming xml
75 Generate_Link() { 85 Generate_Link() {
76 cd "${BASE_DIR}/temp" 86 cd "${BASE_DIR}/temp"
85 <xsl:template match="post"> 95 <xsl:template match="post">
86 <xsl:value-of select="@file_url" /> 96 <xsl:value-of select="@file_url" />
87 </xsl:template> 97 </xsl:template>
88 </xsl:stylesheet> 98 </xsl:stylesheet>
89 EOF 99 EOF
90 NUMFILES=`echo \`wc -l < "${SITE_DIR}-${TARGET_DIR}-list" \`` 100 NUMFILES=$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-list"))
91 [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" 101 [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site"
92 echo "${NUMFILES} file(s) available on server" 102 echo "${NUMFILES} file(s) available on server"
93 #output file: ${TARGET_DIR}-list 103 #output file: ${TARGET_DIR}-list
94 } 104 }
95 105
96 # check tools availability 106 # check tools availability
97 Check_Tools() { 107 Check_Tools() {
98 # verify all programs required do indeed exist 108 # verify all programs required do indeed exist
99 #MD5 109 #MD5
100 if [ ! "${MD5}" ]; then 110 if [ ! "${MD5}" ]; then
101 case `uname` in 111 case $(uname) in
102 *BSD) MD5="md5 -r";; 112 *BSD) MD5="md5 -r";;
103 Linux|SunOS) MD5="md5sum";; 113 Linux|SunOS) MD5="md5sum";;
104 *) Fatal_Err "No known md5 tool for this platform. Please specify manually" 114 *) Fatal_Err "No known md5 tool for this platform. Please specify manually"
105 esac 115 esac
106 fi 116 fi
107 MD5_COMMAND=`echo ${MD5} | cut -d' ' -f1` 117 MD5_COMMAND=$(echo ${MD5} | cut -d' ' -f1)
108 # basic tools 118 # basic tools
109 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}" 119 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}"
110 for COMMAND in ${COMMANDS} 120 for COMMAND in ${COMMANDS}
111 do 121 do
112 COMMAND_CHECK=`command -v "${COMMAND}"` 122 [ "$(command -v "${COMMAND}")" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}"
113 [ "${COMMAND_CHECK}" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}"
114 done 123 done
115 124
116 # grep checking 125 # grep checking
117 # originally created for workaround on solaris 126 # originally created for workaround on solaris
118 #if [ `uname` = "SunOS" ]; then 127 #if [ `uname` = "SunOS" ]; then
119 FAIL="" 128 FAIL=
120 echo "blah" > superrandomtestfile 129 echo "blah" > superrandomtestfile
121 echo "blah" > superrandomtestfile.2 130 echo "blah" > superrandomtestfile.2
122 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1 131 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1
123 rm -f superrandomtestfile superrandomtestfile.2 132 rm -f superrandomtestfile superrandomtestfile.2
124 [ "${FAIL}" ] && Err_Fatal "Your grep is not compatible. Please install or set path of correct grep" 133 [ "${FAIL}" ] && Err_Fatal "Your grep is not compatible. Please install or set path of correct grep"
134 if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then 143 if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then
135 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it" 144 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it"
136 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen" 145 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen"
137 fi 146 fi
138 done 147 done
139 [ `echo \`ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l\`` -eq 0 ] && ISNEW=1 148 [ "$(echo $(ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l))" -eq 0 ] && ISNEW=1
140 # let's move to workdir 149 # let's move to workdir
141 cd "${BASE_DIR}/temp" 150 cd "${BASE_DIR}/temp"
142 for i in error ok list newlist; do 151 for i in error ok list newlist; do
143 touch "${SITE_DIR}-${TARGET_DIR}-${i}" || Fatal_Err "Error creating ${TARGET_DIR}-${i}. This shouldn't happen" 152 touch "${SITE_DIR}-${TARGET_DIR}-${i}" || Fatal_Err "Error creating ${TARGET_DIR}-${i}. This shouldn't happen"
144 done 153 done
158 do 167 do
159 ISTRASH= 168 ISTRASH=
160 if [ -d "${TRASH}" ]; then 169 if [ -d "${TRASH}" ]; then
161 ISTRASH=1 170 ISTRASH=1
162 else 171 else
163 if [ "$(echo "${FILE}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}.*\)//g' | grep -v ^$)" ]; then 172 if [ "$(echo "${TRASH}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then
164 ISTRASH=1 173 ISTRASH=1
165 else 174 else
166 [ "$(cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list" | sed -e 's/.*\/\([^\/]*\)/\1/g' | grep $(echo "${TRASH}" | sed -e 's/.*\/\([^\/]*\)/\1/g'))" ] || ISTRASH=1 175 [ "$(cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list" | sed -e 's/.*\/\([^\/]*\)/\1/g' | grep $(echo "${TRASH}" | sed -e 's/.*\/\([^\/]*\)/\1/g'))" ] || ISTRASH=1
167 fi 176 fi
168 fi 177 fi
182 echo "Checking for errors..." 191 echo "Checking for errors..."
183 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" 192 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}"
184 printf "" > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" 193 printf "" > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"
185 for FILE in * 194 for FILE in *
186 do 195 do
187 if [ "`echo \"${FILE}\" | sed -e 's/\([0-9a-f]\{32\}.*\)//g' | grep -v ^$`" ]; then 196 if [ "$(echo "${FILE}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then
197 echo
188 echo "Not a valid danbooru file: ${FILE}" 198 echo "Not a valid danbooru file: ${FILE}"
189 else 199 else
190 if [ "`${MD5} "${FILE}" | cut -d ' ' -f1 -`" != "`echo "${FILE}" | cut -d '.' -f1`" ] 200 if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | cut -d '.' -f1)" ]
191 then 201 then
192 echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" 202 echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"
193 echo 203 echo
194 echo "Error: ${FILE}" 204 echo "Error: ${FILE}"
195 fi 205 fi
196 printf "." 206 printf "."
197 fi 207 fi
198 done 208 done
199 echo 209 echo
200 TOTAL_ERROR=`echo \`wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"\`` 210 TOTAL_ERROR=$(echo $(wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"))
201 echo "${TOTAL_ERROR} file(s) error" 211 echo "${TOTAL_ERROR} file(s) error"
202 echo "Removing error files" 212 echo "Removing error files"
203 if [ "${TOTAL_ERROR}" -eq 0 ]; then 213 if [ "${TOTAL_ERROR}" -eq 0 ]; then
204 echo "No error file. 0 file removed" 214 echo "No error file. 0 file removed"
205 else 215 else
206 cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm 216 cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm
207 echo "${TOTAL_ERROR} file(s) removed" 217 echo "${TOTAL_ERROR} file(s) removed"
208 fi 218 fi
209 echo "`echo \`ls | wc -l\`` file(s) available locally" 219 echo "$(echo $(ls | wc -l)) file(s) available locally"
210 220
211 # current dir: ${BASE_DIR}/temp 221 # current dir: ${BASE_DIR}/temp
212 cd ${BASE_DIR}/temp 222 cd "${BASE_DIR}/temp"
213 223
214 echo "Generating list of new files..." 224 echo "Generating list of new files..."
215 # THE FILES 225 # THE FILES
216 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" 226 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok"
217 # 227 #
218 ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok" 228 ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok"
219 cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist" 229 cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist"
220 echo "`echo \`wc -l < \"${SITE_DIR}-${TARGET_DIR}-newlist\"\`` file(s) to be downloaded" 230 echo "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist")) file(s) to be downloaded"
221 231
222 else 232 else
223 if [ "${ISQUICK}" ]; then 233 if [ "${ISQUICK}" ]; then
224 echo "quick mode selected. Skipping check" 234 echo "quick mode selected. Skipping check"
225 else 235 else
231 } 241 }
232 242
233 # start downloading the images 243 # start downloading the images
234 Fetch_Images() { 244 Fetch_Images() {
235 cd "${BASE_DIR}/temp" 245 cd "${BASE_DIR}/temp"
236 if [ `echo \`wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"\`` -eq 0 ]; then 246 if [ "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"))" -eq 0 ]; then
237 echo "No new file" 247 echo "No new file"
238 else 248 else
239 echo "Starting wget" 249 echo "Starting wget"
240 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" 250 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}"
241 wget -e continue=on -bi "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-newlist" -o "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}.log" 251 wget -e continue=on -bi "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-newlist" -o "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}.log"
287 [ "${SITE}" ] || SITE="${DEFAULT_SITE}" 297 [ "${SITE}" ] || SITE="${DEFAULT_SITE}"
288 [ "${TAGS}" ] || Err_Fatal "No tag specified" 298 [ "${TAGS}" ] || Err_Fatal "No tag specified"
289 # Get base folder - default, current folder or fallback to ${HOME} 299 # Get base folder - default, current folder or fallback to ${HOME}
290 [ "${BASE_DIR}" ] || BASE_DIR="${PWD}" 300 [ "${BASE_DIR}" ] || BASE_DIR="${PWD}"
291 [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}" 301 [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}"
292 [ "`echo \"${BASE_DIR}\" | cut -c1 | grep \/`" ] || BASE_DIR="/${BASE_DIR}" 302 [ "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" ] || BASE_DIR="/${BASE_DIR}"
293 303
294 echo "Tags: ${TAGS}" 304 echo "Tags: ${TAGS}"
295 # slash is not wanted for folder name 305 # slash is not wanted for folder name
296 TARGET_DIR="`echo "${TAGS}" | sed -e 's/\//_/g'`" 306 TARGET_DIR=$(echo "${TAGS}" | sed -e 's/\//_/g')
297 SITE_DIR="`echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g'`" 307 SITE_DIR=$(echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g')
298 } 308 }
299 309
300 # initialization 310 # initialization
301 Msg_Welcome 311 Msg_Welcome
302 Init "$@" 312 Init "$@"