Mercurial > ec-dotfiles
comparison moefetch.sh @ 193:ac6533a8fb51
- Documentation
- Cleanup
author | edhoprima@gmail.com <edhoprima@gmail.com> |
---|---|
date | Mon, 29 Jun 2009 10:49:40 +0000 |
parents | 47efb7a23425 |
children | a8da49424468 |
comparison
equal
deleted
inserted
replaced
192:47efb7a23425 | 193:ac6533a8fb51 |
---|---|
32 | 32 |
33 # base directory. make sure it's writeable. I do not take responsibility if you don't own the folder and files as no check is done for this one. | 33 # base directory. make sure it's writeable. I do not take responsibility if you don't own the folder and files as no check is done for this one. |
34 # Structure is ${BASE_DIR}/<TAGS> | 34 # Structure is ${BASE_DIR}/<TAGS> |
35 # Absolute path only. | 35 # Absolute path only. |
36 # Leave empty to use whatever folder you're running this at | 36 # Leave empty to use whatever folder you're running this at |
37 BASE_DIR="" | 37 BASE_DIR= |
38 | 38 |
39 # not user modifiable from here | 39 # not user modifiable from here |
40 | |
41 | |
42 ### TODO: | |
43 ### - replace `...` with $(..) | |
44 ### - sanity validator | |
45 ### - unified repository to save bandwidth | |
46 ### - bug stomping | |
47 ### - sanity checking | |
48 ### - replace grep -vf with POSIX compatible command | |
49 ### WILL BE FOR 0.3 | |
40 | 50 |
41 # useless welcome message. Also version | 51 # useless welcome message. Also version |
42 Msg_Welcome() { | 52 Msg_Welcome() { |
43 MOEFETCHVERSION="0.1-beta2" | 53 MOEFETCHVERSION="0.2-beta1" |
44 cat <<EOF | 54 cat <<EOF |
45 moefetch ${MOEFETCHVERSION} | 55 moefetch ${MOEFETCHVERSION} |
46 Copyright (c) 2009 edogawaconan <me@myconan.net> | 56 Copyright (c) 2009 edogawaconan <me@myconan.net> |
47 | 57 |
48 EOF | 58 EOF |
66 -s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE} | 76 -s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE} |
67 | 77 |
68 TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme | 78 TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme |
69 | 79 |
70 EOF | 80 EOF |
71 exit 0 | 81 exit 2 |
72 } | 82 } |
73 | 83 |
74 # generate link by transforming xml | 84 # generate link by transforming xml |
75 Generate_Link() { | 85 Generate_Link() { |
76 cd "${BASE_DIR}/temp" | 86 cd "${BASE_DIR}/temp" |
85 <xsl:template match="post"> | 95 <xsl:template match="post"> |
86 <xsl:value-of select="@file_url" /> | 96 <xsl:value-of select="@file_url" /> |
87 </xsl:template> | 97 </xsl:template> |
88 </xsl:stylesheet> | 98 </xsl:stylesheet> |
89 EOF | 99 EOF |
90 NUMFILES=`echo \`wc -l < "${SITE_DIR}-${TARGET_DIR}-list" \`` | 100 NUMFILES=$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-list")) |
91 [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" | 101 [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" |
92 echo "${NUMFILES} file(s) available on server" | 102 echo "${NUMFILES} file(s) available on server" |
93 #output file: ${TARGET_DIR}-list | 103 #output file: ${TARGET_DIR}-list |
94 } | 104 } |
95 | 105 |
96 # check tools availability | 106 # check tools availability |
97 Check_Tools() { | 107 Check_Tools() { |
98 # verify all programs required do indeed exist | 108 # verify all programs required do indeed exist |
99 #MD5 | 109 #MD5 |
100 if [ ! "${MD5}" ]; then | 110 if [ ! "${MD5}" ]; then |
101 case `uname` in | 111 case $(uname) in |
102 *BSD) MD5="md5 -r";; | 112 *BSD) MD5="md5 -r";; |
103 Linux|SunOS) MD5="md5sum";; | 113 Linux|SunOS) MD5="md5sum";; |
104 *) Fatal_Err "No known md5 tool for this platform. Please specify manually" | 114 *) Fatal_Err "No known md5 tool for this platform. Please specify manually" |
105 esac | 115 esac |
106 fi | 116 fi |
107 MD5_COMMAND=`echo ${MD5} | cut -d' ' -f1` | 117 MD5_COMMAND=$(echo ${MD5} | cut -d' ' -f1) |
108 # basic tools | 118 # basic tools |
109 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}" | 119 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}" |
110 for COMMAND in ${COMMANDS} | 120 for COMMAND in ${COMMANDS} |
111 do | 121 do |
112 COMMAND_CHECK=`command -v "${COMMAND}"` | 122 [ "$(command -v "${COMMAND}")" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}" |
113 [ "${COMMAND_CHECK}" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}" | |
114 done | 123 done |
115 | 124 |
116 # grep checking | 125 # grep checking |
117 # originally created for workaround on solaris | 126 # originally created for workaround on solaris |
118 #if [ `uname` = "SunOS" ]; then | 127 #if [ `uname` = "SunOS" ]; then |
119 FAIL="" | 128 FAIL= |
120 echo "blah" > superrandomtestfile | 129 echo "blah" > superrandomtestfile |
121 echo "blah" > superrandomtestfile.2 | 130 echo "blah" > superrandomtestfile.2 |
122 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1 | 131 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1 |
123 rm -f superrandomtestfile superrandomtestfile.2 | 132 rm -f superrandomtestfile superrandomtestfile.2 |
124 [ "${FAIL}" ] && Err_Fatal "Your grep is not compatible. Please install or set path of correct grep" | 133 [ "${FAIL}" ] && Err_Fatal "Your grep is not compatible. Please install or set path of correct grep" |
134 if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then | 143 if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then |
135 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it" | 144 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it" |
136 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen" | 145 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen" |
137 fi | 146 fi |
138 done | 147 done |
139 [ `echo \`ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l\`` -eq 0 ] && ISNEW=1 | 148 [ "$(echo $(ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l))" -eq 0 ] && ISNEW=1 |
140 # let's move to workdir | 149 # let's move to workdir |
141 cd "${BASE_DIR}/temp" | 150 cd "${BASE_DIR}/temp" |
142 for i in error ok list newlist; do | 151 for i in error ok list newlist; do |
143 touch "${SITE_DIR}-${TARGET_DIR}-${i}" || Fatal_Err "Error creating ${TARGET_DIR}-${i}. This shouldn't happen" | 152 touch "${SITE_DIR}-${TARGET_DIR}-${i}" || Fatal_Err "Error creating ${TARGET_DIR}-${i}. This shouldn't happen" |
144 done | 153 done |
158 do | 167 do |
159 ISTRASH= | 168 ISTRASH= |
160 if [ -d "${TRASH}" ]; then | 169 if [ -d "${TRASH}" ]; then |
161 ISTRASH=1 | 170 ISTRASH=1 |
162 else | 171 else |
163 if [ "$(echo "${FILE}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}.*\)//g' | grep -v ^$)" ]; then | 172 if [ "$(echo "${TRASH}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then |
164 ISTRASH=1 | 173 ISTRASH=1 |
165 else | 174 else |
166 [ "$(cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list" | sed -e 's/.*\/\([^\/]*\)/\1/g' | grep $(echo "${TRASH}" | sed -e 's/.*\/\([^\/]*\)/\1/g'))" ] || ISTRASH=1 | 175 [ "$(cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list" | sed -e 's/.*\/\([^\/]*\)/\1/g' | grep $(echo "${TRASH}" | sed -e 's/.*\/\([^\/]*\)/\1/g'))" ] || ISTRASH=1 |
167 fi | 176 fi |
168 fi | 177 fi |
182 echo "Checking for errors..." | 191 echo "Checking for errors..." |
183 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | 192 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" |
184 printf "" > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | 193 printf "" > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" |
185 for FILE in * | 194 for FILE in * |
186 do | 195 do |
187 if [ "`echo \"${FILE}\" | sed -e 's/\([0-9a-f]\{32\}.*\)//g' | grep -v ^$`" ]; then | 196 if [ "$(echo "${FILE}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then |
197 echo | |
188 echo "Not a valid danbooru file: ${FILE}" | 198 echo "Not a valid danbooru file: ${FILE}" |
189 else | 199 else |
190 if [ "`${MD5} "${FILE}" | cut -d ' ' -f1 -`" != "`echo "${FILE}" | cut -d '.' -f1`" ] | 200 if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | cut -d '.' -f1)" ] |
191 then | 201 then |
192 echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | 202 echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" |
193 echo | 203 echo |
194 echo "Error: ${FILE}" | 204 echo "Error: ${FILE}" |
195 fi | 205 fi |
196 printf "." | 206 printf "." |
197 fi | 207 fi |
198 done | 208 done |
199 echo | 209 echo |
200 TOTAL_ERROR=`echo \`wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"\`` | 210 TOTAL_ERROR=$(echo $(wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error")) |
201 echo "${TOTAL_ERROR} file(s) error" | 211 echo "${TOTAL_ERROR} file(s) error" |
202 echo "Removing error files" | 212 echo "Removing error files" |
203 if [ "${TOTAL_ERROR}" -eq 0 ]; then | 213 if [ "${TOTAL_ERROR}" -eq 0 ]; then |
204 echo "No error file. 0 file removed" | 214 echo "No error file. 0 file removed" |
205 else | 215 else |
206 cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm | 216 cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm |
207 echo "${TOTAL_ERROR} file(s) removed" | 217 echo "${TOTAL_ERROR} file(s) removed" |
208 fi | 218 fi |
209 echo "`echo \`ls | wc -l\`` file(s) available locally" | 219 echo "$(echo $(ls | wc -l)) file(s) available locally" |
210 | 220 |
211 # current dir: ${BASE_DIR}/temp | 221 # current dir: ${BASE_DIR}/temp |
212 cd ${BASE_DIR}/temp | 222 cd "${BASE_DIR}/temp" |
213 | 223 |
214 echo "Generating list of new files..." | 224 echo "Generating list of new files..." |
215 # THE FILES | 225 # THE FILES |
216 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" | 226 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" |
217 # | 227 # |
218 ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok" | 228 ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok" |
219 cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist" | 229 cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist" |
220 echo "`echo \`wc -l < \"${SITE_DIR}-${TARGET_DIR}-newlist\"\`` file(s) to be downloaded" | 230 echo "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist")) file(s) to be downloaded" |
221 | 231 |
222 else | 232 else |
223 if [ "${ISQUICK}" ]; then | 233 if [ "${ISQUICK}" ]; then |
224 echo "quick mode selected. Skipping check" | 234 echo "quick mode selected. Skipping check" |
225 else | 235 else |
231 } | 241 } |
232 | 242 |
233 # start downloading the images | 243 # start downloading the images |
234 Fetch_Images() { | 244 Fetch_Images() { |
235 cd "${BASE_DIR}/temp" | 245 cd "${BASE_DIR}/temp" |
236 if [ `echo \`wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"\`` -eq 0 ]; then | 246 if [ "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"))" -eq 0 ]; then |
237 echo "No new file" | 247 echo "No new file" |
238 else | 248 else |
239 echo "Starting wget" | 249 echo "Starting wget" |
240 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | 250 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" |
241 wget -e continue=on -bi "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-newlist" -o "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}.log" | 251 wget -e continue=on -bi "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-newlist" -o "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}.log" |
287 [ "${SITE}" ] || SITE="${DEFAULT_SITE}" | 297 [ "${SITE}" ] || SITE="${DEFAULT_SITE}" |
288 [ "${TAGS}" ] || Err_Fatal "No tag specified" | 298 [ "${TAGS}" ] || Err_Fatal "No tag specified" |
289 # Get base folder - default, current folder or fallback to ${HOME} | 299 # Get base folder - default, current folder or fallback to ${HOME} |
290 [ "${BASE_DIR}" ] || BASE_DIR="${PWD}" | 300 [ "${BASE_DIR}" ] || BASE_DIR="${PWD}" |
291 [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}" | 301 [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}" |
292 [ "`echo \"${BASE_DIR}\" | cut -c1 | grep \/`" ] || BASE_DIR="/${BASE_DIR}" | 302 [ "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" ] || BASE_DIR="/${BASE_DIR}" |
293 | 303 |
294 echo "Tags: ${TAGS}" | 304 echo "Tags: ${TAGS}" |
295 # slash is not wanted for folder name | 305 # slash is not wanted for folder name |
296 TARGET_DIR="`echo "${TAGS}" | sed -e 's/\//_/g'`" | 306 TARGET_DIR=$(echo "${TAGS}" | sed -e 's/\//_/g') |
297 SITE_DIR="`echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g'`" | 307 SITE_DIR=$(echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g') |
298 } | 308 } |
299 | 309 |
300 # initialization | 310 # initialization |
301 Msg_Welcome | 311 Msg_Welcome |
302 Init "$@" | 312 Init "$@" |