Mercurial > ec-dotfiles
comparison moefetch.sh @ 193:ac6533a8fb51
- Documentation
- Cleanup
| author | edhoprima@gmail.com <edhoprima@gmail.com> |
|---|---|
| date | Mon, 29 Jun 2009 10:49:40 +0000 |
| parents | 47efb7a23425 |
| children | a8da49424468 |
comparison
equal
deleted
inserted
replaced
| 192:47efb7a23425 | 193:ac6533a8fb51 |
|---|---|
| 32 | 32 |
| 33 # base directory. make sure it's writeable. I do not take responsibility if you don't own the folder and files as no check is done for this one. | 33 # base directory. make sure it's writeable. I do not take responsibility if you don't own the folder and files as no check is done for this one. |
| 34 # Structure is ${BASE_DIR}/<TAGS> | 34 # Structure is ${BASE_DIR}/<TAGS> |
| 35 # Absolute path only. | 35 # Absolute path only. |
| 36 # Leave empty to use whatever folder you're running this at | 36 # Leave empty to use whatever folder you're running this at |
| 37 BASE_DIR="" | 37 BASE_DIR= |
| 38 | 38 |
| 39 # not user modifiable from here | 39 # not user modifiable from here |
| 40 | |
| 41 | |
| 42 ### TODO: | |
| 43 ### - replace `...` with $(..) | |
| 44 ### - sanity validator | |
| 45 ### - unified repository to save bandwidth | |
| 46 ### - bug stomping | |
| 47 ### - sanity checking | |
| 48 ### - replace grep -vf with POSIX compatible command | |
| 49 ### WILL BE FOR 0.3 | |
| 40 | 50 |
| 41 # useless welcome message. Also version | 51 # useless welcome message. Also version |
| 42 Msg_Welcome() { | 52 Msg_Welcome() { |
| 43 MOEFETCHVERSION="0.1-beta2" | 53 MOEFETCHVERSION="0.2-beta1" |
| 44 cat <<EOF | 54 cat <<EOF |
| 45 moefetch ${MOEFETCHVERSION} | 55 moefetch ${MOEFETCHVERSION} |
| 46 Copyright (c) 2009 edogawaconan <me@myconan.net> | 56 Copyright (c) 2009 edogawaconan <me@myconan.net> |
| 47 | 57 |
| 48 EOF | 58 EOF |
| 66 -s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE} | 76 -s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE} |
| 67 | 77 |
| 68 TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme | 78 TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme |
| 69 | 79 |
| 70 EOF | 80 EOF |
| 71 exit 0 | 81 exit 2 |
| 72 } | 82 } |
| 73 | 83 |
| 74 # generate link by transforming xml | 84 # generate link by transforming xml |
| 75 Generate_Link() { | 85 Generate_Link() { |
| 76 cd "${BASE_DIR}/temp" | 86 cd "${BASE_DIR}/temp" |
| 85 <xsl:template match="post"> | 95 <xsl:template match="post"> |
| 86 <xsl:value-of select="@file_url" /> | 96 <xsl:value-of select="@file_url" /> |
| 87 </xsl:template> | 97 </xsl:template> |
| 88 </xsl:stylesheet> | 98 </xsl:stylesheet> |
| 89 EOF | 99 EOF |
| 90 NUMFILES=`echo \`wc -l < "${SITE_DIR}-${TARGET_DIR}-list" \`` | 100 NUMFILES=$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-list")) |
| 91 [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" | 101 [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" |
| 92 echo "${NUMFILES} file(s) available on server" | 102 echo "${NUMFILES} file(s) available on server" |
| 93 #output file: ${TARGET_DIR}-list | 103 #output file: ${TARGET_DIR}-list |
| 94 } | 104 } |
| 95 | 105 |
| 96 # check tools availability | 106 # check tools availability |
| 97 Check_Tools() { | 107 Check_Tools() { |
| 98 # verify all programs required do indeed exist | 108 # verify all programs required do indeed exist |
| 99 #MD5 | 109 #MD5 |
| 100 if [ ! "${MD5}" ]; then | 110 if [ ! "${MD5}" ]; then |
| 101 case `uname` in | 111 case $(uname) in |
| 102 *BSD) MD5="md5 -r";; | 112 *BSD) MD5="md5 -r";; |
| 103 Linux|SunOS) MD5="md5sum";; | 113 Linux|SunOS) MD5="md5sum";; |
| 104 *) Fatal_Err "No known md5 tool for this platform. Please specify manually" | 114 *) Fatal_Err "No known md5 tool for this platform. Please specify manually" |
| 105 esac | 115 esac |
| 106 fi | 116 fi |
| 107 MD5_COMMAND=`echo ${MD5} | cut -d' ' -f1` | 117 MD5_COMMAND=$(echo ${MD5} | cut -d' ' -f1) |
| 108 # basic tools | 118 # basic tools |
| 109 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}" | 119 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}" |
| 110 for COMMAND in ${COMMANDS} | 120 for COMMAND in ${COMMANDS} |
| 111 do | 121 do |
| 112 COMMAND_CHECK=`command -v "${COMMAND}"` | 122 [ "$(command -v "${COMMAND}")" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}" |
| 113 [ "${COMMAND_CHECK}" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}" | |
| 114 done | 123 done |
| 115 | 124 |
| 116 # grep checking | 125 # grep checking |
| 117 # originally created for workaround on solaris | 126 # originally created for workaround on solaris |
| 118 #if [ `uname` = "SunOS" ]; then | 127 #if [ `uname` = "SunOS" ]; then |
| 119 FAIL="" | 128 FAIL= |
| 120 echo "blah" > superrandomtestfile | 129 echo "blah" > superrandomtestfile |
| 121 echo "blah" > superrandomtestfile.2 | 130 echo "blah" > superrandomtestfile.2 |
| 122 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1 | 131 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1 |
| 123 rm -f superrandomtestfile superrandomtestfile.2 | 132 rm -f superrandomtestfile superrandomtestfile.2 |
| 124 [ "${FAIL}" ] && Err_Fatal "Your grep is not compatible. Please install or set path of correct grep" | 133 [ "${FAIL}" ] && Err_Fatal "Your grep is not compatible. Please install or set path of correct grep" |
| 134 if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then | 143 if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then |
| 135 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it" | 144 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it" |
| 136 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen" | 145 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen" |
| 137 fi | 146 fi |
| 138 done | 147 done |
| 139 [ `echo \`ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l\`` -eq 0 ] && ISNEW=1 | 148 [ "$(echo $(ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l))" -eq 0 ] && ISNEW=1 |
| 140 # let's move to workdir | 149 # let's move to workdir |
| 141 cd "${BASE_DIR}/temp" | 150 cd "${BASE_DIR}/temp" |
| 142 for i in error ok list newlist; do | 151 for i in error ok list newlist; do |
| 143 touch "${SITE_DIR}-${TARGET_DIR}-${i}" || Fatal_Err "Error creating ${TARGET_DIR}-${i}. This shouldn't happen" | 152 touch "${SITE_DIR}-${TARGET_DIR}-${i}" || Fatal_Err "Error creating ${TARGET_DIR}-${i}. This shouldn't happen" |
| 144 done | 153 done |
| 158 do | 167 do |
| 159 ISTRASH= | 168 ISTRASH= |
| 160 if [ -d "${TRASH}" ]; then | 169 if [ -d "${TRASH}" ]; then |
| 161 ISTRASH=1 | 170 ISTRASH=1 |
| 162 else | 171 else |
| 163 if [ "$(echo "${FILE}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}.*\)//g' | grep -v ^$)" ]; then | 172 if [ "$(echo "${TRASH}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then |
| 164 ISTRASH=1 | 173 ISTRASH=1 |
| 165 else | 174 else |
| 166 [ "$(cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list" | sed -e 's/.*\/\([^\/]*\)/\1/g' | grep $(echo "${TRASH}" | sed -e 's/.*\/\([^\/]*\)/\1/g'))" ] || ISTRASH=1 | 175 [ "$(cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list" | sed -e 's/.*\/\([^\/]*\)/\1/g' | grep $(echo "${TRASH}" | sed -e 's/.*\/\([^\/]*\)/\1/g'))" ] || ISTRASH=1 |
| 167 fi | 176 fi |
| 168 fi | 177 fi |
| 182 echo "Checking for errors..." | 191 echo "Checking for errors..." |
| 183 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | 192 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" |
| 184 printf "" > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | 193 printf "" > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" |
| 185 for FILE in * | 194 for FILE in * |
| 186 do | 195 do |
| 187 if [ "`echo \"${FILE}\" | sed -e 's/\([0-9a-f]\{32\}.*\)//g' | grep -v ^$`" ]; then | 196 if [ "$(echo "${FILE}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then |
| 197 echo | |
| 188 echo "Not a valid danbooru file: ${FILE}" | 198 echo "Not a valid danbooru file: ${FILE}" |
| 189 else | 199 else |
| 190 if [ "`${MD5} "${FILE}" | cut -d ' ' -f1 -`" != "`echo "${FILE}" | cut -d '.' -f1`" ] | 200 if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | cut -d '.' -f1)" ] |
| 191 then | 201 then |
| 192 echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | 202 echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" |
| 193 echo | 203 echo |
| 194 echo "Error: ${FILE}" | 204 echo "Error: ${FILE}" |
| 195 fi | 205 fi |
| 196 printf "." | 206 printf "." |
| 197 fi | 207 fi |
| 198 done | 208 done |
| 199 echo | 209 echo |
| 200 TOTAL_ERROR=`echo \`wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"\`` | 210 TOTAL_ERROR=$(echo $(wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error")) |
| 201 echo "${TOTAL_ERROR} file(s) error" | 211 echo "${TOTAL_ERROR} file(s) error" |
| 202 echo "Removing error files" | 212 echo "Removing error files" |
| 203 if [ "${TOTAL_ERROR}" -eq 0 ]; then | 213 if [ "${TOTAL_ERROR}" -eq 0 ]; then |
| 204 echo "No error file. 0 file removed" | 214 echo "No error file. 0 file removed" |
| 205 else | 215 else |
| 206 cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm | 216 cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm |
| 207 echo "${TOTAL_ERROR} file(s) removed" | 217 echo "${TOTAL_ERROR} file(s) removed" |
| 208 fi | 218 fi |
| 209 echo "`echo \`ls | wc -l\`` file(s) available locally" | 219 echo "$(echo $(ls | wc -l)) file(s) available locally" |
| 210 | 220 |
| 211 # current dir: ${BASE_DIR}/temp | 221 # current dir: ${BASE_DIR}/temp |
| 212 cd ${BASE_DIR}/temp | 222 cd "${BASE_DIR}/temp" |
| 213 | 223 |
| 214 echo "Generating list of new files..." | 224 echo "Generating list of new files..." |
| 215 # THE FILES | 225 # THE FILES |
| 216 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" | 226 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" |
| 217 # | 227 # |
| 218 ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok" | 228 ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok" |
| 219 cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist" | 229 cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist" |
| 220 echo "`echo \`wc -l < \"${SITE_DIR}-${TARGET_DIR}-newlist\"\`` file(s) to be downloaded" | 230 echo "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist")) file(s) to be downloaded" |
| 221 | 231 |
| 222 else | 232 else |
| 223 if [ "${ISQUICK}" ]; then | 233 if [ "${ISQUICK}" ]; then |
| 224 echo "quick mode selected. Skipping check" | 234 echo "quick mode selected. Skipping check" |
| 225 else | 235 else |
| 231 } | 241 } |
| 232 | 242 |
| 233 # start downloading the images | 243 # start downloading the images |
| 234 Fetch_Images() { | 244 Fetch_Images() { |
| 235 cd "${BASE_DIR}/temp" | 245 cd "${BASE_DIR}/temp" |
| 236 if [ `echo \`wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"\`` -eq 0 ]; then | 246 if [ "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"))" -eq 0 ]; then |
| 237 echo "No new file" | 247 echo "No new file" |
| 238 else | 248 else |
| 239 echo "Starting wget" | 249 echo "Starting wget" |
| 240 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | 250 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" |
| 241 wget -e continue=on -bi "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-newlist" -o "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}.log" | 251 wget -e continue=on -bi "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-newlist" -o "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}.log" |
| 287 [ "${SITE}" ] || SITE="${DEFAULT_SITE}" | 297 [ "${SITE}" ] || SITE="${DEFAULT_SITE}" |
| 288 [ "${TAGS}" ] || Err_Fatal "No tag specified" | 298 [ "${TAGS}" ] || Err_Fatal "No tag specified" |
| 289 # Get base folder - default, current folder or fallback to ${HOME} | 299 # Get base folder - default, current folder or fallback to ${HOME} |
| 290 [ "${BASE_DIR}" ] || BASE_DIR="${PWD}" | 300 [ "${BASE_DIR}" ] || BASE_DIR="${PWD}" |
| 291 [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}" | 301 [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}" |
| 292 [ "`echo \"${BASE_DIR}\" | cut -c1 | grep \/`" ] || BASE_DIR="/${BASE_DIR}" | 302 [ "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" ] || BASE_DIR="/${BASE_DIR}" |
| 293 | 303 |
| 294 echo "Tags: ${TAGS}" | 304 echo "Tags: ${TAGS}" |
| 295 # slash is not wanted for folder name | 305 # slash is not wanted for folder name |
| 296 TARGET_DIR="`echo "${TAGS}" | sed -e 's/\//_/g'`" | 306 TARGET_DIR=$(echo "${TAGS}" | sed -e 's/\//_/g') |
| 297 SITE_DIR="`echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g'`" | 307 SITE_DIR=$(echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g') |
| 298 } | 308 } |
| 299 | 309 |
| 300 # initialization | 310 # initialization |
| 301 Msg_Welcome | 311 Msg_Welcome |
| 302 Init "$@" | 312 Init "$@" |
