Mercurial > ec-dotfiles

comparison moefetch.sh @ 232:5438d80244a3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- version bump
author edhoprima
date Sun, 14 Mar 2010 18:35:03 +0000
parents 4c0fd276665e
children 58ad057cd2ec
comparison
equal deleted inserted replaced
231:4c0fd276665e 232:5438d80244a3
32 # Leave empty to use whatever folder you're running this at 32 # Leave empty to use whatever folder you're running this at
33 BASE_DIR= 33 BASE_DIR=
34 34
35 # not user modifiable from here 35 # not user modifiable from here
36 36
37 ### TODO:
38 ### - sanity validator(?)
39 ### - unified repository to save bandwidth
40 ### - bug stomping
41 ### - sanity checking
42 ### - MOAR comments
43 ### WILL BE FOR 0.3
44
45 # useless welcome message. Also version 37 # useless welcome message. Also version
46 msg_welcome() { 38 msg_welcome() {
47 echo "moefetch ${_version} 39 echo "moefetch ${_version}
48 Copyright (c) 2009 edogawaconan <me@myconan.net> 40 Copyright (c) 2009 edogawaconan <me@myconan.net>
49 " 41 "
51 43
52 # Sanitize path. Totally safe. Usage: cmd "$(safe_path "${filename}")" 44 # Sanitize path. Totally safe. Usage: cmd "$(safe_path "${filename}")"
53 safe_path() 45 safe_path()
54 { 46 {
55 # It all depends on the first character. 47 # It all depends on the first character.
56 __start=$(printf "%s" "$*" | cut -c 1) 48 start=$(printf "%s" "$*" | cut -c 1)
57 __path= 49 path=
58 case "${__start}" in 50 case "${start}" in
59 .|/) __path="$*";; # . and / is safe. No change. 51 .|/) path="$*";; # . and / is safe. No change.
60 *) __path="./$*";; # Anything else must be prefixed with ./ 52 *) path="./$*";; # Anything else must be prefixed with ./
61 esac 53 esac
62 printf "%s" "${__path}" # Return. 54 printf "%s" "${path}" # Return.
63 } 55 }
64 56
65 # Checks md5. OpenSSL should be available on anything usable. 57 # Checks md5. OpenSSL should be available on anything usable.
66 get_md5() { cat "$(safe_path "${1}")" | openssl dgst -md5 | tail -n 1 | sed -e 's/.*\([[:xdigit:]]\{32\}\).*/\1/'; } 58 get_md5() { cat "$(safe_path "${1}")" | openssl dgst -md5 | tail -n 1 | sed -e 's/.*\([[:xdigit:]]\{32\}\).*/\1/'; }
67 59
118 110
119 # generate link by transforming xml 111 # generate link by transforming xml
120 Generate_Link() { 112 Generate_Link() {
121 echo " 113 echo "
122 Fetching XML file" 114 Fetching XML file"
123 __tempnum=1000 115 tempnum=1000
124 __iternum=1 116 iternum=1
125 > "${TEMP_PREFIX}-list" 117 > "${TEMP_PREFIX}-list"
126 while [ "${__tempnum}" -ge 1000 ]; do 118 while [ "${tempnum}" -ge 1000 ]; do
127 __url="http://${SITE}/post/index.xml?tags=$(get_cleantags "${TAGS}")&offset=0&limit=1000&page=${__iternum}" 119 url="http://${SITE}/post/index.xml?tags=$(get_cleantags "${TAGS}")&offset=0&limit=1000&page=${iternum}"
128 [ ${_use_login} -eq 1 ] && __url="${__url}&login=${LOGIN_USER}&password_hash=${LOGIN_PASS}" 120 [ ${_use_login} -eq 1 ] && url="${url}&login=${LOGIN_USER}&password_hash=${LOGIN_PASS}"
129 wget --quiet "${__url}" -O "${TEMP_PREFIX}-xml" -e continue=off || Err_Fatal "Failed download catalog file" 121 wget --quiet "${url}" -O "${TEMP_PREFIX}-xml" -e continue=off || Err_Fatal "Failed download catalog file"
130 printf "Processing XML file... " 122 printf "Processing XML file... "
131 # xslt evilry 123 # xslt evilry
132 xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-templist" 124 xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-templist"
133 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> 125 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
134 <xsl:output method="xml" indent="yes"/> 126 <xsl:output method="xml" indent="yes"/>
135 <xsl:template match="post"> 127 <xsl:template match="post">
136 <xsl:value-of select="@file_url" /> 128 <xsl:value-of select="@file_url" />
137 </xsl:template> 129 </xsl:template>
138 </xsl:stylesheet> 130 </xsl:stylesheet>
139 EOF 131 EOF
140 __tempnum=$(echo $(wc -l < "${TEMP_PREFIX}-templist")) 132 tempnum=$(echo $(wc -l < "${TEMP_PREFIX}-templist"))
141 __iternum=$((__iternum + 1)) 133 iternum=$((iternum + 1))
142 cat "${TEMP_PREFIX}-templist" >> "${TEMP_PREFIX}-list" 134 cat "${TEMP_PREFIX}-templist" >> "${TEMP_PREFIX}-list"
143 echo "${__tempnum} file(s) available" 135 echo "${tempnum} file(s) available"
144 done 136 done
145 numfiles=$(echo $(wc -l < "${TEMP_PREFIX}-list")) 137 numfiles=$(echo $(wc -l < "${TEMP_PREFIX}-list"))
146 echo "${numfiles} file(s) available on server" 138 echo "${numfiles} file(s) available on server"
147 [ "${numfiles}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site." 139 [ "${numfiles}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site."
148 } 140 }
241 Check_Files() { 233 Check_Files() {
242 if [ ! -n "${ISNEW}" ]; then 234 if [ ! -n "${ISNEW}" ]; then
243 [ -z "${NOCLEAN}" ] && Cleanup_Repository 235 [ -z "${NOCLEAN}" ] && Cleanup_Repository
244 printf "Checking for errors... " 236 printf "Checking for errors... "
245 progress_init 237 progress_init
246 __files_error="These files do not match its md5:" 238 files_error="These files do not match its md5:"
247 __files_notdanbooru="These files are not checked:" 239 files_notdanbooru="These files are not checked:"
248 __has_err_filename= 240 has_err_filename=
249 __has_err_md5= 241 has_err_md5=
250 > "${TEMP_PREFIX}-error" 242 > "${TEMP_PREFIX}-error"
251 > "${TEMP_PREFIX}-ok" 243 > "${TEMP_PREFIX}-ok"
252 for file in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* 244 for file in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"*
253 do 245 do
254 if [ "${file}" != "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/*" ]; then 246 if [ "${file}" != "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/*" ]; then
255 if [ -n "$(is_not_md5 "${file}")" ] || [ -d "${file}" ]; then 247 if [ -n "$(is_not_md5 "${file}")" ] || [ -d "${file}" ]; then
256 __files_notdanbooru="${__files_notdanbooru} 248 files_notdanbooru="${files_notdanbooru}
257 $(get_basename "${file}")" 249 $(get_basename "${file}")"
258 __has_err_filename=1 250 has_err_filename=1
259 else 251 else
260 if [ "$(get_md5 "${file}")" = "$(get_filename "${file}")" ]; then 252 if [ "$(get_md5 "${file}")" = "$(get_filename "${file}")" ]; then
261 echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-ok" 253 echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-ok"
262 else 254 else
263 rm "${file}" || Err_Fatal "Error removing ${file}" 255 rm "${file}" || Err_Fatal "Error removing ${file}"
264 echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-error" 256 echo "$(get_basename "${file}")" >> "${TEMP_PREFIX}-error"
265 __files_error="${__files_error} 257 files_error="${files_error}
266 $(get_basename "${file}")" 258 $(get_basename "${file}")"
267 __has_err_md5=1 259 has_err_md5=1
268 fi 260 fi
269 fi 261 fi
270 fi 262 fi
271 progress_anim 263 progress_anim
272 done 264 done
273 progress_done 265 progress_done
274 if [ ! -n "${__has_err_md5}" ] && [ ! -n "${__has_err_filename}" ]; then 266 if [ ! -n "${has_err_md5}" ] && [ ! -n "${has_err_filename}" ]; then
275 echo "All files OK" 267 echo "All files OK"
276 else 268 else
277 if [ -n "${__has_err_md5}" ]; then 269 if [ -n "${has_err_md5}" ]; then
278 echo "${__files_error}" 270 echo "${files_error}"
279 echo "$(echo $(wc -l < "${TEMP_PREFIX}-error")) file(s) removed" 271 echo "$(echo $(wc -l < "${TEMP_PREFIX}-error")) file(s) removed"
280 fi 272 fi
281 [ -n "${__has_err_filename}" ] && echo "${__files_notdanbooru}" 273 [ -n "${has_err_filename}" ] && echo "${files_notdanbooru}"
282 fi 274 fi
283 echo "$(echo $(wc -l < "${TEMP_PREFIX}-ok")) file(s) available locally" 275 echo "$(echo $(wc -l < "${TEMP_PREFIX}-ok")) file(s) available locally"
284 276
285 printf "Generating list of new files... " 277 printf "Generating list of new files... "
286 progress_init 278 progress_init
341 ;; 333 ;;
342 esac 334 esac
343 shift 335 shift
344 SITE= 336 SITE=
345 TAGS= 337 TAGS=
346 __has_pass=0 338 has_pass=0
347 __has_user=0 339 has_user=0
348 x=1 340 x=1
349 while getopts "s:nu:p:" opt 341 while getopts "s:nu:p:" opt
350 do 342 do
351 case "$opt" in 343 case "$opt" in
352 s) SITE="$OPTARG";; 344 s) SITE="$OPTARG";;
353 n) NOCLEAN=1;; 345 n) NOCLEAN=1;;
354 p) 346 p)
355 LOGIN_PASS=$(printf "%s" "$OPTARG" | openssl dgst -sha1) 347 LOGIN_PASS=$(printf "%s" "$OPTARG" | openssl dgst -sha1)
356 __has_pass=1 348 has_pass=1
357 ;; 349 ;;
358 u) 350 u)
359 LOGIN_USER="$OPTARG" 351 LOGIN_USER="$OPTARG"
360 __has_user=1 352 has_user=1
361 ;; 353 ;;
362 esac 354 esac
363 x=$OPTIND 355 x=$OPTIND
364 done 356 done
365 shift $(($x-1)) 357 shift $(($x-1))
370 # Get base folder - default, current folder or fallback to ${HOME} 362 # Get base folder - default, current folder or fallback to ${HOME}
371 [ -n "${BASE_DIR}" ] || BASE_DIR=${PWD} 363 [ -n "${BASE_DIR}" ] || BASE_DIR=${PWD}
372 [ -n "${BASE_DIR}" ] || BASE_DIR=${HOME} 364 [ -n "${BASE_DIR}" ] || BASE_DIR=${HOME}
373 [ -n "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" ] || BASE_DIR="/${BASE_DIR}" 365 [ -n "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" ] || BASE_DIR="/${BASE_DIR}"
374 # see if both pass and use are set. If they're set, switch _use_login variable content to 1. 366 # see if both pass and use are set. If they're set, switch _use_login variable content to 1.
375 [ ${__has_pass} -eq 1 -a ${__has_user} -eq 1 ] && _use_login=1 367 [ ${has_pass} -eq 1 -a ${has_user} -eq 1 ] && _use_login=1
376 368
377 echo "Tags: ${TAGS}" 369 echo "Tags: ${TAGS}"
378 # slash is not wanted for folder name 370 # slash is not wanted for folder name
379 TARGET_DIR=$(echo "${TAGS}" | sed -e 's/\//_/g') 371 TARGET_DIR=$(echo "${TAGS}" | sed -e 's/\//_/g')
380 SITE_DIR=$(echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g') 372 SITE_DIR=$(echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g')
382 } 374 }
383 375
384 # global variables goes here 376 # global variables goes here
385 init_globals() 377 init_globals()
386 { 378 {
387 _version="0.3-rc1" # version of this script 379 _version="1.0-rc2" # version of this script
388 _use_login=0 # variable to check whether a login is used or not 380 _use_login=0 # variable to check whether a login is used or not
389 } 381 }
390 382
391 main() 383 main()
392 { 384 {