Mercurial > ec-dotfiles

comparison moefetch.sh @ 215:710082ce6788

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
major cleanup part2. also called major mess up part 2.
author edhoprima@gmail.com <edhoprima@gmail.com>
date Thu, 02 Jul 2009 20:01:13 +0000
parents a6624fb9b317
children a869987c4646
comparison
equal deleted inserted replaced
214:a6624fb9b317 215:710082ce6788
36 # Leave empty to use whatever folder you're running this at 36 # Leave empty to use whatever folder you're running this at
37 BASE_DIR= 37 BASE_DIR=
38 38
39 # not user modifiable from here 39 # not user modifiable from here
40 40
41 SED_GET_FILENAME="s/.*\/\([^\/]*\)/\1/g"
42 SED_IS_MD5_FILE="s/\([0-9a-f]\{32\}\..*\)//g"
43
44 ### TODO: 41 ### TODO:
45 ### - sanity validator(?) 42 ### - sanity validator(?)
46 ### - unified repository to save bandwidth 43 ### - unified repository to save bandwidth
47 ### - bug stomping 44 ### - bug stomping
48 ### - sanity checking 45 ### - sanity checking
49 ### WILL BE FOR 0.3 46 ### WILL BE FOR 0.3
50 47
51 # useless welcome message. Also version 48 # useless welcome message. Also version
52 Msg_Welcome() { 49 Msg_Welcome() {
53 MOEFETCHVERSION="0.2.1" 50 MOEFETCHVERSION="0.3-beta1"
54 echo "moefetch ${MOEFETCHVERSION} 51 echo "moefetch ${MOEFETCHVERSION}
55 Copyright (c) 2009 edogawaconan <me@myconan.net>" 52 Copyright (c) 2009 edogawaconan <me@myconan.net>
53 "
56 } 54 }
57 55
58 # fatal error handler 56 # fatal error handler
59 Err_Fatal() { 57 Err_Fatal() {
60 echo " 58 echo "
61 Fatal error: ${1}" 59 Fatal error: ${1}"
62 exit 1 60 exit 1
63 } 61 }
64 62
63 Err_Impossible() {
64 echo "
65 Impossible error. Or you modified content of the working directories when the script is running.
66 Please report to moefetch.googlecode.com if you see this message (complete with entire run log)"
67 exit 1
68 }
69
65 # help message 70 # help message
66 Err_Help() { 71 Err_Help() {
67 echo "moefetch.sh COMMAND [-s SITE_URL] TAGS 72 echo "moefetch.sh COMMAND [-s SITE_URL] TAGS
68 73
69 COMMAND: 74 COMMAND:
77 } 82 }
78 83
79 # generate link by transforming xml 84 # generate link by transforming xml
80 Generate_Link() { 85 Generate_Link() {
81 echo " 86 echo "
82 Fetching xml file" 87 Fetching XML file"
83 TEMPNUM=1000 88 tempnum=1000
84 _i=1 89 _i=1
85 > "${TEMP_PREFIX}-list" 90 > "${TEMP_PREFIX}-list"
86 while [ "${TEMPNUM}" -ge 1000 ]; do 91 while [ "${tempnum}" -ge 1000 ]; do
87 wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=1000&page=${_i}" -O "${TEMP_PREFIX}-xml" -e continue=off 92 wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=1000&page=${_i}" -O "${TEMP_PREFIX}-xml" -e continue=off
88 printf "Processing XML file... " 93 printf "Processing XML file... "
89 # xslt evilry 94 # xslt evilry
90 > "${TEMP_PREFIX}-templist" 95 xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-templist"
91 xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-templist" 2>/dev/null
92 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> 96 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
93 <xsl:output method="xml" indent="yes"/> 97 <xsl:output method="xml" indent="yes"/>
94 <xsl:template match="post"> 98 <xsl:template match="post">
95 <xsl:value-of select="@file_url" /> 99 <xsl:value-of select="@file_url" />
96 </xsl:template> 100 </xsl:template>
97 </xsl:stylesheet> 101 </xsl:stylesheet>
98 EOF 102 EOF
99 TEMPNUM=$(echo $(wc -l < "${TEMP_PREFIX}-templist")) 103 tempnum=$(echo $(wc -l < "${TEMP_PREFIX}-templist"))
100 _i=$((_i+1)) 104 _i=$((_i+1))
101 cat "${TEMP_PREFIX}-templist" >> "${TEMP_PREFIX}-list" 105 cat "${TEMP_PREFIX}-templist" >> "${TEMP_PREFIX}-list"
102 echo "${TEMPNUM} file(s) available" 106 echo "${tempnum} file(s) available"
103 done 107 done
104 NUMFILES=$(echo $(wc -l < "${TEMP_PREFIX}-list")) 108 numfiles=$(echo $(wc -l < "${TEMP_PREFIX}-list"))
105 echo "${NUMFILES} file(s) available on server" 109 echo "${numfiles} file(s) available on server"
106 [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" 110 [ "${numfiles}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site."
107 } 111 }
108 112
109 Is_NotMD5() { 113 Is_NotMD5() {
110 echo "${*}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g' 114 echo "${*}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g'
111 } 115 }
129 133
130 # getting rid of ls (as per suggestion) 134 # getting rid of ls (as per suggestion)
131 Count_Files() { 135 Count_Files() {
132 _i=0 136 _i=0
133 for _f in "${*}/"* "${*}/".*; do 137 for _f in "${*}/"* "${*}/".*; do
134 if [ "${_f}" != "${*}/"'*' ] || [ -e "${_f}" ]; then 138 if test "${_f}" != "${*}/"'*' || test -e "${_f}"; then
135 _i=$((_i + 1)) 139 _i=$((_i + 1))
136 fi 140 fi
137 done 141 done
138 echo $((_i - 2)) 142 echo $((_i - 2))
139 } 143 }
140 144
141 # check tools availability 145 # check tools availability
142 Check_Tools() { 146 Check_Tools() {
143 # verify all programs required do indeed exist 147 # verify all programs required do indeed exist
144 #MD5 148 #MD5
145 if [ ! "${MD5}" ]; then 149 if test -z "${MD5}"; then
146 case $(uname) in 150 case "$(uname)" in
147 *BSD) MD5="md5 -r";; 151 *BSD) MD5="md5 -r";;
148 Linux|SunOS) MD5="md5sum";; 152 Linux|SunOS) MD5="md5sum";;
149 *) Fatal_Err "No known md5 tool for this platform. Please specify manually" 153 *) Fatal_Err "No known md5 tool for this platform. Please specify manually";;
150 esac 154 esac
151 fi 155 fi
152 MD5_COMMAND=$(echo ${MD5} | cut -d' ' -f1) 156 md5_command=$(echo ${MD5} | cut -d' ' -f1)
153 # basic tools 157 # basic tools
154 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}" 158 commands="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${md5_command}"
155 for COMMAND in ${COMMANDS} 159 for cmd in ${commands}
156 do 160 do
157 [ "$(command -v "${COMMAND}")" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}" 161 [ "$(command -v "${cmd}")" ] || Err_Fatal "${cmd} doesn't exist in ${PATH}"
158 done 162 done
159 } 163 }
160 164
161 # verify required folders exist and writeable 165 # verify required folders exist and writeable
162 Check_Folders(){ 166 Check_Folders(){
163 [ -O "${BASE_DIR}" ] || Err_Fatal "You don't own ${BASE_DIR}. Please fix ${BASE_DIR}." 167 test -O "${BASE_DIR}" || Err_Fatal "You don't own ${BASE_DIR}. Please fix ${BASE_DIR} or run this script in your own directory."
164 for FOLDER in temp trash deleted "${SITE_DIR}/${TARGET_DIR}"; do 168 for directory in temp trash deleted "${SITE_DIR}/${TARGET_DIR}"; do
165 if [ ! -d "${BASE_DIR}/${FOLDER}" ]; then 169 if [ ! -d "${BASE_DIR}/${directory}" ]; then
166 mkdir "${BASE_DIR}/${FOLDER}" || Err_Fatal "${FOLDER} folder creation failed" 170 mkdir "${BASE_DIR}/${directory}" || Err_Impossible
167 fi 171 fi
168 if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then 172 if [ ! -O "${BASE_DIR}/${directory}" ]; then
169 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it" 173 echo "You don't own the ${BASE_DIR}/${directory}, applying globally writeable permission on it"
170 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen" 174 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${directory}" || Err_Impossible
171 fi 175 fi
172 done 176 done
173 [ "$(Count_Files "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}")" -eq 0 ] && ISNEW=1 177 test "$(Count_Files "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}")" -eq 0 && ISNEW=1
174 for i in error ok list newlist templist; do 178 for i in error ok list newlist templist; do
175 touch "${TEMP_PREFIX}-${i}" || Fatal_Err "Error creating ${TEMP_PREFIX}-${i}. This shouldn't happen" 179 touch "${TEMP_PREFIX}-${i}" || Fatal_Err "Error creating ${TEMP_PREFIX}-${i}. This shouldn't happen"
176 done 180 done
177 # 181 #
178 } 182 }
180 # Do some cleanup 184 # Do some cleanup
181 Cleanup_Repository() { 185 Cleanup_Repository() {
182 # current dir: ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR} 186 # current dir: ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}
183 printf "Cleaning up repository folder... " 187 printf "Cleaning up repository folder... "
184 Progress_Init 188 Progress_Init
185 TRASH_DIR=$(date -u "+${SITE_DIR}-${TARGET_DIR}-%Y%m%d-%H.%M") 189 trash_dir="${BASE_DIR}/trash/${trash_dir}/$(date -u "+${SITE_DIR}-${TARGET_DIR}-%Y%m%d-%H.%M")"
186 TRASHES= 190 trashes="These files have been moved to ${trash_dir}:"
187 mkdir -p "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Unable to create trash folder" 191 has_trash=
188 for TRASH in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* 192 if test ! -d "${trash_dir}"; then
193 mkdir -p "${BASE_DIR}/trash/${trash_dir}" || Err_Impossible
194 else
195 if test ! -o "${trash_dir}"; then
196 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${directory}" || Err_Impossible
197 fi
198 fi
199 for trash in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"*
189 do 200 do
190 ISTRASH= 201 is_trash=
191 if [ -d "${TRASH}" ]; then 202 if test -d "${trash}" || test -n "$(Is_NotMD5 "$(basename "${trash}")")" || test -n "$(grep "$(basename "${trash}")" "${TEMP_PREFIX}-list")"; then
192 ISTRASH=1 203 is_trash=1
193 else 204 has_trash=1
194 if [ "$(Is_NotMD5 "$(basename "${TRASH}")")" ]; then 205 mv -f "${trash}" "${trash_dir}" || Err_Impossible
195 ISTRASH=1 206 trashes="${trashes}
196 else 207 $(basename "${trash}")"
197 [ "$(grep "$(basename "${TRASH}")" "${TEMP_PREFIX}-list")" ] || ISTRASH=1
198 fi
199 fi
200 if [ "${ISTRASH}" ]; then
201 mv -f "${TRASH}" "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Error deleting files"
202 TRASHES="${TRASHES}
203 $(basename "${TRASH}")"
204 fi 208 fi
205 Progress_Anim 209 Progress_Anim
206 done 210 done
207 rmdir "${BASE_DIR}/trash/${TRASH_DIR}" 2>/dev/null 211 rmdir "${trash_dir}" 2>/dev/null
208 Progress_Done 212 Progress_Done
209 echo "These files have been moved to ${BASE_DIR}/trash/${TRASH_DIR}: 213 test -z "${has_trash}" || echo "${trashes}"
210 ${TRASHES}"
211 } 214 }
212 215
213 # check files correctness 216 # check files correctness
214 Check_Files() { 217 Check_Files() {
215 if test ! -n "${ISNEW}"; then 218 if test ! -n "${ISNEW}"; then
256 printf "Generating list of new files... " 259 printf "Generating list of new files... "
257 Progress_Init 260 Progress_Init
258 cat "${TEMP_PREFIX}-list" > "${TEMP_PREFIX}-templist" 261 cat "${TEMP_PREFIX}-list" > "${TEMP_PREFIX}-templist"
259 while read -r is_ok; do 262 while read -r is_ok; do
260 grep -v "${is_ok}" "${TEMP_PREFIX}-templist" > "${TEMP_PREFIX}-newlist" 263 grep -v "${is_ok}" "${TEMP_PREFIX}-templist" > "${TEMP_PREFIX}-newlist"
261 cat "${TEMP_PREFIX}-newlist" > "${TEMP_PREFIX}-templist" 264 cp -f "${TEMP_PREFIX}-newlist" "${TEMP_PREFIX}-templist" || Err_Impossible
262 Progress_Anim 265 Progress_Anim
263 done < "${TEMP_PREFIX}-ok" 266 done < "${TEMP_PREFIX}-ok"
264 Progress_Done 267 Progress_Done
265 echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded" 268 echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded"
266 else 269 else
273 fi 276 fi
274 } 277 }
275 278
276 # start downloading the images 279 # start downloading the images
277 Fetch_Images() { 280 Fetch_Images() {
278 if [ "$(echo $(wc -l < "${TEMP_PREFIX}-newlist"))" -eq 0 ]; then 281 if test "$(echo $(wc -l < "${TEMP_PREFIX}-newlist"))" -eq 0; then
279 echo "No new file" 282 echo "No new file"
280 else 283 else
281 printf "Starting wget... " 284 printf "Starting wget... "
282 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" 285 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}"
283 wget -e continue=on -bi "${TEMP_PREFIX}-newlist" -o "${TEMP_PREFIX}.log" 286 wget -e continue=on -bi "${TEMP_PREFIX}-newlist" -o "${TEMP_PREFIX}.log"
285 } 288 }
286 289
287 # initialize base variables and initial command check 290 # initialize base variables and initial command check
288 Init(){ 291 Init(){
289 # path initialization 292 # path initialization
290 [ "${ADDITIONAL_PATH}" ] && PATH="${ADDITIONAL_PATH}:${PATH}" 293 test -n "${ADDITIONAL_PATH}" && PATH="${ADDITIONAL_PATH}:${PATH}"
291 export PATH 294 export PATH
292 295
293 # misc variables 296 # misc variables
294 ISQUICK= 297 ISQUICK=
295 ISNEW= 298 ISNEW=
296 299
297 [ $# -lt 2 ] && Err_Help 300 [ $# -lt 2 ] && Err_Help
298 case "$1" in 301 case "$1" in
299 check|fetch|quickfetch) 302 check|fetch|quickfetch)
300 echo "Starting..." 303 echo "Starting..."
301 JOB="$1" 304 JOB="$1"
324 fi 327 fi
325 ;; 328 ;;
326 esac 329 esac
327 shift 330 shift
328 done 331 done
329 [ "${SITE}" ] || SITE="${DEFAULT_SITE}" 332 test -n "${SITE}" || SITE=${DEFAULT_SITE}
330 [ "${TAGS}" ] || Err_Fatal "No tag specified" 333 test -n "${TAGS}" || Err_Fatal "No tag specified"
331 # Get base folder - default, current folder or fallback to ${HOME} 334 # Get base folder - default, current folder or fallback to ${HOME}
332 [ "${BASE_DIR}" ] || BASE_DIR="${PWD}" 335 test -n "${BASE_DIR}" || BASE_DIR=${PWD}
333 [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}" 336 test -n "${BASE_DIR}" || BASE_DIR=${HOME}
334 [ "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" ] || BASE_DIR="/${BASE_DIR}" 337 test -n "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" || BASE_DIR="/${BASE_DIR}"
335 338
336 echo "Tags: ${TAGS}" 339 echo "Tags: ${TAGS}"
337 # slash is not wanted for folder name 340 # slash is not wanted for folder name
338 TARGET_DIR=$(echo "${TAGS}" | sed -e 's/\//_/g') 341 TARGET_DIR=$(echo "${TAGS}" | sed -e 's/\//_/g')
339 SITE_DIR=$(echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g') 342 SITE_DIR=$(echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g')