Mercurial > ec-dotfiles
annotate moefetch.sh @ 201:30d2fb656029
scrapping grep -vf
| author | edhoprima@gmail.com <edhoprima@gmail.com> |
|---|---|
| date | Mon, 29 Jun 2009 15:40:28 +0000 |
| parents | 8efa600ebfdb |
| children | 3dbb5a6678f9 |
| rev | line source |
|---|---|
| 148 | 1 #!/bin/sh |
| 2 | |
| 3 # Copyright (c) 2009, edogawaconan <me@myconan.net> | |
| 4 # | |
| 5 # Permission to use, copy, modify, and/or distribute this software for any | |
| 6 # purpose with or without fee is hereby granted, provided that the above | |
| 7 # copyright notice and this permission notice appear in all copies. | |
| 8 # | |
| 9 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES | |
| 10 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF | |
| 11 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR | |
| 12 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES | |
| 13 # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN | |
| 14 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF | |
| 15 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | |
| 159 | 16 # |
| 17 # Lots of bugs here. Use with care | |
| 148 | 18 # USE WITH CARE |
| 159 | 19 # |
| 20 # what it does: fetch every picture that has the specified TAGS. | |
| 148 | 21 # requirement: wget, libxslt, md5sum (or md5) |
| 22 | |
| 159 | 23 # program additional paths for: cut, sed, wc, MD5(sum), wget, xsltproc, grep |
| 24 ADDITIONAL_PATH= | |
| 148 | 25 |
| 159 | 26 # custom md5 path with arguments, expected output: <32digit md5><space(s)><filename> |
| 27 # Leave empty for "md5sum" (Linux, Solaris), "md5 -r" (*BSD) | |
| 28 MD5= | |
| 148 | 29 |
| 159 | 30 # default server address. Danbooru only! I do not take responsibility of stupidity. |
| 31 DEFAULT_SITE="moe.imouto.org" | |
| 148 | 32 |
| 33 # base directory. make sure it's writeable. I do not take responsibility if you don't own the folder and files as no check is done for this one. | |
| 159 | 34 # Structure is ${BASE_DIR}/<TAGS> |
| 35 # Absolute path only. | |
| 36 # Leave empty to use whatever folder you're running this at | |
| 193 | 37 BASE_DIR= |
| 148 | 38 |
| 39 # not user modifiable from here | |
| 40 | |
| 196 | 41 SED_GET_FILENAME="s/.*\/\([^\/]*\)/\1/g" |
| 42 SED_IS_MD5_FILE="s/\([0-9a-f]\{32\}\..*\)//g" | |
| 193 | 43 |
| 44 ### TODO: | |
| 196 | 45 ### - replace `...` with $(..) (DONE) |
| 193 | 46 ### - sanity validator |
| 47 ### - unified repository to save bandwidth | |
| 48 ### - bug stomping | |
| 49 ### - sanity checking | |
| 50 ### - replace grep -vf with POSIX compatible command | |
| 51 ### WILL BE FOR 0.3 | |
| 52 | |
| 159 | 53 # useless welcome message. Also version |
| 54 Msg_Welcome() { | |
| 193 | 55 MOEFETCHVERSION="0.2-beta1" |
| 159 | 56 cat <<EOF |
| 57 moefetch ${MOEFETCHVERSION} | |
| 58 Copyright (c) 2009 edogawaconan <me@myconan.net> | |
| 59 | |
| 60 EOF | |
| 61 } | |
| 62 | |
| 63 # fatal error handler | |
| 64 Err_Fatal() { | |
| 196 | 65 printf "\nFatal error: ${1}\n" |
| 159 | 66 exit 1 |
| 67 } | |
| 68 | |
| 69 # help message | |
| 70 Err_Help() { | |
| 71 cat <<EOF | |
| 177 | 72 moefetch.sh COMMAND [-s SITE_URL] TAGS |
|
174
0948e76a57a1
added help. Bump to 0.1-beta2
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
173
diff
changeset
|
73 |
|
176
3d2ae9417273
even more improvement
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
175
diff
changeset
|
74 COMMAND: |
|
3d2ae9417273
even more improvement
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
175
diff
changeset
|
75 (quick)fetch: do a complete update. Add prefix quick to skip file checking |
|
3d2ae9417273
even more improvement
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
175
diff
changeset
|
76 check: get list of new files, clean up local folder and print total new files |
|
175
5b7a154dbd21
cosmetics fix for help message
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
174
diff
changeset
|
77 |
| 177 | 78 -s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE} |
|
175
5b7a154dbd21
cosmetics fix for help message
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
174
diff
changeset
|
79 |
|
176
3d2ae9417273
even more improvement
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
175
diff
changeset
|
80 TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme |
|
174
0948e76a57a1
added help. Bump to 0.1-beta2
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
173
diff
changeset
|
81 |
| 159 | 82 EOF |
| 193 | 83 exit 2 |
| 159 | 84 } |
| 85 | |
| 86 # generate link by transforming xml | |
| 87 Generate_Link() { | |
|
195
652d9e268cee
test migration to printf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
194
diff
changeset
|
88 printf "\nFetching xml file\n" |
|
652d9e268cee
test migration to printf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
194
diff
changeset
|
89 wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=100000" -O "${TEMP_PREFIX}-xml" -e continue=off |
|
652d9e268cee
test migration to printf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
194
diff
changeset
|
90 printf "Processing XML file..." |
| 148 | 91 # xslt evilry |
|
195
652d9e268cee
test migration to printf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
194
diff
changeset
|
92 > "${TEMP_PREFIX}-list" |
|
652d9e268cee
test migration to printf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
194
diff
changeset
|
93 xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-list" 2>/dev/null |
| 148 | 94 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> |
| 95 <xsl:output method="xml" indent="yes"/> | |
| 96 <xsl:template match="post"> | |
| 97 <xsl:value-of select="@file_url" /> | |
| 98 </xsl:template> | |
| 99 </xsl:stylesheet> | |
| 100 EOF | |
|
195
652d9e268cee
test migration to printf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
194
diff
changeset
|
101 NUMFILES=$(echo $(wc -l < "${TEMP_PREFIX}-list")) |
|
181
d3b7927bdb2b
restructuring and add check if the xml is processed properly
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
180
diff
changeset
|
102 [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" |
| 200 | 103 echo " ${NUMFILES} file(s) available on server" |
| 159 | 104 #output file: ${TARGET_DIR}-list |
| 148 | 105 } |
| 106 | |
| 200 | 107 # getting rid of ls (as per suggestion) |
| 108 Count_Files() { | |
| 109 _i=0 | |
| 110 for _f in "${*}/"* "${*}/".*; do | |
| 111 if [ "${_f}" != '*' ] || [ -e "${_f}" ]; then | |
| 112 _i=$((_i + 1)) | |
| 113 fi | |
| 114 done | |
| 115 echo $((_i - 2)) | |
| 116 } | |
| 117 | |
| 159 | 118 # check tools availability |
| 119 Check_Tools() { | |
| 120 # verify all programs required do indeed exist | |
| 121 #MD5 | |
| 122 if [ ! "${MD5}" ]; then | |
| 193 | 123 case $(uname) in |
| 159 | 124 *BSD) MD5="md5 -r";; |
| 125 Linux|SunOS) MD5="md5sum";; | |
| 126 *) Fatal_Err "No known md5 tool for this platform. Please specify manually" | |
| 127 esac | |
| 128 fi | |
| 193 | 129 MD5_COMMAND=$(echo ${MD5} | cut -d' ' -f1) |
| 159 | 130 # basic tools |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
131 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}" |
| 159 | 132 for COMMAND in ${COMMANDS} |
| 133 do | |
| 193 | 134 [ "$(command -v "${COMMAND}")" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}" |
| 159 | 135 done |
| 136 | |
| 137 # grep checking | |
| 138 # originally created for workaround on solaris | |
| 139 #if [ `uname` = "SunOS" ]; then | |
| 193 | 140 FAIL= |
| 159 | 141 echo "blah" > superrandomtestfile |
| 142 echo "blah" > superrandomtestfile.2 | |
| 143 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1 | |
| 144 rm -f superrandomtestfile superrandomtestfile.2 | |
| 145 [ "${FAIL}" ] && Err_Fatal "Your grep is not compatible. Please install or set path of correct grep" | |
| 146 } | |
| 147 | |
| 148 # verify required folders exist and writeable | |
| 149 Check_Folders(){ | |
| 150 [ -O "${BASE_DIR}" ] || Err_Fatal "You don't own ${BASE_DIR}. Please fix ${BASE_DIR}." | |
| 180 | 151 for FOLDER in temp trash deleted "${SITE_DIR}/${TARGET_DIR}"; do |
| 159 | 152 if [ ! -d "${BASE_DIR}/${FOLDER}" ]; then |
| 153 mkdir "${BASE_DIR}/${FOLDER}" || Err_Fatal "${FOLDER} folder creation failed" | |
| 154 fi | |
| 155 if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then | |
| 167 | 156 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it" |
| 159 | 157 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen" |
| 158 fi | |
| 159 done | |
| 200 | 160 [ "$(Count_Files "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}")" -eq 0 ] && ISNEW=1 |
|
201
30d2fb656029
scrapping grep -vf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
200
diff
changeset
|
161 for i in error ok list newlist templist; do |
| 196 | 162 touch "${TEMP_PREFIX}-${i}" || Fatal_Err "Error creating ${TEMP_PREFIX}-${i}. This shouldn't happen" |
| 159 | 163 done |
| 164 # | |
| 165 } | |
| 166 | |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
167 # Do some cleanup |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
168 Cleanup_Repository() { |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
169 |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
170 # THE FILES |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
171 |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
172 # current dir: ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR} |
| 188 | 173 echo "Cleaning up repository folder..." |
|
191
a4ceb952b05a
more noclean support code
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
190
diff
changeset
|
174 TRASH_DIR=$(date -u "+${SITE_DIR}-${TARGET_DIR}-%Y%m%d-%H.%M") |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
175 mkdir -p "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Unable to create trash folder" |
|
190
1061a214a1f1
thanks to folks at #bash!
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
189
diff
changeset
|
176 for TRASH in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
177 do |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
178 ISTRASH= |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
179 if [ -d "${TRASH}" ]; then |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
180 ISTRASH=1 |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
181 else |
| 196 | 182 if [ "$(echo "${TRASH}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then |
|
191
a4ceb952b05a
more noclean support code
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
190
diff
changeset
|
183 ISTRASH=1 |
|
a4ceb952b05a
more noclean support code
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
190
diff
changeset
|
184 else |
| 196 | 185 [ "$(cat "${TEMP_PREFIX}-list" | sed -e "${SED_GET_FILENAME}" | grep $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}"))" ] || ISTRASH=1 |
|
191
a4ceb952b05a
more noclean support code
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
190
diff
changeset
|
186 fi |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
187 fi |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
188 if [ "${ISTRASH}" ]; then |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
189 mv -f "${TRASH}" "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Error deleting files" |
| 196 | 190 echo "Moved $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}") to ${BASE_DIR}/trash/${TRASH_DIR}" |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
191 fi |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
192 done |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
193 rmdir "${BASE_DIR}/trash/${TRASH_DIR}" 2>/dev/null |
|
191
a4ceb952b05a
more noclean support code
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
190
diff
changeset
|
194 |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
195 } |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
196 |
| 159 | 197 # check files correctness |
| 198 Check_Files() { | |
| 166 | 199 if [ ! "${ISNEW}" ]; then |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
200 [ "${NOCLEAN}" ] || Cleanup_Repository |
| 148 | 201 echo "Checking for errors..." |
| 196 | 202 > "${TEMP_PREFIX}-error" |
| 200 | 203 for FILE in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* |
| 159 | 204 do |
| 200 | 205 if [ "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then |
| 193 | 206 echo |
| 200 | 207 echo "Not a valid danbooru file: $(echo ${FILE} | sed -e "${SED_GET_FILENAME}")" |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
208 else |
| 200 | 209 if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME}" | cut -d '.' -f1)" ] |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
210 then |
| 196 | 211 echo "${FILE}" >> "${TEMP_PREFIX}-error" |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
212 echo |
| 200 | 213 echo "Error: $(echo "${FILE}" | sed -e "${SED_GET_FILENAME}")" |
|
187
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
214 fi |
|
efd957294c8c
refactoring. cleanup. etc.
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
186
diff
changeset
|
215 printf "." |
| 148 | 216 fi |
| 217 done | |
| 218 echo | |
| 200 | 219 TOTAL_ERROR=$(echo $(wc -l < "${TEMP_PREFIX}-error")) |
|
178
3f5ee8b2791f
error when restructuring
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
177
diff
changeset
|
220 echo "${TOTAL_ERROR} file(s) error" |
| 170 | 221 echo "Removing error files" |
| 222 if [ "${TOTAL_ERROR}" -eq 0 ]; then | |
| 223 echo "No error file. 0 file removed" | |
| 224 else | |
| 196 | 225 cat "${TEMP_PREFIX}-error" | xargs rm |
| 170 | 226 echo "${TOTAL_ERROR} file(s) removed" |
| 227 fi | |
| 200 | 228 echo "$(Count_Files "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}") file(s) available locally" |
| 148 | 229 |
| 230 echo "Generating list of new files..." | |
| 231 # THE FILES | |
| 159 | 232 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" |
|
156
d3b002fd944e
fix: my attempt at speeding up things failed. reverting back to trusty grep -vf
edhoprima
parents:
155
diff
changeset
|
233 # |
| 200 | 234 find "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${TEMP_PREFIX}-error" - | sed -e "${SED_GET_FILENAME}" > "${TEMP_PREFIX}-ok" |
|
201
30d2fb656029
scrapping grep -vf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
200
diff
changeset
|
235 cat "${TEMP_PREFIX}-list" > "${TEMP_PREFIX}-templist" |
|
30d2fb656029
scrapping grep -vf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
200
diff
changeset
|
236 while read -r IS_OK; do |
|
30d2fb656029
scrapping grep -vf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
200
diff
changeset
|
237 cat "${TEMP_PREFIX}-templist" | grep -v "${IS_OK}" > "${TEMP_PREFIX}-newlist" |
|
30d2fb656029
scrapping grep -vf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
200
diff
changeset
|
238 cat "${TEMP_PREFIX}-newlist" > "${TEMP_PREFIX}-templist" |
|
30d2fb656029
scrapping grep -vf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
200
diff
changeset
|
239 done < "${TEMP_PREFIX}-ok" |
|
30d2fb656029
scrapping grep -vf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
200
diff
changeset
|
240 #cat "${TEMP_PREFIX}-list" | grep -vf "${TEMP_PREFIX}-ok" > "${TEMP_PREFIX}-newlist" |
| 198 | 241 echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded" |
| 159 | 242 |
| 148 | 243 else |
| 159 | 244 if [ "${ISQUICK}" ]; then |
| 152 | 245 echo "quick mode selected. Skipping check" |
| 246 else | |
| 247 echo "Empty local repository" | |
| 248 fi | |
| 200 | 249 cat "${TEMP_PREFIX}-list" > "${TEMP_PREFIX}-newlist" |
| 148 | 250 fi |
| 251 } | |
| 252 | |
| 159 | 253 # start downloading the images |
| 254 Fetch_Images() { | |
| 196 | 255 if [ "$(echo $(wc -l < "${TEMP_PREFIX}-newlist"))" -eq 0 ]; then |
| 148 | 256 echo "No new file" |
| 257 else | |
| 258 echo "Starting wget" | |
|
160
68227a30d0b3
forgot to fix Fetch_Images to reflect new folder naming scheme
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
159
diff
changeset
|
259 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" |
| 196 | 260 wget -e continue=on -bi "${TEMP_PREFIX}-newlist" -o "${TEMP_PREFIX}.log" |
| 148 | 261 fi |
| 262 } | |
| 263 | |
| 159 | 264 # initialize base variables and initial command check |
| 265 Init(){ | |
| 266 # path initialization | |
|
181
d3b7927bdb2b
restructuring and add check if the xml is processed properly
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
180
diff
changeset
|
267 [ "${ADDITIONAL_PATH}" ] && PATH="${ADDITIONAL_PATH}:${PATH}" |
| 159 | 268 export PATH |
|
158
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
269 |
| 159 | 270 # misc variables |
| 166 | 271 ISQUICK= |
| 272 ISNEW= | |
|
158
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
273 |
| 159 | 274 [ $# -lt 2 ] && Err_Help |
| 275 case "$1" in | |
|
174
0948e76a57a1
added help. Bump to 0.1-beta2
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
173
diff
changeset
|
276 check|fetch|quickfetch) |
| 159 | 277 echo "Starting..." |
| 278 JOB="$1" | |
| 279 ;; | |
| 280 *) | |
| 281 Err_Help | |
| 282 ;; | |
| 283 esac | |
| 284 shift | |
| 285 SITE= | |
|
185
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
286 TAGS= |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
287 while [ "${1}" ]; do |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
288 case "$1" in |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
289 -s|--site) |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
290 shift |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
291 SITE="$1" |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
292 ;; |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
293 -nc|--noclean) |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
294 NOCLEAN=1 |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
295 ;; |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
296 *) |
| 186 | 297 if [ "${TAGS}" ]; then |
| 298 TAGS="$1 ${TAGS}" | |
| 299 else | |
| 300 TAGS="$1" | |
| 301 fi | |
|
185
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
302 ;; |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
303 esac |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
304 shift |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
305 done |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
306 [ "${SITE}" ] || SITE="${DEFAULT_SITE}" |
|
6d926d4b3c5a
initial clean system support
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
184
diff
changeset
|
307 [ "${TAGS}" ] || Err_Fatal "No tag specified" |
|
181
d3b7927bdb2b
restructuring and add check if the xml is processed properly
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
180
diff
changeset
|
308 # Get base folder - default, current folder or fallback to ${HOME} |
|
d3b7927bdb2b
restructuring and add check if the xml is processed properly
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
180
diff
changeset
|
309 [ "${BASE_DIR}" ] || BASE_DIR="${PWD}" |
|
d3b7927bdb2b
restructuring and add check if the xml is processed properly
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
180
diff
changeset
|
310 [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}" |
| 193 | 311 [ "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" ] || BASE_DIR="/${BASE_DIR}" |
|
181
d3b7927bdb2b
restructuring and add check if the xml is processed properly
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
180
diff
changeset
|
312 |
| 159 | 313 echo "Tags: ${TAGS}" |
| 314 # slash is not wanted for folder name | |
| 193 | 315 TARGET_DIR=$(echo "${TAGS}" | sed -e 's/\//_/g') |
| 316 SITE_DIR=$(echo "${SITE}" | sed -e 's/\/$//g;s/\//_/g') | |
|
195
652d9e268cee
test migration to printf
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
194
diff
changeset
|
317 TEMP_PREFIX="${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}" |
| 159 | 318 } |
| 148 | 319 |
|
181
d3b7927bdb2b
restructuring and add check if the xml is processed properly
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
180
diff
changeset
|
320 # initialization |
| 159 | 321 Msg_Welcome |
| 322 Init "$@" | |
| 323 Check_Tools | |
| 324 Check_Folders | |
|
158
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
325 |
| 148 | 326 |
| 159 | 327 # let's do the job! |
| 328 case "${JOB}" in | |
|
174
0948e76a57a1
added help. Bump to 0.1-beta2
edhoprima@gmail.com <edhoprima@gmail.com>
parents:
173
diff
changeset
|
329 check) |
| 159 | 330 Generate_Link |
| 331 Check_Files | |
| 148 | 332 ;; |
| 159 | 333 fetch) |
| 334 Generate_Link | |
| 335 Check_Files | |
| 336 Fetch_Images | |
| 337 ;; | |
| 338 quickfetch) | |
| 339 ISNEW=1 | |
| 340 ISQUICK=1 | |
| 341 Generate_Link | |
| 342 Check_Files | |
| 343 Fetch_Images | |
| 148 | 344 ;; |
| 345 esac |