annotate moefetch.sh @ 159:75fe19903b74

Major cleanup
author edhoprima@gmail.com <edhoprima@gmail.com>
date Fri, 05 Jun 2009 15:20:36 +0000
parents cba73f6a96bb
children 68227a30d0b3
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
148
edhoprima
parents:
diff changeset
1 #!/bin/sh
edhoprima
parents:
diff changeset
2
edhoprima
parents:
diff changeset
3 # Copyright (c) 2009, edogawaconan <me@myconan.net>
edhoprima
parents:
diff changeset
4 #
edhoprima
parents:
diff changeset
5 # Permission to use, copy, modify, and/or distribute this software for any
edhoprima
parents:
diff changeset
6 # purpose with or without fee is hereby granted, provided that the above
edhoprima
parents:
diff changeset
7 # copyright notice and this permission notice appear in all copies.
edhoprima
parents:
diff changeset
8 #
edhoprima
parents:
diff changeset
9 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
edhoprima
parents:
diff changeset
10 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
edhoprima
parents:
diff changeset
11 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
edhoprima
parents:
diff changeset
12 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
edhoprima
parents:
diff changeset
13 # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
edhoprima
parents:
diff changeset
14 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
edhoprima
parents:
diff changeset
15 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
16 #
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
17 # Lots of bugs here. Use with care
148
edhoprima
parents:
diff changeset
18 # USE WITH CARE
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
19 #
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
20 # what it does: fetch every picture that has the specified TAGS.
148
edhoprima
parents:
diff changeset
21 # requirement: wget, libxslt, md5sum (or md5)
edhoprima
parents:
diff changeset
22
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
23 # program additional paths for: cut, sed, wc, MD5(sum), wget, xsltproc, grep
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
24 ADDITIONAL_PATH=
148
edhoprima
parents:
diff changeset
25
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
26 # custom md5 path with arguments, expected output: <32digit md5><space(s)><filename>
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
27 # Leave empty for "md5sum" (Linux, Solaris), "md5 -r" (*BSD)
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
28 MD5=
148
edhoprima
parents:
diff changeset
29
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
30 # default server address. Danbooru only! I do not take responsibility of stupidity.
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
31 DEFAULT_SITE="moe.imouto.org"
148
edhoprima
parents:
diff changeset
32
edhoprima
parents:
diff changeset
33 # base directory. make sure it's writeable. I do not take responsibility if you don't own the folder and files as no check is done for this one.
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
34 # Structure is ${BASE_DIR}/<TAGS>
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
35 # Absolute path only.
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
36 # Leave empty to use whatever folder you're running this at
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
37 BASE_DIR=""
148
edhoprima
parents:
diff changeset
38
edhoprima
parents:
diff changeset
39 # not user modifiable from here
edhoprima
parents:
diff changeset
40
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
41 # useless welcome message. Also version
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
42 Msg_Welcome() {
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
43 MOEFETCHVERSION="0.1-beta"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
44 cat <<EOF
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
45 moefetch ${MOEFETCHVERSION}
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
46 Copyright (c) 2009 edogawaconan <me@myconan.net>
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
47
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
48 EOF
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
49 }
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
50
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
51 # fatal error handler
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
52 Err_Fatal() {
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
53 echo "Fatal error: ${1}"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
54 exit 1
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
55 }
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
56
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
57 # help message
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
58 Err_Help() {
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
59 cat <<EOF
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
60 Usage: moefetch (quick)fetch|status <TAGS>
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
61 EOF
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
62 exit 0
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
63 }
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
64
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
65 # generate link by transforming xml
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
66 Generate_Link() {
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
67 cd "${BASE_DIR}/temp"
148
edhoprima
parents:
diff changeset
68 echo
edhoprima
parents:
diff changeset
69 echo "Fetching xml file"
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
70 wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=100000" -O "${SITE_DIR}-${TARGET_DIR}-xml" -e continue=off
148
edhoprima
parents:
diff changeset
71 echo "Processing XML file..."
edhoprima
parents:
diff changeset
72 # xslt evilry
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
73 xsltproc - "${TARGET_DIR}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([jp][pn]g\)/\1\2.\3/g' | grep ^http > "${TARGET_DIR}-list"
148
edhoprima
parents:
diff changeset
74 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
edhoprima
parents:
diff changeset
75 <xsl:output method="xml" indent="yes"/>
edhoprima
parents:
diff changeset
76 <xsl:template match="post">
edhoprima
parents:
diff changeset
77 <xsl:value-of select="@file_url" />
edhoprima
parents:
diff changeset
78 </xsl:template>
edhoprima
parents:
diff changeset
79 </xsl:stylesheet>
edhoprima
parents:
diff changeset
80 EOF
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
81 echo "`echo \`wc -l < \"${SITE_DIR}-${TARGET_DIR}-list\" \`` file(s) available on server"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
82 #output file: ${TARGET_DIR}-list
148
edhoprima
parents:
diff changeset
83 }
edhoprima
parents:
diff changeset
84
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
85 # check tools availability
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
86 Check_Tools() {
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
87 # verify all programs required do indeed exist
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
88 #MD5
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
89 if [ ! "${MD5}" ]; then
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
90 case `uname` in
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
91 *BSD) MD5="md5 -r";;
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
92 Linux|SunOS) MD5="md5sum";;
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
93 *) Fatal_Err "No known md5 tool for this platform. Please specify manually"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
94 esac
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
95 fi
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
96 MD5_COMMAND=`echo ${MD5} | cut -d' ' -f1`
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
97 # basic tools
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
98 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep ${MD5_COMMAND}"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
99 for COMMAND in ${COMMANDS}
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
100 do
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
101 COMMAND_CHECK=`command -v "${COMMAND}"`
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
102 [ "${COMMAND_CHECK}" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
103 done
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
104
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
105 # grep checking
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
106 # originally created for workaround on solaris
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
107 #if [ `uname` = "SunOS" ]; then
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
108 FAIL=""
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
109 echo "blah" > superrandomtestfile
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
110 echo "blah" > superrandomtestfile.2
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
111 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
112 rm -f superrandomtestfile superrandomtestfile.2
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
113 [ "${FAIL}" ] && Err_Fatal "Your grep is not compatible. Please install or set path of correct grep"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
114 }
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
115
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
116 # verify required folders exist and writeable
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
117 Check_Folders(){
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
118 [ -O "${BASE_DIR}" ] || Err_Fatal "You don't own ${BASE_DIR}. Please fix ${BASE_DIR}."
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
119 for FOLDER in temp trash deleted ${TARGET_DIR}
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
120 do
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
121 if [ ! -d "${BASE_DIR}/${FOLDER}" ]; then
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
122 mkdir "${BASE_DIR}/${FOLDER}" || Err_Fatal "${FOLDER} folder creation failed"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
123 fi
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
124 if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
125 echo "You don't own the ${BASE_DIR}/{$FOLDER}, applying globally writeable permission on it"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
126 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
127 fi
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
128 done
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
129 [ `echo \`ls "${BASE_DIR}/${TARGET_DIR}" | wc -l\`` -eq 0 ] && ISNEW=1
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
130 # let's move to workdir
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
131 cd "${BASE_DIR}/temp"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
132 for i in error ok list newlist; do
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
133 touch "${TARGET_DIR}-${i}" || Fatal_Err "Error creating ${TARGET_DIR}-${i}. This shouldn't happen"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
134 done
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
135 #
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
136 }
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
137
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
138 # check files correctness
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
139 Check_Files() {
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
140 if [ "$ISNEW" -ne 1 ]; then
148
edhoprima
parents:
diff changeset
141 echo "Checking for errors..."
edhoprima
parents:
diff changeset
142 # THE FILES
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
143
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
144 # current dir: ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
145 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
146 for TRASH in `ls * | sed -e 's/\([0-9a-f]\{32\}.*\)//g' | grep -v ^$`
148
edhoprima
parents:
diff changeset
147 do
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
148 mv -f "${TRASH}" "${BASE_DIR}/trash"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
149 echo "Moved ${TRASH} to ${BASE_DIR}/trash"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
150 done
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
151 printf "" > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
152 for FILE in *
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
153 do
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
154 if [ `${MD5} "${FILE}" | cut -d ' ' -f1 -` != `echo "${FILE}" | cut -d '.' -f1` ]
148
edhoprima
parents:
diff changeset
155 then
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
156 echo
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
157 echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
158 echo "Error: ${FILE}"
148
edhoprima
parents:
diff changeset
159 fi
edhoprima
parents:
diff changeset
160 printf "."
edhoprima
parents:
diff changeset
161 done
edhoprima
parents:
diff changeset
162 echo
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
163
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
164 # current dir: ${BASE_DIR}/temp
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
165 cd ${BASE_DIR}/temp
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
166 TOTAL_ERROR=`echo \`wc -l < "${SITE_DIR}-${TARGET_DIR}-error"\``
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
167 echo "${TOTAL_ERROR} file(s) error"
148
edhoprima
parents:
diff changeset
168
edhoprima
parents:
diff changeset
169 echo "Generating list of new files..."
edhoprima
parents:
diff changeset
170 # THE FILES
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
171 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok"
156
d3b002fd944e fix: my attempt at speeding up things failed. reverting back to trusty grep -vf
edhoprima
parents: 155
diff changeset
172 #
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
173 ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
174 cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
175 echo "`echo \`wc -l < \"${SITE_DIR}-${TARGET_DIR}-newlist\"\`` file(s) to be downloaded"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
176
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
177 # back to target dir
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
178 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}"
148
edhoprima
parents:
diff changeset
179
edhoprima
parents:
diff changeset
180 echo "Removing error files"
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
181 if [ "${TOTAL_ERROR}" -eq 0 ]; then
153
7b07448e46f1 fix: when there is no error file
edhoprima
parents: 152
diff changeset
182 echo "No error file. 0 file removed"
156
d3b002fd944e fix: my attempt at speeding up things failed. reverting back to trusty grep -vf
edhoprima
parents: 155
diff changeset
183 else
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
184 cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
185 echo "${TOTAL_ERROR} file(s) removed"
148
edhoprima
parents:
diff changeset
186 fi
edhoprima
parents:
diff changeset
187 echo "`echo \`ls | wc -l\`` file(s) available locally"
155
78ef726d3845 fix: whoops. Forgot to move back to workdir after
edhoprima
parents: 154
diff changeset
188 cd ../temp
148
edhoprima
parents:
diff changeset
189 else
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
190 if [ "${ISQUICK}" ]; then
152
67df02877319 added quickfetch for skipping file checking
edhoprima
parents: 151
diff changeset
191 echo "quick mode selected. Skipping check"
67df02877319 added quickfetch for skipping file checking
edhoprima
parents: 151
diff changeset
192 else
67df02877319 added quickfetch for skipping file checking
edhoprima
parents: 151
diff changeset
193 echo "Empty local repository"
67df02877319 added quickfetch for skipping file checking
edhoprima
parents: 151
diff changeset
194 fi
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
195 cd "${BASE_DIR}/temp"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
196 cat "${SITE_DIR}-${TARGET_DIR}-list" > "${TARGET_DIR}-newlist"
148
edhoprima
parents:
diff changeset
197 fi
edhoprima
parents:
diff changeset
198 }
edhoprima
parents:
diff changeset
199
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
200 # start downloading the images
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
201 Fetch_Images() {
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
202 if [ `echo \`wc -l < "${TARGET_DIR}-newlist"\`` -eq 0 ]; then
148
edhoprima
parents:
diff changeset
203 echo "No new file"
edhoprima
parents:
diff changeset
204 else
edhoprima
parents:
diff changeset
205 echo "Starting wget"
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
206 cd "../${TARGET_DIR}"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
207 wget -e continue=on -bi "../temp/${TARGET_DIR}-newlist" -o "../temp/${TARGET_DIR}.log"
148
edhoprima
parents:
diff changeset
208 fi
edhoprima
parents:
diff changeset
209 }
edhoprima
parents:
diff changeset
210
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
211 # initialize base variables and initial command check
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
212 Init(){
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
213 # Get base folder - current folder or fallback to ${HOME}
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
214 [ "${BASE_DIR}" ] || BASE_DIR="${PWD}"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
215 [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
216 [ "`echo ${BASE_DIR} | cut -c1 | grep \/`" ] || BASE_DIR="/${BASE_DIR}"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
217 # path initialization
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
218 [ "${ADDITIONAL_PATH}" ] && PATH=${ADDITIONAL_PATH}:${PATH}
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
219 export PATH
158
cba73f6a96bb grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents: 157
diff changeset
220
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
221 # misc variables
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
222 ISQUICK=""
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
223 ISNEW=""
158
cba73f6a96bb grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents: 157
diff changeset
224
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
225 [ $# -lt 2 ] && Err_Help
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
226 case "$1" in
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
227 status|fetch|quickfetch)
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
228 echo "Starting..."
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
229 JOB="$1"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
230 ;;
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
231 *)
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
232 Err_Help
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
233 ;;
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
234 esac
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
235 shift
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
236 SITE=
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
237 case "$1" in
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
238 -s|--site)
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
239 shift
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
240 SITE="$1"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
241 ;;
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
242 *)
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
243 SITE=DEFAULT_SITE
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
244 ;;
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
245 esac
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
246 shift
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
247 TAGS="$@"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
248 echo "Tags: ${TAGS}"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
249 # slash is not wanted for folder name
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
250 TARGET_DIR=`echo "${TAGS}" | sed -e 's/\//_/g'`
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
251 SITE_DIR=`echo "${SITE}" | sed -e 's/\//_/g'`
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
252 }
148
edhoprima
parents:
diff changeset
253
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
254 Msg_Welcome
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
255 Init "$@"
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
256 Check_Tools
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
257 Check_Folders
158
cba73f6a96bb grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents: 157
diff changeset
258
148
edhoprima
parents:
diff changeset
259
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
260 # let's do the job!
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
261 case "${JOB}" in
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
262 status)
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
263 Generate_Link
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
264 Check_Files
148
edhoprima
parents:
diff changeset
265 ;;
159
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
266 fetch)
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
267 Generate_Link
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
268 Check_Files
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
269 Fetch_Images
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
270 ;;
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
271 quickfetch)
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
272 ISNEW=1
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
273 ISQUICK=1
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
274 Generate_Link
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
275 Check_Files
75fe19903b74 Major cleanup
edhoprima@gmail.com <edhoprima@gmail.com>
parents: 158
diff changeset
276 Fetch_Images
148
edhoprima
parents:
diff changeset
277 ;;
edhoprima
parents:
diff changeset
278 esac