Mercurial > ec-dotfiles

comparison moefetch.sh @ 196:4d28f3a957ee

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cleanup~
author edhoprima@gmail.com <edhoprima@gmail.com>
date Mon, 29 Jun 2009 12:22:59 +0000
parents 652d9e268cee
children a17a2f366680
comparison
equal deleted inserted replaced
195:652d9e268cee 196:4d28f3a957ee
36 # Leave empty to use whatever folder you're running this at 36 # Leave empty to use whatever folder you're running this at
37 BASE_DIR= 37 BASE_DIR=
38 38
39 # not user modifiable from here 39 # not user modifiable from here
40 40
41 SED_GET_FILENAME="s/.*\/\([^\/]*\)/\1/g"
42 SED_IS_MD5_FILE="s/\([0-9a-f]\{32\}\..*\)//g"
41 43
42 ### TODO: 44 ### TODO:
43 ### - replace `...` with $(..) 45 ### - replace `...` with $(..) (DONE)
44 ### - sanity validator 46 ### - sanity validator
45 ### - unified repository to save bandwidth 47 ### - unified repository to save bandwidth
46 ### - bug stomping 48 ### - bug stomping
47 ### - sanity checking 49 ### - sanity checking
48 ### - replace grep -vf with POSIX compatible command 50 ### - replace grep -vf with POSIX compatible command
58 EOF 60 EOF
59 } 61 }
60 62
61 # fatal error handler 63 # fatal error handler
62 Err_Fatal() { 64 Err_Fatal() {
63 printf "\nFatal error: ${1}" 65 printf "\nFatal error: ${1}\n"
64 exit 1 66 exit 1
65 } 67 }
66 68
67 # help message 69 # help message
68 Err_Help() { 70 Err_Help() {
143 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it" 145 echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it"
144 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen" 146 chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen"
145 fi 147 fi
146 done 148 done
147 [ "$(echo $(ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l))" -eq 0 ] && ISNEW=1 149 [ "$(echo $(ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l))" -eq 0 ] && ISNEW=1
148 # let's move to workdir
149 cd "${BASE_DIR}/temp"
150 for i in error ok list newlist; do 150 for i in error ok list newlist; do
151 touch "${SITE_DIR}-${TARGET_DIR}-${i}" || Fatal_Err "Error creating ${TARGET_DIR}-${i}. This shouldn't happen" 151 touch "${TEMP_PREFIX}-${i}" || Fatal_Err "Error creating ${TEMP_PREFIX}-${i}. This shouldn't happen"
152 done 152 done
153 # 153 #
154 } 154 }
155 155
156 # Do some cleanup 156 # Do some cleanup
166 do 166 do
167 ISTRASH= 167 ISTRASH=
168 if [ -d "${TRASH}" ]; then 168 if [ -d "${TRASH}" ]; then
169 ISTRASH=1 169 ISTRASH=1
170 else 170 else
171 if [ "$(echo "${TRASH}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then 171 if [ "$(echo "${TRASH}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then
172 ISTRASH=1 172 ISTRASH=1
173 else 173 else
174 [ "$(cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list" | sed -e 's/.*\/\([^\/]*\)/\1/g' | grep $(echo "${TRASH}" | sed -e 's/.*\/\([^\/]*\)/\1/g'))" ] || ISTRASH=1 174 [ "$(cat "${TEMP_PREFIX}-list" | sed -e "${SED_GET_FILENAME}" | grep $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}"))" ] || ISTRASH=1
175 fi 175 fi
176 fi 176 fi
177 if [ "${ISTRASH}" ]; then 177 if [ "${ISTRASH}" ]; then
178 mv -f "${TRASH}" "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Error deleting files" 178 mv -f "${TRASH}" "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Error deleting files"
179 echo "Moved ${TRASH} to ${BASE_DIR}/trash/${TRASH_DIR}" 179 echo "Moved $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}") to ${BASE_DIR}/trash/${TRASH_DIR}"
180 fi 180 fi
181 done 181 done
182 rmdir "${BASE_DIR}/trash/${TRASH_DIR}" 2>/dev/null 182 rmdir "${BASE_DIR}/trash/${TRASH_DIR}" 2>/dev/null
183 183
184 } 184 }
187 Check_Files() { 187 Check_Files() {
188 if [ ! "${ISNEW}" ]; then 188 if [ ! "${ISNEW}" ]; then
189 [ "${NOCLEAN}" ] || Cleanup_Repository 189 [ "${NOCLEAN}" ] || Cleanup_Repository
190 echo "Checking for errors..." 190 echo "Checking for errors..."
191 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" 191 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}"
192 > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" 192 > "${TEMP_PREFIX}-error"
193 for FILE in * 193 for FILE in *
194 do 194 do
195 if [ "$(echo "${FILE}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then 195 if [ "$(echo "${FILE}" | sed -e '${SED_IS_MD5_FILE}' | grep -v ^$)" ]; then
196 echo 196 echo
197 echo "Not a valid danbooru file: ${FILE}" 197 echo "Not a valid danbooru file: ${FILE}"
198 else 198 else
199 if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | cut -d '.' -f1)" ] 199 if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | cut -d '.' -f1)" ]
200 then 200 then
201 echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" 201 echo "${FILE}" >> "${TEMP_PREFIX}-error"
202 echo 202 echo
203 echo "Error: ${FILE}" 203 echo "Error: ${FILE}"
204 fi 204 fi
205 printf "." 205 printf "."
206 fi 206 fi
207 done 207 done
208 echo 208 echo
209 TOTAL_ERROR=$(echo $(wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error")) 209 TOTAL_ERROR=$(echo "$(wc -l < "${TEMP_PREFIX}-error")")
210 echo "${TOTAL_ERROR} file(s) error" 210 echo "${TOTAL_ERROR} file(s) error"
211 echo "Removing error files" 211 echo "Removing error files"
212 if [ "${TOTAL_ERROR}" -eq 0 ]; then 212 if [ "${TOTAL_ERROR}" -eq 0 ]; then
213 echo "No error file. 0 file removed" 213 echo "No error file. 0 file removed"
214 else 214 else
215 cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm 215 cat "${TEMP_PREFIX}-error" | xargs rm
216 echo "${TOTAL_ERROR} file(s) removed" 216 echo "${TOTAL_ERROR} file(s) removed"
217 fi 217 fi
218 echo "$(echo $(ls | wc -l)) file(s) available locally" 218 echo "$(echo $(ls | wc -l)) file(s) available locally"
219 219
220 # current dir: ${BASE_DIR}/temp 220 # current dir: ${BASE_DIR}/temp
222 222
223 echo "Generating list of new files..." 223 echo "Generating list of new files..."
224 # THE FILES 224 # THE FILES
225 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" 225 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok"
226 # 226 #
227 ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok" 227 find "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${TEMP_PREFIX}-error" - > "${TEMP_PREFIX}-ok"
228 cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist" 228 cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist"
229 echo "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist")) file(s) to be downloaded" 229 echo "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist")) file(s) to be downloaded"
230 230
231 else 231 else
232 if [ "${ISQUICK}" ]; then 232 if [ "${ISQUICK}" ]; then
239 fi 239 fi
240 } 240 }
241 241
242 # start downloading the images 242 # start downloading the images
243 Fetch_Images() { 243 Fetch_Images() {
244 cd "${BASE_DIR}/temp" 244 if [ "$(echo $(wc -l < "${TEMP_PREFIX}-newlist"))" -eq 0 ]; then
245 if [ "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"))" -eq 0 ]; then
246 echo "No new file" 245 echo "No new file"
247 else 246 else
248 echo "Starting wget" 247 echo "Starting wget"
249 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" 248 cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}"
250 wget -e continue=on -bi "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-newlist" -o "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}.log" 249 wget -e continue=on -bi "${TEMP_PREFIX}-newlist" -o "${TEMP_PREFIX}.log"
251 fi 250 fi
252 } 251 }
253 252
254 # initialize base variables and initial command check 253 # initialize base variables and initial command check
255 Init(){ 254 Init(){