Mercurial > ec-dotfiles

comparison moefetch.sh @ 202:3dbb5a6678f9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
egrep is not grep
author edhoprima@gmail.com <edhoprima@gmail.com>
date Mon, 29 Jun 2009 16:13:22 +0000
parents 30d2fb656029
children 94a585031e3b
comparison
equal deleted inserted replaced
201:30d2fb656029 202:3dbb5a6678f9
45 ### - replace `...` with $(..) (DONE) 45 ### - replace `...` with $(..) (DONE)
46 ### - sanity validator 46 ### - sanity validator
47 ### - unified repository to save bandwidth 47 ### - unified repository to save bandwidth
48 ### - bug stomping 48 ### - bug stomping
49 ### - sanity checking 49 ### - sanity checking
50 ### - replace grep -vf with POSIX compatible command
51 ### WILL BE FOR 0.3 50 ### WILL BE FOR 0.3
52 51
53 # useless welcome message. Also version 52 # useless welcome message. Also version
54 Msg_Welcome() { 53 Msg_Welcome() {
55 MOEFETCHVERSION="0.2-beta1" 54 MOEFETCHVERSION="0.2-beta1"
88 printf "\nFetching xml file\n" 87 printf "\nFetching xml file\n"
89 wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=100000" -O "${TEMP_PREFIX}-xml" -e continue=off 88 wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=100000" -O "${TEMP_PREFIX}-xml" -e continue=off
90 printf "Processing XML file..." 89 printf "Processing XML file..."
91 # xslt evilry 90 # xslt evilry
92 > "${TEMP_PREFIX}-list" 91 > "${TEMP_PREFIX}-list"
93 xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-list" 2>/dev/null 92 xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | egrep ^http > "${TEMP_PREFIX}-list" 2>/dev/null
94 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> 93 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
95 <xsl:output method="xml" indent="yes"/> 94 <xsl:output method="xml" indent="yes"/>
96 <xsl:template match="post"> 95 <xsl:template match="post">
97 <xsl:value-of select="@file_url" /> 96 <xsl:value-of select="@file_url" />
98 </xsl:template> 97 </xsl:template>
126 *) Fatal_Err "No known md5 tool for this platform. Please specify manually" 125 *) Fatal_Err "No known md5 tool for this platform. Please specify manually"
127 esac 126 esac
128 fi 127 fi
129 MD5_COMMAND=$(echo ${MD5} | cut -d' ' -f1) 128 MD5_COMMAND=$(echo ${MD5} | cut -d' ' -f1)
130 # basic tools 129 # basic tools
131 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}" 130 COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm egrep grep date ${MD5_COMMAND}"
132 for COMMAND in ${COMMANDS} 131 for COMMAND in ${COMMANDS}
133 do 132 do
134 [ "$(command -v "${COMMAND}")" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}" 133 [ "$(command -v "${COMMAND}")" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}"
135 done 134 done
136
137 # grep checking
138 # originally created for workaround on solaris
139 #if [ `uname` = "SunOS" ]; then
140 FAIL=
141 echo "blah" > superrandomtestfile
142 echo "blah" > superrandomtestfile.2
143 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1
144 rm -f superrandomtestfile superrandomtestfile.2
145 [ "${FAIL}" ] && Err_Fatal "Your grep is not compatible. Please install or set path of correct grep"
146 } 135 }
147 136
148 # verify required folders exist and writeable 137 # verify required folders exist and writeable
149 Check_Folders(){ 138 Check_Folders(){
150 [ -O "${BASE_DIR}" ] || Err_Fatal "You don't own ${BASE_DIR}. Please fix ${BASE_DIR}." 139 [ -O "${BASE_DIR}" ] || Err_Fatal "You don't own ${BASE_DIR}. Please fix ${BASE_DIR}."
177 do 166 do
178 ISTRASH= 167 ISTRASH=
179 if [ -d "${TRASH}" ]; then 168 if [ -d "${TRASH}" ]; then
180 ISTRASH=1 169 ISTRASH=1
181 else 170 else
182 if [ "$(echo "${TRASH}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then 171 if [ "$(echo "${TRASH}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | egrep -v ^$)" ]; then
183 ISTRASH=1 172 ISTRASH=1
184 else 173 else
185 [ "$(cat "${TEMP_PREFIX}-list" | sed -e "${SED_GET_FILENAME}" | grep $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}"))" ] || ISTRASH=1 174 [ "$(cat "${TEMP_PREFIX}-list" | sed -e "${SED_GET_FILENAME}" | grep $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}"))" ] || ISTRASH=1
186 fi 175 fi
187 fi 176 fi
200 [ "${NOCLEAN}" ] || Cleanup_Repository 189 [ "${NOCLEAN}" ] || Cleanup_Repository
201 echo "Checking for errors..." 190 echo "Checking for errors..."
202 > "${TEMP_PREFIX}-error" 191 > "${TEMP_PREFIX}-error"
203 for FILE in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* 192 for FILE in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"*
204 do 193 do
205 if [ "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then 194 if [ "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | egrep -v ^$)" ]; then
206 echo 195 echo
207 echo "Not a valid danbooru file: $(echo ${FILE} | sed -e "${SED_GET_FILENAME}")" 196 echo "Not a valid danbooru file: $(echo ${FILE} | sed -e "${SED_GET_FILENAME}")"
208 else 197 else
209 if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME}" | cut -d '.' -f1)" ] 198 if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME}" | cut -d '.' -f1)" ]
210 then 199 then
231 # THE FILES 220 # THE FILES
232 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" 221 #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok"
233 # 222 #
234 find "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${TEMP_PREFIX}-error" - | sed -e "${SED_GET_FILENAME}" > "${TEMP_PREFIX}-ok" 223 find "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${TEMP_PREFIX}-error" - | sed -e "${SED_GET_FILENAME}" > "${TEMP_PREFIX}-ok"
235 cat "${TEMP_PREFIX}-list" > "${TEMP_PREFIX}-templist" 224 cat "${TEMP_PREFIX}-list" > "${TEMP_PREFIX}-templist"
236 while read -r IS_OK; do 225 #while read -r IS_OK; do
237 cat "${TEMP_PREFIX}-templist" | grep -v "${IS_OK}" > "${TEMP_PREFIX}-newlist" 226 # cat "${TEMP_PREFIX}-templist" | grep -v "${IS_OK}" > "${TEMP_PREFIX}-newlist"
238 cat "${TEMP_PREFIX}-newlist" > "${TEMP_PREFIX}-templist" 227 # cat "${TEMP_PREFIX}-newlist" > "${TEMP_PREFIX}-templist"
239 done < "${TEMP_PREFIX}-ok" 228 #done < "${TEMP_PREFIX}-ok"
240 #cat "${TEMP_PREFIX}-list" | grep -vf "${TEMP_PREFIX}-ok" > "${TEMP_PREFIX}-newlist" 229 cat "${TEMP_PREFIX}-list" | egrep -vf "${TEMP_PREFIX}-ok" > "${TEMP_PREFIX}-newlist"
241 echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded" 230 echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded"
242 231
243 else 232 else
244 if [ "${ISQUICK}" ]; then 233 if [ "${ISQUICK}" ]; then
245 echo "quick mode selected. Skipping check" 234 echo "quick mode selected. Skipping check"