Mercurial > ec-dotfiles
annotate moefetch.sh @ 158:cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
| author | edhoprima |
|---|---|
| date | Tue, 05 May 2009 17:56:54 +0000 |
| parents | ffca564daefa |
| children | 75fe19903b74 |
| rev | line source |
|---|---|
| 148 | 1 #!/bin/sh |
| 2 | |
| 3 # Copyright (c) 2009, edogawaconan <me@myconan.net> | |
| 4 # | |
| 5 # Permission to use, copy, modify, and/or distribute this software for any | |
| 6 # purpose with or without fee is hereby granted, provided that the above | |
| 7 # copyright notice and this permission notice appear in all copies. | |
| 8 # | |
| 9 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES | |
| 10 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF | |
| 11 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR | |
| 12 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES | |
| 13 # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN | |
| 14 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF | |
| 15 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | |
| 16 | |
| 17 # Version 0.1-alpha1. Lots of bugs here. Use with care | |
| 18 # USE WITH CARE | |
| 19 | |
| 20 # what it does: fetch every picture that has the specified tags. | |
| 21 | |
| 22 # requirement: wget, libxslt, md5sum (or md5) | |
| 23 | |
| 24 # configs | |
|
158
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
25 # program additional paths for: cut, sed, wc, md5(sum), wget, xsltproc, grep |
| 148 | 26 extrapath= |
| 27 | |
| 28 # md5 calculation, expected output: <32digit md5><space(s)><filename> | |
| 29 # gnu: "md5sum", bsd: "md5 -r" | |
| 30 md5="md5 -r" | |
| 31 | |
| 32 # server address. Danbooru only! I do not take responsibility of stupidity. | |
| 33 site="moe.imouto.org" | |
| 34 | |
| 35 # base directory. make sure it's writeable. I do not take responsibility if you don't own the folder and files as no check is done for this one. | |
| 36 # Structure is $basedir/<tags> | |
| 37 basedir="/home/ifail/test" | |
| 38 | |
| 39 # not user modifiable from here | |
| 40 | |
| 41 GENERATE() | |
| 42 { | |
| 43 echo | |
| 44 echo "Fetching xml file" | |
| 45 wget "http://$site/post/index.xml?tags=$tags&offset=0&limit=100000" -O "$outdir-xml" -e continue=off | |
| 46 echo "Processing XML file..." | |
| 47 # xslt evilry | |
| 48 xsltproc - "$outdir-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([jp][pn]g\)/\1\2.\3/g' | grep ^http > "$outdir-list" | |
| 49 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> | |
| 50 <xsl:output method="xml" indent="yes"/> | |
| 51 <xsl:template match="post"> | |
| 52 <xsl:value-of select="@file_url" /> | |
| 53 </xsl:template> | |
| 54 </xsl:stylesheet> | |
| 55 EOF | |
| 56 echo "`echo \`wc -l < \"$outdir-list\" \`` file(s) available on server" | |
| 57 #output file: $outdir-list | |
| 58 } | |
| 59 | |
| 60 CHECK() | |
| 61 { | |
| 62 if [ "$ISNEW" -ne 1 ];then | |
| 63 echo "Checking for errors..." | |
| 64 # THE FILES | |
| 154 | 65 printf "" > "$outdir-error" |
| 148 | 66 cd "../$outdir" |
| 67 for file in `ls` | |
| 68 do | |
| 69 if [ `$md5 "$file" | cut -d ' ' -f1 -` != `echo "$file" | cut -d '.' -f1` ] | |
| 70 then | |
| 71 echo "$file" >> "../temp/$outdir-error" | |
| 72 echo "Error: $file" | |
| 73 fi | |
| 74 printf "." | |
| 75 done | |
| 76 echo | |
| 77 cd ../temp | |
| 153 | 78 totalerr=`echo \`wc -l < "$outdir-error"\`` |
| 148 | 79 echo "$totalerr file(s) error" |
| 80 | |
| 81 echo "Generating list of new files..." | |
| 82 # THE FILES | |
| 83 #ls "../$outdir" | grep -vf "$outdir-error" > "$outdir-ok" | |
|
156
d3b002fd944e
fix: my attempt at speeding up things failed. reverting back to trusty grep -vf
edhoprima
parents:
155
diff
changeset
|
84 # |
| 157 | 85 ls "../$outdir" | comm -1 -3 "$outdir-error" - > "$outdir-ok" |
|
156
d3b002fd944e
fix: my attempt at speeding up things failed. reverting back to trusty grep -vf
edhoprima
parents:
155
diff
changeset
|
86 cat "$outdir-list" | grep -vf "$outdir-ok" > "$outdir-newlist" |
| 148 | 87 echo "`echo \`wc -l < \"$outdir-newlist\"\`` file(s) to be downloaded" |
| 88 cd "../$outdir" | |
| 89 | |
| 90 echo "Removing error files" | |
| 153 | 91 if [ $totalerr -eq 0 ]; then |
| 92 echo "No error file. 0 file removed" | |
|
156
d3b002fd944e
fix: my attempt at speeding up things failed. reverting back to trusty grep -vf
edhoprima
parents:
155
diff
changeset
|
93 else |
| 148 | 94 cat "../temp/$outdir-error" | xargs rm |
|
156
d3b002fd944e
fix: my attempt at speeding up things failed. reverting back to trusty grep -vf
edhoprima
parents:
155
diff
changeset
|
95 echo "$totalerr file(s) removed" |
| 148 | 96 fi |
| 97 echo "`echo \`ls | wc -l\`` file(s) available locally" | |
|
155
78ef726d3845
fix: whoops. Forgot to move back to workdir after
edhoprima
parents:
154
diff
changeset
|
98 cd ../temp |
| 148 | 99 else |
| 152 | 100 if [ $ISQUICK -eq 1 ]; then |
| 101 echo "quick mode selected. Skipping check" | |
| 102 else | |
| 103 echo "Empty local repository" | |
| 104 fi | |
| 148 | 105 cat "$outdir-list" > "$outdir-newlist" |
| 106 fi | |
| 107 } | |
| 108 | |
| 109 FETCH() | |
| 110 { | |
|
155
78ef726d3845
fix: whoops. Forgot to move back to workdir after
edhoprima
parents:
154
diff
changeset
|
111 if [ `echo \`wc -l < "$outdir-newlist"\`` -eq 0 ]; then |
| 148 | 112 echo "No new file" |
| 113 else | |
| 114 echo "Starting wget" | |
| 115 cd "../$outdir" | |
| 157 | 116 wget -e continue=on -bi "../temp/$outdir-newlist" -o "../temp/$outdir.log" |
| 148 | 117 fi |
| 118 } | |
| 119 | |
| 120 | |
| 121 # path initialization | |
|
158
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
122 export PATH=$extrapath:${PATH} |
| 148 | 123 |
| 124 # verify all programs required do indeed exist | |
|
158
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
125 |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
126 |
| 148 | 127 # basic tools |
|
158
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
128 commands="cut sed wc wget xsltproc xargs rm mkdir chown comm grep" |
| 148 | 129 cmderr=" " |
| 130 for cmd in $commands | |
| 131 do | |
| 132 command -v "$cmd" >/dev/null || cmderr="$cmderr $cmd" | |
| 133 done | |
| 134 if [ x"$cmderr" != x" " ]; then | |
| 135 echo "$cmderr doesn't exist in $PATH" | |
| 136 exit 1 | |
| 137 fi | |
| 138 #md5 | |
| 139 md5base=`echo $md5 | cut -d ' ' -f 1 -` | |
| 140 if [ x`command -v "$md5base" >/dev/null || echo x` != "x" ]; then | |
| 141 echo "$md5base doesn't exist in $PATH" | |
| 142 exit 1 | |
| 143 fi | |
| 150 | 144 mdtest= |
| 151 | 145 if [ `echo test | $md5 | cut -d ' ' -f 1 -` != "d8e8fca2dc0f896fd7cb4cb0031ba249" ]; then |
| 148 | 146 echo "$md5 doesn't produce wanted output" |
| 147 exit 1 | |
| 148 fi | |
| 149 | |
|
158
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
150 # grep checking |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
151 # originally created for workaround on solaris |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
152 #if [ `uname` = "SunOS" ]; then |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
153 FAIL=0 |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
154 echo "blah" > superrandomtestfile |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
155 echo "blah" > superrandomtestfile.2 |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
156 grep -f superrandomtestfile.2 superrandomtestfile > /dev/null 2>&1 || FAIL=1 |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
157 rm superrandomtestfile superrandomtestfile.2 superrandomtestfile.3 > /dev/null 2>&1 |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
158 if [ $FAIL = 1 ]; then |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
159 echo "Your grep is not compatible. Please install or set path of correct grep" |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
160 exit 1 |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
161 fi |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
162 #fi |
|
cba73f6a96bb
grep check. OpenSolaris' default grep doesn't support -f
edhoprima
parents:
157
diff
changeset
|
163 |
| 148 | 164 |
| 165 # all green (part 1)! let's go (until we check the tag) | |
| 166 | |
| 167 # initialization | |
| 168 # are we really doing it? | |
| 152 | 169 HELP="Usage: moefetch (quick)fetch|status <tags>" |
| 170 | |
| 148 | 171 if [ $# -lt 2 ]; then |
| 152 | 172 echo "$HELP" |
| 148 | 173 exit 1 |
| 174 fi | |
| 175 | |
| 176 case "$1" in | |
| 152 | 177 status|fetch|quickfetch) |
| 148 | 178 echo "Starting..." |
| 179 ;; | |
| 180 *) | |
| 152 | 181 echo "$HELP" |
| 148 | 182 exit 1 |
| 183 ;; | |
| 184 esac | |
| 185 | |
| 186 # we did it indeed | |
| 187 # get started | |
| 188 | |
| 189 # do we own the files | |
| 190 tags=`echo "$@" | cut -d ' ' -f 2- -` | |
| 191 echo "Tags: $tags" | |
| 192 # slash do not want | |
| 193 outdir=`echo "$tags" | sed -e 's/\//_/g'` | |
| 152 | 194 ISNEW=0 |
| 148 | 195 if [ -O "$basedir" ]; then |
| 196 if [ ! -d "$basedir/$outdir" ]; then | |
| 197 ISNEW=1 | |
| 198 mkdir "$basedir/$outdir" | |
| 199 fi | |
| 200 if [ ! -O "$basedir/$outdir" ]; then | |
| 201 echo "You don't own the $basedir/$outdir, applying globally writeable permission on it" | |
| 202 chmod -R u=rwX,g=rwX,o=rwX "$basedir/$outdir" | |
| 203 fi | |
| 204 if [ `echo \`ls "$basedir/$outdir" | wc -l\`` -eq 0 ]; then | |
| 205 ISNEW=1 | |
| 206 fi | |
| 207 if [ ! -d "$basedir/temp" ]; then | |
| 208 mkdir "$basedir/temp" | |
| 209 fi | |
| 210 if [ ! -O "$basedir/temp" ]; then | |
| 211 echo "You don't own the $basedir/temp, applying globally writeable permission on it" | |
| 212 chmod -R u=rwX,g=rwX,o=rwX "$basedir/temp" | |
| 213 fi | |
| 214 else | |
| 215 echo "Fatal error: you don't own ${basedir}. Please fix ${basedir}. Stopping" | |
| 216 exit 1 | |
| 217 fi | |
| 218 # let's move to workdir | |
| 219 cd "$basedir/temp" | |
| 220 touch "$outdir-error" | |
| 221 touch "$outdir-ok" | |
| 222 touch "$outdir-list" | |
| 223 touch "$outdir-newlist" | |
| 224 # | |
| 225 | |
| 226 # let's do the job! | |
| 152 | 227 ISQUICK=0 |
| 148 | 228 case "$1" in |
| 229 status) | |
| 230 GENERATE | |
| 231 CHECK | |
| 232 ;; | |
| 152 | 233 fetch) |
| 148 | 234 GENERATE |
| 235 CHECK | |
| 236 FETCH | |
| 237 ;; | |
| 152 | 238 quickfetch) |
| 239 GENERATE | |
| 240 ISNEW=1 | |
| 241 ISQUICK=1 | |
| 242 CHECK | |
| 148 | 243 FETCH |
| 244 esac |