Mercurial > ec-dotfiles
changeset 213:dd95cf01602c
working around limit
| author | edhoprima@gmail.com <edhoprima@gmail.com> |
|---|---|
| date | Tue, 30 Jun 2009 09:23:02 +0000 |
| parents | da4c03c7377c |
| children | a6624fb9b317 |
| files | moefetch.sh |
| diffstat | 1 files changed, 12 insertions(+), 4 deletions(-) [+] |
line wrap: on
line diff
--- a/moefetch.sh Mon Jun 29 17:35:43 2009 +0000 +++ b/moefetch.sh Tue Jun 30 09:23:02 2009 +0000 @@ -84,11 +84,15 @@ # generate link by transforming xml Generate_Link() { printf "\nFetching xml file\n" - wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=100000" -O "${TEMP_PREFIX}-xml" -e continue=off - printf "Processing XML file..." - # xslt evilry + TEMPNUM=1001 + _i=1 > "${TEMP_PREFIX}-list" - xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-list" 2>/dev/null + while [ "${TEMPNUM}" -ge 1000 ]; do + wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=1000&page=${_i}" -O "${TEMP_PREFIX}-xml" -e continue=off + printf "Processing XML file..." + # xslt evilry + > "${TEMP_PREFIX}-templist" + xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-templist" 2>/dev/null <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> <xsl:output method="xml" indent="yes"/> <xsl:template match="post"> @@ -96,6 +100,10 @@ </xsl:template> </xsl:stylesheet> EOF + TEMPNUM=$(echo $(wc -l < "${TEMP_PREFIX}-templist")) + _i=$((_i+1)) + cat "${TEMP_PREFIX}-templist" >> "${TEMP_PREFIX}-list" + done NUMFILES=$(echo $(wc -l < "${TEMP_PREFIX}-list")) [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" echo " ${NUMFILES} file(s) available on server"