# HG changeset patch # User edhoprima@gmail.com # Date 1246564873 0 # Node ID 710082ce6788753a735dce3ea68e3d335a5deed6 # Parent a6624fb9b317d0b29f502aeef4d1ef6ffe724b40 major cleanup part2. also called major mess up part 2. diff -r a6624fb9b317 -r 710082ce6788 moefetch.sh --- a/moefetch.sh Thu Jul 02 19:10:36 2009 +0000 +++ b/moefetch.sh Thu Jul 02 20:01:13 2009 +0000 @@ -38,9 +38,6 @@ # not user modifiable from here -SED_GET_FILENAME="s/.*\/\([^\/]*\)/\1/g" -SED_IS_MD5_FILE="s/\([0-9a-f]\{32\}\..*\)//g" - ### TODO: ### - sanity validator(?) ### - unified repository to save bandwidth @@ -50,9 +47,10 @@ # useless welcome message. Also version Msg_Welcome() { - MOEFETCHVERSION="0.2.1" + MOEFETCHVERSION="0.3-beta1" echo "moefetch ${MOEFETCHVERSION} -Copyright (c) 2009 edogawaconan " +Copyright (c) 2009 edogawaconan +" } # fatal error handler @@ -62,6 +60,13 @@ exit 1 } +Err_Impossible() { + echo " +Impossible error. Or you modified content of the working directories when the script is running. +Please report to moefetch.googlecode.com if you see this message (complete with entire run log)" + exit 1 +} + # help message Err_Help() { echo "moefetch.sh COMMAND [-s SITE_URL] TAGS @@ -79,16 +84,15 @@ # generate link by transforming xml Generate_Link() { echo " -Fetching xml file" - TEMPNUM=1000 +Fetching XML file" + tempnum=1000 _i=1 > "${TEMP_PREFIX}-list" - while [ "${TEMPNUM}" -ge 1000 ]; do + while [ "${tempnum}" -ge 1000 ]; do wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=1000&page=${_i}" -O "${TEMP_PREFIX}-xml" -e continue=off printf "Processing XML file... " # xslt evilry - > "${TEMP_PREFIX}-templist" - xsltproc - "${TEMP_PREFIX}-xml" < "${TEMP_PREFIX}-templist" 2>/dev/null + xsltproc - "${TEMP_PREFIX}-xml" < "${TEMP_PREFIX}-templist" @@ -96,14 +100,14 @@ EOF - TEMPNUM=$(echo $(wc -l < "${TEMP_PREFIX}-templist")) + tempnum=$(echo $(wc -l < "${TEMP_PREFIX}-templist")) _i=$((_i+1)) cat "${TEMP_PREFIX}-templist" >> "${TEMP_PREFIX}-list" - echo "${TEMPNUM} file(s) available" + echo "${tempnum} file(s) available" done - NUMFILES=$(echo $(wc -l < "${TEMP_PREFIX}-list")) - echo "${NUMFILES} file(s) available on server" - [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" + numfiles=$(echo $(wc -l < "${TEMP_PREFIX}-list")) + echo "${numfiles} file(s) available on server" + [ "${numfiles}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site." } Is_NotMD5() { @@ -131,7 +135,7 @@ Count_Files() { _i=0 for _f in "${*}/"* "${*}/".*; do - if [ "${_f}" != "${*}/"'*' ] || [ -e "${_f}" ]; then + if test "${_f}" != "${*}/"'*' || test -e "${_f}"; then _i=$((_i + 1)) fi done @@ -142,35 +146,35 @@ Check_Tools() { # verify all programs required do indeed exist #MD5 - if [ ! "${MD5}" ]; then - case $(uname) in + if test -z "${MD5}"; then + case "$(uname)" in *BSD) MD5="md5 -r";; Linux|SunOS) MD5="md5sum";; - *) Fatal_Err "No known md5 tool for this platform. Please specify manually" + *) Fatal_Err "No known md5 tool for this platform. Please specify manually";; esac fi - MD5_COMMAND=$(echo ${MD5} | cut -d' ' -f1) + md5_command=$(echo ${MD5} | cut -d' ' -f1) # basic tools - COMMANDS="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${MD5_COMMAND}" - for COMMAND in ${COMMANDS} + commands="cut sed wc wget xsltproc xargs rm mkdir chown comm grep date ${md5_command}" + for cmd in ${commands} do - [ "$(command -v "${COMMAND}")" ] || Err_Fatal "${COMMAND} doesn't exist in ${PATH}" + [ "$(command -v "${cmd}")" ] || Err_Fatal "${cmd} doesn't exist in ${PATH}" done } # verify required folders exist and writeable Check_Folders(){ - [ -O "${BASE_DIR}" ] || Err_Fatal "You don't own ${BASE_DIR}. Please fix ${BASE_DIR}." - for FOLDER in temp trash deleted "${SITE_DIR}/${TARGET_DIR}"; do - if [ ! -d "${BASE_DIR}/${FOLDER}" ]; then - mkdir "${BASE_DIR}/${FOLDER}" || Err_Fatal "${FOLDER} folder creation failed" + test -O "${BASE_DIR}" || Err_Fatal "You don't own ${BASE_DIR}. Please fix ${BASE_DIR} or run this script in your own directory." + for directory in temp trash deleted "${SITE_DIR}/${TARGET_DIR}"; do + if [ ! -d "${BASE_DIR}/${directory}" ]; then + mkdir "${BASE_DIR}/${directory}" || Err_Impossible fi - if [ ! -O "${BASE_DIR}/${FOLDER}" ]; then - echo "You don't own the ${BASE_DIR}/${FOLDER}, applying globally writeable permission on it" - chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${FOLDER}" || Err_Fatal "Error changing ownership. This shouldn't happen" + if [ ! -O "${BASE_DIR}/${directory}" ]; then + echo "You don't own the ${BASE_DIR}/${directory}, applying globally writeable permission on it" + chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${directory}" || Err_Impossible fi done - [ "$(Count_Files "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}")" -eq 0 ] && ISNEW=1 + test "$(Count_Files "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}")" -eq 0 && ISNEW=1 for i in error ok list newlist templist; do touch "${TEMP_PREFIX}-${i}" || Fatal_Err "Error creating ${TEMP_PREFIX}-${i}. This shouldn't happen" done @@ -182,32 +186,31 @@ # current dir: ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR} printf "Cleaning up repository folder... " Progress_Init - TRASH_DIR=$(date -u "+${SITE_DIR}-${TARGET_DIR}-%Y%m%d-%H.%M") - TRASHES= - mkdir -p "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Unable to create trash folder" - for TRASH in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* + trash_dir="${BASE_DIR}/trash/${trash_dir}/$(date -u "+${SITE_DIR}-${TARGET_DIR}-%Y%m%d-%H.%M")" + trashes="These files have been moved to ${trash_dir}:" + has_trash= + if test ! -d "${trash_dir}"; then + mkdir -p "${BASE_DIR}/trash/${trash_dir}" || Err_Impossible + else + if test ! -o "${trash_dir}"; then + chmod -R u=rwX,g=rwX,o=rwX "${BASE_DIR}/${directory}" || Err_Impossible + fi + fi + for trash in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* do - ISTRASH= - if [ -d "${TRASH}" ]; then - ISTRASH=1 - else - if [ "$(Is_NotMD5 "$(basename "${TRASH}")")" ]; then - ISTRASH=1 - else - [ "$(grep "$(basename "${TRASH}")" "${TEMP_PREFIX}-list")" ] || ISTRASH=1 - fi - fi - if [ "${ISTRASH}" ]; then - mv -f "${TRASH}" "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Error deleting files" - TRASHES="${TRASHES} -$(basename "${TRASH}")" + is_trash= + if test -d "${trash}" || test -n "$(Is_NotMD5 "$(basename "${trash}")")" || test -n "$(grep "$(basename "${trash}")" "${TEMP_PREFIX}-list")"; then + is_trash=1 + has_trash=1 + mv -f "${trash}" "${trash_dir}" || Err_Impossible + trashes="${trashes} +$(basename "${trash}")" fi Progress_Anim done - rmdir "${BASE_DIR}/trash/${TRASH_DIR}" 2>/dev/null + rmdir "${trash_dir}" 2>/dev/null Progress_Done - echo "These files have been moved to ${BASE_DIR}/trash/${TRASH_DIR}: -${TRASHES}" + test -z "${has_trash}" || echo "${trashes}" } # check files correctness @@ -258,7 +261,7 @@ cat "${TEMP_PREFIX}-list" > "${TEMP_PREFIX}-templist" while read -r is_ok; do grep -v "${is_ok}" "${TEMP_PREFIX}-templist" > "${TEMP_PREFIX}-newlist" - cat "${TEMP_PREFIX}-newlist" > "${TEMP_PREFIX}-templist" + cp -f "${TEMP_PREFIX}-newlist" "${TEMP_PREFIX}-templist" || Err_Impossible Progress_Anim done < "${TEMP_PREFIX}-ok" Progress_Done @@ -275,7 +278,7 @@ # start downloading the images Fetch_Images() { - if [ "$(echo $(wc -l < "${TEMP_PREFIX}-newlist"))" -eq 0 ]; then + if test "$(echo $(wc -l < "${TEMP_PREFIX}-newlist"))" -eq 0; then echo "No new file" else printf "Starting wget... " @@ -287,13 +290,13 @@ # initialize base variables and initial command check Init(){ # path initialization - [ "${ADDITIONAL_PATH}" ] && PATH="${ADDITIONAL_PATH}:${PATH}" + test -n "${ADDITIONAL_PATH}" && PATH="${ADDITIONAL_PATH}:${PATH}" export PATH # misc variables ISQUICK= ISNEW= - + [ $# -lt 2 ] && Err_Help case "$1" in check|fetch|quickfetch) @@ -326,12 +329,12 @@ esac shift done - [ "${SITE}" ] || SITE="${DEFAULT_SITE}" - [ "${TAGS}" ] || Err_Fatal "No tag specified" + test -n "${SITE}" || SITE=${DEFAULT_SITE} + test -n "${TAGS}" || Err_Fatal "No tag specified" # Get base folder - default, current folder or fallback to ${HOME} - [ "${BASE_DIR}" ] || BASE_DIR="${PWD}" - [ "${BASE_DIR}" ] || BASE_DIR="{$HOME}" - [ "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" ] || BASE_DIR="/${BASE_DIR}" + test -n "${BASE_DIR}" || BASE_DIR=${PWD} + test -n "${BASE_DIR}" || BASE_DIR=${HOME} + test -n "$(echo "${BASE_DIR}" | cut -c1 | grep \/)" || BASE_DIR="/${BASE_DIR}" echo "Tags: ${TAGS}" # slash is not wanted for folder name