Mercurial > ec-dotfiles
diff moefetch.sh @ 214:a6624fb9b317
major cleanup. tweaking.
untested
author | edhoprima@gmail.com <edhoprima@gmail.com> |
---|---|
date | Thu, 02 Jul 2009 19:10:36 +0000 |
parents | dd95cf01602c |
children | 710082ce6788 |
line wrap: on
line diff
--- a/moefetch.sh Tue Jun 30 09:23:02 2009 +0000 +++ b/moefetch.sh Thu Jul 02 19:10:36 2009 +0000 @@ -51,23 +51,20 @@ # useless welcome message. Also version Msg_Welcome() { MOEFETCHVERSION="0.2.1" - cat <<EOF -moefetch ${MOEFETCHVERSION} -Copyright (c) 2009 edogawaconan <me@myconan.net> - -EOF + echo "moefetch ${MOEFETCHVERSION} +Copyright (c) 2009 edogawaconan <me@myconan.net>" } # fatal error handler Err_Fatal() { - printf "\nFatal error: ${1}\n" + echo " +Fatal error: ${1}" exit 1 } # help message Err_Help() { - cat <<EOF -moefetch.sh COMMAND [-s SITE_URL] TAGS + echo "moefetch.sh COMMAND [-s SITE_URL] TAGS COMMAND: (quick)fetch: do a complete update. Add prefix quick to skip file checking @@ -75,21 +72,20 @@ -s SITE_URL: Specify URL of the Danbooru powered site you want to leech from. Default is ${DEFAULT_SITE} -TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme - -EOF +TAGS: Tags you want to download. Separated by spaces. Tag name follows standard Danbooru tagging scheme" exit 2 } # generate link by transforming xml Generate_Link() { - printf "\nFetching xml file\n" - TEMPNUM=1001 + echo " +Fetching xml file" + TEMPNUM=1000 _i=1 > "${TEMP_PREFIX}-list" while [ "${TEMPNUM}" -ge 1000 ]; do wget "http://${SITE}/post/index.xml?tags=${TAGS}&offset=0&limit=1000&page=${_i}" -O "${TEMP_PREFIX}-xml" -e continue=off - printf "Processing XML file..." + printf "Processing XML file... " # xslt evilry > "${TEMP_PREFIX}-templist" xsltproc - "${TEMP_PREFIX}-xml" <<EOF | sed 's/.*\(http.*\)\(\/[a-f0-9]\{32\}\).*\.\([^\.]*\)/\1\2.\3/g' | grep ^http > "${TEMP_PREFIX}-templist" 2>/dev/null @@ -103,11 +99,15 @@ TEMPNUM=$(echo $(wc -l < "${TEMP_PREFIX}-templist")) _i=$((_i+1)) cat "${TEMP_PREFIX}-templist" >> "${TEMP_PREFIX}-list" + echo "${TEMPNUM} file(s) available" done NUMFILES=$(echo $(wc -l < "${TEMP_PREFIX}-list")) + echo "${NUMFILES} file(s) available on server" [ "${NUMFILES}" -gt 0 ] || Err_Fatal "Error in processing list or no files can be found with specified tag(s) or site" - echo " ${NUMFILES} file(s) available on server" - #output file: ${TARGET_DIR}-list +} + +Is_NotMD5() { + echo "${*}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g' } Progress_Init() { @@ -179,13 +179,11 @@ # Do some cleanup Cleanup_Repository() { - - # THE FILES - # current dir: ${BASE_DIR}/${SITE_DIR}/${TARGET_DIR} printf "Cleaning up repository folder... " Progress_Init TRASH_DIR=$(date -u "+${SITE_DIR}-${TARGET_DIR}-%Y%m%d-%H.%M") + TRASHES= mkdir -p "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Unable to create trash folder" for TRASH in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* do @@ -193,71 +191,80 @@ if [ -d "${TRASH}" ]; then ISTRASH=1 else - if [ "$(echo "${TRASH}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then + if [ "$(Is_NotMD5 "$(basename "${TRASH}")")" ]; then ISTRASH=1 else - [ "$(cat "${TEMP_PREFIX}-list" | sed -e "${SED_GET_FILENAME}" | grep $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}"))" ] || ISTRASH=1 + [ "$(grep "$(basename "${TRASH}")" "${TEMP_PREFIX}-list")" ] || ISTRASH=1 fi fi if [ "${ISTRASH}" ]; then mv -f "${TRASH}" "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Error deleting files" - printf "\bMoved $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}") to ${BASE_DIR}/trash/${TRASH_DIR}\n${_last}" + TRASHES="${TRASHES} +$(basename "${TRASH}")" fi Progress_Anim done rmdir "${BASE_DIR}/trash/${TRASH_DIR}" 2>/dev/null Progress_Done + echo "These files have been moved to ${BASE_DIR}/trash/${TRASH_DIR}: +${TRASHES}" } # check files correctness Check_Files() { - if [ ! "${ISNEW}" ]; then - [ "${NOCLEAN}" ] || Cleanup_Repository + if test ! -n "${ISNEW}"; then + test -z "${NOCLEAN}" && Cleanup_Repository printf "Checking for errors... " Progress_Init + files_error="These files do not match its md5:" + files_notdanbooru="These files are not checked:" + has_err_filename= + has_err_md5= > "${TEMP_PREFIX}-error" - for FILE in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* + > "${TEMP_PREFIX}-ok" + for file in "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}/"* do - if [ "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then - printf "\bNot a valid danbooru file: $(echo ${FILE} | sed -e "${SED_GET_FILENAME}")\n${_last}" + if test -n "$(Is_NotMD5 "$(basename "${file}")")" || test -d "${file}"; then + files_notdanbooru="${files_notdanbooru} +$(basename "${file}")" + has_err_filename=1 else - if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | sed -e "${SED_GET_FILENAME}" | cut -d '.' -f1)" ] - then - echo "${FILE}" >> "${TEMP_PREFIX}-error" - printf "\bError: $(echo "${FILE}" | sed -e "${SED_GET_FILENAME}")\n${_last}" + if test "$(${MD5} "${file}" | cut -d ' ' -f1 -)" = "$(basename "${file}" | cut -d '.' -f1)"; then + echo "$(basename "${file}")" >> "${TEMP_PREFIX}-ok" + else + rm "${file}" || Err_Fatal "Error removing ${file}" + echo "$(basename "${file}")" >> "${TEMP_PREFIX}-error" + files_error="${files_error} +$(basename "${file}")" + has_err_md5=1 fi - Progress_Anim fi + Progress_Anim done Progress_Done - TOTAL_ERROR=$(echo $(wc -l < "${TEMP_PREFIX}-error")) - if [ "${TOTAL_ERROR}" -eq 0 ]; then + if test ! -n "${has_err_md5}" && test ! -n "${has_err_filename}"; then echo "All files OK" else - printf "${TOTAL_ERROR} file(s) broken: removing..." - cat "${TEMP_PREFIX}-error" | xargs rm - echo " ${TOTAL_ERROR} file(s) removed" + if test ! -n "${has_err_md5}"; then + echo "${files_error}" + echo "$(echo $(wc -l < "${TEMP_PREFIX}-error")) file(s) removed" + fi + test -n "${has_err_filename}" && echo "${files_notdanbooru}" fi - echo "$(Count_Files "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}") file(s) available locally" + echo "$(echo $(wc -l < "${TEMP_PREFIX}-ok")) file(s) available locally" printf "Generating list of new files... " Progress_Init - # THE FILES - #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" - # - find "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${TEMP_PREFIX}-error" - | sed -e "${SED_GET_FILENAME}" > "${TEMP_PREFIX}-ok" cat "${TEMP_PREFIX}-list" > "${TEMP_PREFIX}-templist" - while read -r IS_OK; do - cat "${TEMP_PREFIX}-templist" | grep -v "${IS_OK}" > "${TEMP_PREFIX}-newlist" + while read -r is_ok; do + grep -v "${is_ok}" "${TEMP_PREFIX}-templist" > "${TEMP_PREFIX}-newlist" cat "${TEMP_PREFIX}-newlist" > "${TEMP_PREFIX}-templist" Progress_Anim done < "${TEMP_PREFIX}-ok" Progress_Done - #cat "${TEMP_PREFIX}-list" | egrep -vf "${TEMP_PREFIX}-ok" > "${TEMP_PREFIX}-newlist" - echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded" - + echo "$(echo $(wc -l < "${TEMP_PREFIX}-newlist")) file(s) to be downloaded" else - if [ "${ISQUICK}" ]; then + if test -n "${ISQUICK}"; then echo "Quick mode selected. Skipping check" else echo "Empty local repository"