# HG changeset patch # User edhoprima@gmail.com # Date 1246278179 0 # Node ID 4d28f3a957ee943d7e48718147aa13792c957ddc # Parent 652d9e268ceecf7302ad98d9e1dc9c6b07b6c080 cleanup~ diff -r 652d9e268cee -r 4d28f3a957ee moefetch.sh --- a/moefetch.sh Mon Jun 29 11:59:49 2009 +0000 +++ b/moefetch.sh Mon Jun 29 12:22:59 2009 +0000 @@ -38,9 +38,11 @@ # not user modifiable from here +SED_GET_FILENAME="s/.*\/\([^\/]*\)/\1/g" +SED_IS_MD5_FILE="s/\([0-9a-f]\{32\}\..*\)//g" ### TODO: -### - replace `...` with $(..) +### - replace `...` with $(..) (DONE) ### - sanity validator ### - unified repository to save bandwidth ### - bug stomping @@ -60,7 +62,7 @@ # fatal error handler Err_Fatal() { - printf "\nFatal error: ${1}" + printf "\nFatal error: ${1}\n" exit 1 } @@ -145,10 +147,8 @@ fi done [ "$(echo $(ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | wc -l))" -eq 0 ] && ISNEW=1 - # let's move to workdir - cd "${BASE_DIR}/temp" for i in error ok list newlist; do - touch "${SITE_DIR}-${TARGET_DIR}-${i}" || Fatal_Err "Error creating ${TARGET_DIR}-${i}. This shouldn't happen" + touch "${TEMP_PREFIX}-${i}" || Fatal_Err "Error creating ${TEMP_PREFIX}-${i}. This shouldn't happen" done # } @@ -168,15 +168,15 @@ if [ -d "${TRASH}" ]; then ISTRASH=1 else - if [ "$(echo "${TRASH}" | sed -e 's/.*\/\([\^\/]*\)/\1/g;s/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then + if [ "$(echo "${TRASH}" | sed -e "${SED_GET_FILENAME};${SED_IS_MD5_FILE}" | grep -v ^$)" ]; then ISTRASH=1 else - [ "$(cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-list" | sed -e 's/.*\/\([^\/]*\)/\1/g' | grep $(echo "${TRASH}" | sed -e 's/.*\/\([^\/]*\)/\1/g'))" ] || ISTRASH=1 + [ "$(cat "${TEMP_PREFIX}-list" | sed -e "${SED_GET_FILENAME}" | grep $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}"))" ] || ISTRASH=1 fi fi if [ "${ISTRASH}" ]; then mv -f "${TRASH}" "${BASE_DIR}/trash/${TRASH_DIR}" || Err_Fatal "Error deleting files" - echo "Moved ${TRASH} to ${BASE_DIR}/trash/${TRASH_DIR}" + echo "Moved $(echo "${TRASH}" | sed -e "${SED_GET_FILENAME}") to ${BASE_DIR}/trash/${TRASH_DIR}" fi done rmdir "${BASE_DIR}/trash/${TRASH_DIR}" 2>/dev/null @@ -189,16 +189,16 @@ [ "${NOCLEAN}" ] || Cleanup_Repository echo "Checking for errors..." cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" - > "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" + > "${TEMP_PREFIX}-error" for FILE in * do - if [ "$(echo "${FILE}" | sed -e 's/\([0-9a-f]\{32\}\..*\)//g' | grep -v ^$)" ]; then + if [ "$(echo "${FILE}" | sed -e '${SED_IS_MD5_FILE}' | grep -v ^$)" ]; then echo echo "Not a valid danbooru file: ${FILE}" else if [ "$(${MD5} "${FILE}" | cut -d ' ' -f1 -)" != "$(echo "${FILE}" | cut -d '.' -f1)" ] then - echo "${FILE}" >> "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" + echo "${FILE}" >> "${TEMP_PREFIX}-error" echo echo "Error: ${FILE}" fi @@ -206,13 +206,13 @@ fi done echo - TOTAL_ERROR=$(echo $(wc -l < "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error")) + TOTAL_ERROR=$(echo "$(wc -l < "${TEMP_PREFIX}-error")") echo "${TOTAL_ERROR} file(s) error" echo "Removing error files" if [ "${TOTAL_ERROR}" -eq 0 ]; then echo "No error file. 0 file removed" else - cat "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-error" | xargs rm + cat "${TEMP_PREFIX}-error" | xargs rm echo "${TOTAL_ERROR} file(s) removed" fi echo "$(echo $(ls | wc -l)) file(s) available locally" @@ -224,7 +224,7 @@ # THE FILES #ls "../${TARGET_DIR}" | grep -vf "${TARGET_DIR}-error" > "${TARGET_DIR}-ok" # - ls "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${SITE_DIR}-${TARGET_DIR}-error" - > "${SITE_DIR}-${TARGET_DIR}-ok" + find "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" | comm -1 -3 "${TEMP_PREFIX}-error" - > "${TEMP_PREFIX}-ok" cat "${SITE_DIR}-${TARGET_DIR}-list" | grep -vf "${SITE_DIR}-${TARGET_DIR}-ok" > "${SITE_DIR}-${TARGET_DIR}-newlist" echo "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist")) file(s) to be downloaded" @@ -241,13 +241,12 @@ # start downloading the images Fetch_Images() { - cd "${BASE_DIR}/temp" - if [ "$(echo $(wc -l < "${SITE_DIR}-${TARGET_DIR}-newlist"))" -eq 0 ]; then + if [ "$(echo $(wc -l < "${TEMP_PREFIX}-newlist"))" -eq 0 ]; then echo "No new file" else echo "Starting wget" cd "${BASE_DIR}/${SITE_DIR}/${TARGET_DIR}" - wget -e continue=on -bi "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}-newlist" -o "${BASE_DIR}/temp/${SITE_DIR}-${TARGET_DIR}.log" + wget -e continue=on -bi "${TEMP_PREFIX}-newlist" -o "${TEMP_PREFIX}.log" fi }