#!/usr/bin/env bash # for use with elasticsearch container named `sist2-elasticsearch`; change as needed # expected directory layout is: # ${IMAGE_DIRECTORY}/YYYY-MM # ${SCRIPT_FILE} should be written in Painless Scripting Language; # an example is available at https://cdn.seedno.de/txt/scripts/elasticsearchexif.psl # exit if a command fails set -o errexit # exit if required variables are not set set -o nounset # return the exit status of the final command before a failure set -o pipefail # configure environment SIST2_VERSION="latest" THREAD_COUNT="$(nproc)" THUMBNAIL_QUALITY="1.0" BATCH_SIZE="1000" REWRITE_URL="https://storage.seedno.de/archive" SIST2_VOLUME="traefik_sist2-archive" IMAGE_DIRECTORY="/storage/archive" ELASTICSEARCH_URL="http://sist2-elasticsearch:9200" SCRIPT_FILE="${HOME}/bin/elasticsearchexif.psl" DOCKER_NETWORK="traefik" SIST2_CONTAINER="sist2-archive" SIST2_INDEX="archive" TAG_USERNAME="" TAG_PASSWORD="" # initialize processed directory counter DIRECTORIES_PROCESSED=0 # iterate through image directories for PHOTO_DIR in "${IMAGE_DIRECTORY}"/*; do # store image paths in variables FULL_PATH="${PHOTO_DIR}" DATE_PATH="$(echo "${PHOTO_DIR}" | rev | cut --delimiter="/" --fields=1 | rev)" # default to performing scan SKIP_SCAN=false # skip scan if directory exists and is up-to-date OLD_CONTENTS="${HOME}"/.archive/"${DATE_PATH}" NEW_CONTENTS="$(mktemp)" ls "${FULL_PATH}" > "${NEW_CONTENTS}" if [ -f "${OLD_CONTENTS}" ]; then if cmp -s "${OLD_CONTENTS}" "${NEW_CONTENTS}"; then echo "Directory ${FULL_PATH} has not changed since last run. Skipping scan..." SKIP_SCAN=true fi # create directory listing file else # create empty file for directory listing install -D /dev/null "${OLD_CONTENTS}" fi # scan if needed if [ "${SKIP_SCAN}" != "true" ]; then # delete existing directory listing, if needed INDEX_EXISTS="$(docker run --rm --interactive --mount type=volume,source="${SIST2_VOLUME}",destination=/files busybox:latest ls -d /files/"${DATE_PATH}" >/dev/null 2>&1; echo $?)" if [ "${INDEX_EXISTS}" -eq 0 ]; then echo "Removing outdated index ${DATE_PATH}..." docker run --rm --interactive --mount type=volume,source="${SIST2_VOLUME}",destination=/index busybox:latest rm -rf /index/"${DATE_PATH}" fi # scan files echo "Scanning files into index ${DATE_PATH}..." docker run --rm --interactive --cpu-shares 512 --mount type=bind,source="${FULL_PATH}",destination="${FULL_PATH}",readonly --mount type=volume,source="${SIST2_VOLUME}",destination=/out simon987/sist2:"${SIST2_VERSION}" scan --threads="${THREAD_COUNT}" --quality="${THUMBNAIL_QUALITY}" --rewrite-url="${REWRITE_URL}/${DATE_PATH}" --name="${DATE_PATH}" --output=/out/"${DATE_PATH}" "${FULL_PATH}" fi # reset index if this is the first iteration if [ "${DIRECTORIES_PROCESSED}" -eq 0 ]; then RESET_FLAGS="--force-reset" else RESET_FLAGS="" fi # apply user script if provided if [ ! -z "$SCRIPT_FILE" ]; then SCRIPT_VOLUME="--mount type=bind,source=${SCRIPT_FILE},destination=/script.psl" SCRIPT_FLAGS="--script-file=/script.psl" else SCRIPT_VOLUME="" SCRIPT_FLAGS="" fi # index files echo "Indexing files in ${DATE_PATH}..." docker run --rm --interactive --cpu-shares 512 --mount type=volume,source="${SIST2_VOLUME}",destination=/out ${SCRIPT_VOLUME} --network "${DOCKER_NETWORK}" simon987/sist2:"${SIST2_VERSION}" index ${RESET_FLAGS} --batch-size="${BATCH_SIZE}" --es-url="${ELASTICSEARCH_URL}" --es-index="${SIST2_INDEX}" "${SCRIPT_FLAGS}" /out/"${DATE_PATH}" # populate directory listing cp "${NEW_CONTENTS}" "${OLD_CONTENTS}" # increment counter DIRECTORIES_PROCESSED=$((DIRECTORIES_PROCESSED+1)) done # display number of directories indexed echo "Indexed ${DIRECTORIES_PROCESSED} directories." # stop exiting on error set +o errexit # stop and remove old sist2 instance docker stop "${SIST2_CONTAINER}" >/dev/null 2>&1 docker rm "${SIST2_CONTAINER}" >/dev/null 2>&1 # start new sist2 instance docker run --detach \ --mount type=volume,source="${SIST2_VOLUME}",destination=/index \ --network="${DOCKER_NETWORK}" \ --name="${SIST2_CONTAINER}" \ --entrypoint=/bin/sh \ simon987/sist2:"${SIST2_VERSION}" \ -c "/root/sist2 web \ --bind=0.0.0.0:4321 \ --es-url=${ELASTICSEARCH_URL} \ --es-index=${SIST2_INDEX} \ --tag-auth=${TAG_USERNAME}:${TAG_PASSWORD} \ /index/*" \ >/dev/null