ScriptsCollections/DumpDatabasePostgreSQL
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update with commands in array for calls, logging for single file restore

Browse Source 
All commands are now run through a single array variable
All logs are in logs folder for the restore which is a sub folder to where the SQL file is located
On restore abort if the basic database creation failed or skip to the next database in block restore
This commit is contained in:
Clemens Schwaighofer
2025-10-17 10:30:11 +09:00
parent 0b938f31c8
commit b979cdd51f
3 changed files with 271 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
pg_db_dump_file.sh
View File

@@ -344,7 +344,8 @@ fi;
# check if we can connect to template1 table, if not we abort here # check if we can connect to template1 table, if not we abort here
_PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}"); _PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}");
_PG_PARAMS_SELECT+=("SELECT datname FROM pg_catalog.pg_database WHERE datname = 'template1';"); _PG_PARAMS_SELECT+=("SELECT datname FROM pg_catalog.pg_database WHERE datname = 'template1';");
connect=$(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}") || echo "[!] pgsql connect error"; PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS_SELECT[@]}");
connect=$("${PG_COMMAND[@]}") || echo "[!] pgsql connect error";
if [ "${connect}" != "template1" ]; then if [ "${connect}" != "template1" ]; then
echo "Failed to connect to template1 with user '${DB_USER}' at host '${DB_HOST}' on port '${DB_PORT}'"; echo "Failed to connect to template1 with user '${DB_USER}' at host '${DB_HOST}' on port '${DB_PORT}'";
exit 0; exit 0;
@@ -491,7 +492,8 @@ function get_dump_databases
fi; fi;
_PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}"); _PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}");
_PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)';"); _PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)';");
for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS_SELECT[@]}");
for owner_db in $("${PG_COMMAND[@]}"); do
db=$(echo "${owner_db}" | cut -d "," -f 2); db=$(echo "${owner_db}" | cut -d "," -f 2);
# check if we exclude this db # check if we exclude this db
exclude=0; exclude=0;
@@ -615,10 +617,11 @@ if [ ${GLOBALS} -eq 1 ]; then
# build dump parms # build dump parms
_PG_PARAMS_DUMP=("${PG_PARAMS[@]}"); _PG_PARAMS_DUMP=("${PG_PARAMS[@]}");
_PG_PARAMS_DUMP+=("--globals-only") _PG_PARAMS_DUMP+=("--globals-only")
PG_COMMAND=("${PG_DUMPALL}" "${_PG_PARAMS_DUMP[@]}");
if [ ${TEST} -eq 0 ]; then if [ ${TEST} -eq 0 ]; then
${PG_DUMPALL} "${_PG_PARAMS_DUMP[@]}" > "${filename}"; "${PG_COMMAND[@]}" > "${filename}";
else else
echo "${PG_DUMPALL} ${_PG_PARAMS_DUMP[*]} > ${filename}"; echo "${PG_COMMAND[*]} > ${filename}";
fi; fi;
echo "done"; echo "done";
else else
@@ -641,7 +644,8 @@ fi;
filesize_sum=0; filesize_sum=0;
_PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}"); _PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}");
_PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;"); _PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;");
for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS_SELECT[@]}");
for owner_db in $("${PG_COMMAND[@]}"); do
# get the user who owns the DB too # get the user who owns the DB too
owner=$(echo "${owner_db}" | cut -d "," -f 1); owner=$(echo "${owner_db}" | cut -d "," -f 1);
db=$(echo "${owner_db}" | cut -d "," -f 2); db=$(echo "${owner_db}" | cut -d "," -f 2);
@@ -673,10 +677,11 @@ for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do
# build dump parms # build dump parms
_PG_PARAMS_DUMP=("${PG_PARAMS[@]}"); _PG_PARAMS_DUMP=("${PG_PARAMS[@]}");
_PG_PARAMS_DUMP+=("-c" "--format=c" "${db}") _PG_PARAMS_DUMP+=("-c" "--format=c" "${db}")
PG_COMMAND=("${PG_DUMP}" "${_PG_PARAMS_DUMP[@]}");
if [ ${TEST} -eq 0 ]; then if [ ${TEST} -eq 0 ]; then
${PG_DUMP} "${_PG_PARAMS_DUMP[@]}" > "${filename}"; "${PG_COMMAND[@]}" > "${filename}";
else else
echo "${PG_DUMP} ${_PG_PARAMS_DUMP[*]} > ${filename}"; echo "${PG_COMMAND[*]} > ${filename}";
fi; fi;
# get the file size for the dumped file and convert it to a human readable format # get the file size for the dumped file and convert it to a human readable format
filesize=0; filesize=0;

136
pg_drop_restore.sh
View File

@@ -39,6 +39,8 @@ DRY_RUN=0;
BC='/usr/bin/bc'; BC='/usr/bin/bc';
PORT_REGEX="^[0-9]{4,5}$"; PORT_REGEX="^[0-9]{4,5}$";
OPTARG_REGEX="^-"; OPTARG_REGEX="^-";
# log path
LOG_PATH='';
MAX_JOBS=''; MAX_JOBS='';
PG_PARAMS=(); PG_PARAMS=();
PG_PARAM_ROLE=(); PG_PARAM_ROLE=();
@@ -359,6 +361,18 @@ else
PG_PATH="${PG_BASE_PATH}${ident}/bin/'"; PG_PATH="${PG_BASE_PATH}${ident}/bin/'";
fi; fi;
# set log path, this is the base path of the file + logs
LOG_PATH=$(dirname "${file}")"/logs/";
# create logs folder if missing
if [ ! -d "$LOG_PATH" ]; then
echo "+ Creating '$LOG_PATH' folder";
mkdir -p "$LOG_PATH";
if [ ! -d "$LOG_PATH" ]; then
echo "[!] Creation of '$LOG_PATH' folder failed";
exit 1;
fi;
fi;
PG_DROPDB="${PG_PATH}dropdb"; PG_DROPDB="${PG_PATH}dropdb";
PG_CREATEDB="${PG_PATH}createdb"; PG_CREATEDB="${PG_PATH}createdb";
PG_CREATELANG="${PG_PATH}createlang"; PG_CREATELANG="${PG_PATH}createlang";
@@ -390,11 +404,11 @@ fi;
if [ $DRY_RUN -eq 1 ]; then if [ $DRY_RUN -eq 1 ]; then
echo "**** [DRY RUN] ****"; echo "**** [DRY RUN] ****";
fi; fi;
echo "Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'"; echo "[.] Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'";
if [ $SCHEMA_ONLY -eq 1 ]; then if [ $SCHEMA_ONLY -eq 1 ]; then
echo "!!!!!!! WILL ONLY RESTORE SCHEMA, NO DATA !!!!!!!"; echo "!!!!!!! WILL ONLY RESTORE SCHEMA, NO DATA !!!!!!!";
fi; fi;
if [ $NO_ASK -eq 1 ]; then if [ $NO_ASK -eq 1 ] || [ $DRY_RUN -eq 1 ]; then
go='yes'; go='yes';
else else
echo "Continue? type 'yes'"; echo "Continue? type 'yes'";
@@ -406,75 +420,135 @@ if [ "$go" != 'yes' ]; then
else else
start_time=$(date +"%F %T"); start_time=$(date +"%F %T");
START=$(date +'%s'); START=$(date +'%s');
echo "Drop DB $database [$_host:$_port] @ $start_time"; echo "[1] - Drop DB $database [$_host:$_port] @ $start_time";
# DROP DATABASE # DROP DATABASE
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "${database}"); _PG_PARAMS+=("-U" "postgres" "${database}");
PG_COMMAND=("${PG_DROPDB}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
"${PG_DROPDB}" "${_PG_PARAMS[@]}"; "${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not drop database $database, aborting";
exit 1;
fi;
else else
echo "${PG_DROPDB} ${_PG_PARAMS[*]}"; echo "${PG_COMMAND[*]}";
fi; fi;
# CREATE DATABASE # CREATE DATABASE
echo "Create DB $database with $owner and encoding $encoding on [$_host:$_port] @ $(date +"%F %T")"; echo "[2] + Create DB $database with $owner and encoding $encoding on [$_host:$_port] @ $(date +"%F %T")";
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-O" "${owner}" "-E" "${encoding}" "-T" "${TEMPLATEDB}" "${database}"); _PG_PARAMS+=("-U" "postgres" "-O" "${owner}" "-E" "${encoding}" "-T" "${TEMPLATEDB}" "${database}");
PG_COMMAND=("${PG_CREATEDB}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
"${PG_CREATEDB}" "${_PG_PARAMS[@]}"; "${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not create database $database, aborting";
exit 1;
fi;
else else
echo "${PG_CREATEDB} ${_PG_PARAMS[*]}"; echo "${PG_COMMAND[*]}";
fi; fi;
# CREATE plpgsql LANG # CREATE plpgsql LANG
if [ -f "$PG_CREATELANG" ]; then if [ -f "$PG_CREATELANG" ]; then
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "plpgsql" "${database}"); _PG_PARAMS+=("-U" "postgres" "plpgsql" "${database}");
echo "Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")"; PG_COMMAND=("${PG_CREATELANG}" "${_PG_PARAMS[@]}");
echo "[3] + Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")";
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
"${PG_CREATELANG}" "${_PG_PARAMS[@]}"; "${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not create plgpgsql language in $database, aborting";
exit 1;
fi;
else else
echo "${PG_CREATELANG} ${_PG_PARAMS[*]}"; echo "${PG_COMMAND[*]}";
fi; fi;
fi; fi;
# RESTORE DATA # RESTORE DATA
echo "Restore data from $file to DB $database on [$_host:$_port] with Jobs $MAX_JOBS @ $(date +"%F %T")"; echo "[4] % Restore data from $file to DB $database on [$_host:$_port] with Jobs $MAX_JOBS @ $(date +"%F %T")";
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("${PG_PARAM_ROLE[@]}") _PG_PARAMS+=("${PG_PARAM_ROLE[@]}")
_PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "${schema}" "-j" "${MAX_JOBS}" "${file}"); if [ -n "$schema" ]; then
_PG_PARAMS+=("${schema}");
fi;
_PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "-j" "${MAX_JOBS}" "${file}");
PG_COMMAND=("${PG_RESTORE}" "${_PG_PARAMS[@]}");
LOG_ERROR_FILE="${LOG_PATH}/restore_errors.${LOG_FILE_EXT}";
LOG_OUTPUT_FILE="${LOG_PATH}/restore_output.${LOG_FILE_EXT}";
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
"${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"restore_errors.${LOG_FILE_EXT}"; "${PG_COMMAND[@]}" 1>"${LOG_OUTPUT_FILE}" 2>"${LOG_ERROR_FILE}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not restore the database $database successfully, check ${LOG_ERROR_FILE} for details";
fi;
else else
echo "${PG_RESTORE} ${_PG_PARAMS[*]} 2>restore_errors.${LOG_FILE_EXT}"; echo "${PG_COMMAND[*]} 1>${LOG_OUTPUT_FILE} 2>${LOG_ERROR_FILE}";
fi; fi;
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump # BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data, check if the ACL dump is missing and do a basic restore # it does not dump the default public ACL so the owner of the DB cannot access the data, check if the ACL dump is missing and do a basic restore
if ! "${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then if ! "${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
echo "Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")"; echo "[5] ? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# grant usage on schema public to public; # grant usage on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}"); _PG_PARAMS+=("-U" "postgres" "-AtqX" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}"; PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not fix usage access to basic public schema ACLs in $database";
fi;
else
echo "${PG_COMMAND[*]}";
fi;
# grant create on schema public to public; # grant create on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}"); _PG_PARAMS+=("-U" "postgres" "-AtqX" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}"; PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not fix create access to basic public schema ACLs in $database";
fi;
else
echo "${PG_COMMAND[*]}";
fi;
fi; fi;
# SEQUENCE RESET DATA COLLECTION # SEQUENCE RESET DATA COLLECTION
echo "Resetting all sequences from DB $database [$_host:$_port] @ $(date +"%F %T")"; echo "[6] ? Resetting all sequences from DB $database [$_host:$_port] @ $(date +"%F %T")";
reset_query="SELECT 'SELECT SETVAL(' ||quote_literal(S.relname)|| ', MAX(' ||quote_ident(C.attname)|| ') ) FROM ' ||quote_ident(T.relname)|| ';' FROM pg_class AS S, pg_depend AS D, pg_class AS T, pg_attribute AS C WHERE S.relkind = 'S' AND S.oid = D.objid AND D.refobjid = T.oid AND D.refobjid = C.attrelid AND D.refobjsubid = C.attnum ORDER BY S.relname;"; reset_query="SELECT 'SELECT SETVAL(' ||quote_literal(S.relname)|| ', MAX(' ||quote_ident(C.attname)|| ') ) FROM ' ||quote_ident(T.relname)|| ';' FROM pg_class AS S, pg_depend AS D, pg_class AS T, pg_attribute AS C WHERE S.relkind = 'S' AND S.oid = D.objid AND D.refobjid = T.oid AND D.refobjid = C.attrelid AND D.refobjsubid = C.attnum ORDER BY S.relname;";
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-o" "${TEMP_FILE}" "-c" "${reset_query}" "${database}"); _PG_PARAMS+=("-U" "postgres" "-AtqX" "-o" "${TEMP_FILE}" "-c" "${reset_query}" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
_PG_PARAMS_OUT=("${PG_PARAMS[@]}"); _PG_PARAMS_OUT=("${PG_PARAMS[@]}");
_PG_PARAMS_OUT+=("-U" "postgres" "-e" "-f" "${TEMP_FILE}" "${database}"); _PG_PARAMS_OUT+=("-U" "postgres" "-X" "-e" "-f" "${TEMP_FILE}" "${database}");
PG_COMMAND_OUT=("${PG_PSQL}" "${_PG_PARAMS_OUT[@]}");
LOG_OUTPUT_FILE="${LOG_PATH}/output_sequence.${LOG_FILE_EXT}";
LOG_ERROR_FILE="${LOG_PATH}/errors_sequence.${database}.${LOG_FILE_EXT}";
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
"${PG_PSQL}" "${_PG_PARAMS[@]}"; "${PG_COMMAND[@]}";
"${PG_PSQL}" "${_PG_PARAMS_OUT[@]}" 1>"output_sequence.${LOG_FILE_EXT}" 2>"errors_sequence.${database}.${LOG_FILE_EXT}"; RETURN_CODE=$?;
rm "${TEMP_FILE}"; if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not create sequence reset query for database $database";
fi;
if [ -f "${TEMP_FILE}" ]; then
"${PG_COMMAND_OUT[@]}" 1>"${LOG_OUTPUT_FILE}" 2>"${LOG_ERROR_FILE}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not reset sequences for database $database";
fi;
rm "${TEMP_FILE}";
fi;
else else
echo "${PG_PSQL}　${_PG_PARAMS[*]}"; echo "${PG_COMMAND[*]}";
echo "${PG_PSQL} ${_PG_PARAMS_OUT[*]} 1>output_sequence.${LOG_FILE_EXT} 2>errors_sequence.${database}.${LOG_FILE_EXT}"; echo "${PG_COMMAND_OUT[*]} 1>${LOG_OUTPUT_FILE} 2>${LOG_ERROR_FILE}";
fi; fi;
echo "Restore of data $file for DB $database [$_host:$_port] finished"; echo "[.] $ Restore of data $file for DB $database [$_host:$_port] finished";
DURATION=$(($(date "+%s")-START)); DURATION=$(($(date "+%s")-START));
echo "Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION})"; echo "[.] * Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION})";
echo "=== END RESTORE" >>"restore_errors.${LOG_FILE_EXT}"; echo "=== END RESTORE" >>"${LOG_PATH}/restore_errors.${LOG_FILE_EXT}";
fi; fi;

255
pg_restore_db_file.sh
View File

@@ -181,14 +181,14 @@ if [ ! -d "$DUMP_FOLDER" ]; then
exit; exit;
fi; fi;
LOGS=$DUMP_FOLDER'/logs/'; LOG_PATH=$DUMP_FOLDER'/logs/';
# create logs folder if missing # create logs folder if missing
if [ ! -d "$LOGS" ]; then if [ ! -d "$LOG_PATH" ]; then
echo "Creating '$LOGS' folder"; echo "+ Creating '$LOG_PATH' folder";
mkdir -p "$LOGS"; mkdir -p "$LOG_PATH";
if [ ! -d "$LOGS" ]; then if [ ! -d "$LOG_PATH" ]; then
echo "Creation of '$LOGS' folder failed"; echo "[!] Creation of '$LOG_PATH' folder failed";
exit; exit 1;
fi; fi;
fi; fi;
@@ -263,7 +263,7 @@ PG_PSQL="psql";
# default port and host # default port and host
EXCLUDE_LIST="pg_globals"; # space separated EXCLUDE_LIST="pg_globals"; # space separated
LOGFILE="tee -a "$LOGS/PG_RESTORE_DB_FILE.$(date +"%Y%m%d_%H%M%S").log""; LOG_FILE="tee -a "$LOG_PATH/PG_RESTORE_DB_FILE.$(date +"%Y%m%d_%H%M%S").log"";
# get the count for DBs to import # get the count for DBs to import
db_count=$(find "${DUMP_FOLDER}" -name "*.sql" -print | wc -l); db_count=$(find "${DUMP_FOLDER}" -name "*.sql" -print | wc -l);
@@ -286,11 +286,11 @@ fi;
if [ ${DRY_RUN} ]; then if [ ${DRY_RUN} ]; then
echo "**** [DRY RUN] ****"; echo "**** [DRY RUN] ****";
fi; fi;
echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOGFILE; echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOG_FILE;
echo "= into the DB server $_HOST:$_PORT" | $LOGFILE; echo "= into the DB server $_HOST:$_PORT" | $LOG_FILE;
echo "= running $MAX_JOBS jobs" | $LOGFILE; echo "= running $MAX_JOBS jobs" | $LOG_FILE;
echo "= import logs: $LOGS" | $LOGFILE; echo "= import logs: $LOG_PATH" | $LOG_FILE;
echo "" | $LOGFILE; echo "" | $LOG_FILE;
pos=1; pos=1;
# go through all the files an import them into the database # go through all the files an import them into the database
MASTERSTART=$(date +"%s"); MASTERSTART=$(date +"%s");
@@ -300,7 +300,7 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then
start_time=$(date +"%F %T"); start_time=$(date +"%F %T");
START=$(date +"%s"); START=$(date +"%s");
# get the pg_globals file # get the pg_globals file
echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOGFILE; echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOG_FILE;
# get newest and only the first one # get newest and only the first one
file=$(find "$DUMP_FOLDER" -name "pg_global*" -type f -printf "%Ts\t%p\n" | sort -nr | head -1); file=$(find "$DUMP_FOLDER" -name "pg_global*" -type f -printf "%Ts\t%p\n" | sort -nr | head -1);
filename=$(basename "$file"); filename=$(basename "$file");
@@ -334,23 +334,24 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then
PG_PATH_VERSION_LOCAL="${PG_PATH_VERSION}"; PG_PATH_VERSION_LOCAL="${PG_PATH_VERSION}";
fi; fi;
PG_PATH="${PG_BASE_PATH}${PG_PATH_VERSION_LOCAL}${PG_PATH_BIN}"; PG_PATH="${PG_BASE_PATH}${PG_PATH_VERSION_LOCAL}${PG_PATH_BIN}";
echo "+ Restore globals file: $filename to [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE; echo "+ Restore globals file: $filename to [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("-U" "postgres"); _PG_PARAMS=("-U" "postgres");
_PG_PARAMS+=("${PG_PARAM_HOST[@]}"); _PG_PARAMS+=("${PG_PARAM_HOST[@]}");
_PG_PARAMS+=("${PG_PARAM_PORT[@]}"); _PG_PARAMS+=("${PG_PARAM_PORT[@]}");
_PG_PARAMS+=("-f" "$file" "-e" "-q" "-X" "template1"); _PG_PARAMS+=("-f" "$file" "-e" "-q" "-X" "template1");
PG_COMMAND=("${PG_PATH}${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_PSQL}" "${_PG_PARAMS[@]}" | $LOGFILE; "${PG_COMMAND[@]}" | $LOG_FILE;
else else
echo "${PG_PATH}${PG_PSQL} ${_PG_PARAMS[*]}" | $LOGFILE; echo "${PG_COMMAND[*]}" | $LOG_FILE;
fi; fi;
DURATION=$(($(date +"%s")-START)); DURATION=$(($(date +"%s")-START));
printf "=[Globals Restore]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOGFILE; printf "=[Globals Restore]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE;
fi; fi;
for file in "$DUMP_FOLDER/"*.sql; do for file in "$DUMP_FOLDER/"*.sql; do
start_time=$(date +"%F %T"); start_time=$(date +"%F %T");
START=$(date +"%s"); START=$(date +"%s");
echo "=[$pos/$db_count]=START=[$start_time]==================================================>" | $LOGFILE; echo "=[$pos/$db_count]=START=[$start_time]==================================================>" | $LOG_FILE;
# the encoding # the encoding
set_encoding=''; set_encoding='';
# get the filename # get the filename
@@ -358,6 +359,22 @@ for file in "$DUMP_FOLDER/"*.sql; do
# get the databse, user # get the databse, user
# default file name is <database>.<owner>.<encoding>.<type>-<version>_<host>_<port>_<date>_<time>_<sequence> # default file name is <database>.<owner>.<encoding>.<type>-<version>_<host>_<port>_<date>_<time>_<sequence>
database=$(echo "$filename" | cut -d "." -f 1); database=$(echo "$filename" | cut -d "." -f 1);
# check this is skip or not
exclude=0;
for exclude_db in $EXCLUDE_LIST; do
if [ "$exclude_db" = "$database" ]; then
exclude=1;
break;
fi;
done;
if [ $exclude -eq 1 ]; then
DURATION=0;
echo "# Skipped DB '$database'" | $LOG_FILE;
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
# restore DB
owner=$(echo "$filename" | cut -d "." -f 2); owner=$(echo "$filename" | cut -d "." -f 2);
__encoding=$(echo "$filename" | cut -d "." -f 3); __encoding=$(echo "$filename" | cut -d "." -f 3);
# the last _ part if for version 10 # the last _ part if for version 10
@@ -403,93 +420,129 @@ for file in "$DUMP_FOLDER/"*.sql; do
PG_PATH_VERSION_LOCAL="${PG_PATH_VERSION}"; PG_PATH_VERSION_LOCAL="${PG_PATH_VERSION}";
fi; fi;
PG_PATH="${PG_BASE_PATH}${PG_PATH_VERSION_LOCAL}${PG_PATH_BIN}"; PG_PATH="${PG_BASE_PATH}${PG_PATH_VERSION_LOCAL}${PG_PATH_BIN}";
# check this is skip or not # create user if not exist yet
exclude=0; # check query for user
for exclude_db in $EXCLUDE_LIST; do # for all calls
if [ "$exclude_db" = "$database" ]; then _PG_PARAMS_ALL=("-U" "postgres");
exclude=1; _PG_PARAMS_ALL+=("${PG_PARAM_HOST[@]}");
fi; _PG_PARAMS_ALL+=("${PG_PARAM_PORT[@]}");
done; # for the call
if [ $exclude -eq 0 ]; then _PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
# create user if not exist yet _PG_PARAMS+=("-A" "-F" "," "-t" "-q" "-X" "-c" "SELECT oid FROM pg_roles WHERE rolname = '$owner';" "template1");
# check query for user user_oid=$("$PG_PSQL" "${_PG_PARAMS[@]}");
# for all calls if [ -z "$user_oid" ]; then
_PG_PARAMS_ALL=("-U" "postgres"); echo "+ Create USER '$owner' for DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS_ALL+=("${PG_PARAM_HOST[@]}");
_PG_PARAMS_ALL+=("${PG_PARAM_PORT[@]}");
# for the call
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-A" "-F" "," "-t" "-q" "-X" "-c" "SELECT oid FROM pg_roles WHERE rolname = '$owner';" "template1"); _PG_PARAMS+=("-D" "-R" "-S" "$owner");
user_oid=$("$PG_PSQL" "${_PG_PARAMS[@]}"); PG_COMMAND=("${PG_PATH}${PG_CREATEUSER}" "${_PG_PARAMS[@]}");
if [ -z "$user_oid" ]; then if [ ${DRY_RUN} -eq 0 ]; then
echo "+ Create USER '$owner' for DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE; "${PG_COMMAND[@]}";
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); RETURN_CODE=$?;
_PG_PARAMS+=("-D" "-R" "-S" "$owner"); if [ $RETURN_CODE -ne 0 ]; then
if [ ${DRY_RUN} -eq 0 ]; then echo "[!] Creation of user '$owner' failed, skipping database '$database'";
"${PG_PATH}${PG_CREATEUSER}" "${_PG_PARAMS[@]}"; printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
else pos=$((pos+1));
echo "${PG_PATH}${PG_CREATEUSER} ${_PG_PARAMS[*]}"; continue;
fi; fi;
fi;
# before importing the data, drop this database
echo "- Drop DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("$database");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_DROPDB}" "${_PG_PARAMS[@]}";
else else
echo "${PG_PATH}${PG_DROPDB} ${_PG_PARAMS[*]}"; echo "${PG_COMMAND[*]}";
fi; fi;
echo "+ Create DB '$database' with '$owner' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-O" "$owner" "-E" "$set_encoding" "-T" "$TEMPLATEDB" "$database");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_CREATEDB}" "${_PG_PARAMS[@]}";
else
echo "${PG_PATH}${PG_CREATEDB} ${_PG_PARAMS[*]}";
fi;
if [ -f "${PG_PATH}${PG_CREATELANG}" ]; then
echo "+ Create plpgsql lang in DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("plpgsql" "$database");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_CREATELANG}" "${_PG_PARAMS[@]}";
else
echo "${PG_PATH}${PG_CREATELANG} ${_PG_PARAMS[*]}";
fi;
fi;
echo "% Restore data from '$filename' to DB '$database' using $MAX_JOBS jobs [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-d" "$database" "-F" "c" "-v" "-c" "-j" "$MAX_JOBS" "$file");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"$LOGS/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
else
echo "${PG_PATH}${PG_RESTORE} ${_PG_PARAMS[*]} 2>${LOGS}/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
fi;
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data,
# check if the ACL dump is missing and do a basic restore
if ! "${PG_PATH}${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
echo "? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# grant usage on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
# grant create on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-Atq" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
fi;
echo "$ Restore of data '$filename' for DB '$database' [$_host:$_port] finished" | $LOGFILE;
DURATION=$(($(date "+%s")-START));
echo "* Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds" | $LOGFILE;
else
DURATION=0;
echo "# Skipped DB '$database'" | $LOGFILE;
fi; fi;
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOGFILE; # before importing the data, drop this database
echo "- Drop DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("$database");
PG_COMMAND=("${PG_PATH}${PG_DROPDB}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Could not drop database, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
else
echo "${PG_COMMAND[*]}";
fi;
echo "+ Create DB '$database' with '$owner' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-O" "$owner" "-E" "$set_encoding" "-T" "$TEMPLATEDB" "$database");
PG_COMMAND=("${PG_PATH}${PG_CREATEDB}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Could not create database, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
else
echo "${PG_COMMAND[*]}";
fi;
if [ -f "${PG_PATH}${PG_CREATELANG}" ]; then
echo "+ Create plpgsql lang in DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("plpgsql" "$database");
PG_COMMAND=("${PG_PATH}${PG_CREATELANG}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Could not create plpgsql language, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
else
echo "${PG_COMMAND[*]}";
fi;
fi;
echo "% Restore data from '$filename' to DB '$database' using $MAX_JOBS jobs [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-d" "$database" "-F" "c" "-v" "-c" "-j" "$MAX_JOBS" "$file");
PG_COMMAND=("${PG_PATH}${PG_RESTORE}" "${_PG_PARAMS[@]}");
LOG_ERROR_FILE="$LOG_PATH/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}" 2>"${LOG_ERROR_FILE}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Restore of database '$database' failed, see ${LOG_ERROR_FILE} for details";
fi;
else
echo "${PG_COMMAND[*]} 2>${LOG_ERROR_FILE}";
fi;
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data,
# check if the ACL dump is missing and do a basic restore
if ! "${PG_PATH}${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
echo "? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# grant usage on schema public to public;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-AtqX" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
else
echo "${PG_COMMAND[*]}";
fi;
# grant create on schema public to public;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-AtqX" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
else
echo "${PG_COMMAND[*]}";
fi;
fi;
echo "$ Restore of data '$filename' for DB '$database' [$_host:$_port] finished" | $LOG_FILE;
DURATION=$(($(date "+%s")-START));
echo "* Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds" | $LOG_FILE;
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE;
pos=$((pos+1)); pos=$((pos+1));
done; done;
DURATION=$(($(date "+%s")-MASTERSTART)); DURATION=$(($(date "+%s")-MASTERSTART));
echo "" | $LOGFILE; echo "" | $LOG_FILE;
echo "= Start at $master_start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds. Imported $db_count databases." | $LOGFILE; echo "= Start at $master_start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds. Imported $db_count databases." | $LOG_FILE;