ScriptsCollections/DumpDatabasePostgreSQL
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update with commands in array for calls, logging for single file restore

Browse Source 
All commands are now run through a single array variable
All logs are in logs folder for the restore which is a sub folder to where the SQL file is located
On restore abort if the basic database creation failed or skip to the next database in block restore
This commit is contained in:
Clemens Schwaighofer
2025-10-17 10:30:11 +09:00
parent 0b938f31c8
commit b979cdd51f
3 changed files with 271 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
pg_db_dump_file.sh
View File

@@ -344,7 +344,8 @@ fi;
# check if we can connect to template1 table, if not we abort here
_PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}");
_PG_PARAMS_SELECT+=("SELECT datname FROM pg_catalog.pg_database WHERE datname = 'template1';");
connect=$(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}") || echo "[!] pgsql connect error";
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS_SELECT[@]}");
connect=$("${PG_COMMAND[@]}") || echo "[!] pgsql connect error";
if [ "${connect}" != "template1" ]; then
echo "Failed to connect to template1 with user '${DB_USER}' at host '${DB_HOST}' on port '${DB_PORT}'";
exit 0;
@@ -491,7 +492,8 @@ function get_dump_databases
fi;
_PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}");
_PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)';");
for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS_SELECT[@]}");
for owner_db in $("${PG_COMMAND[@]}"); do
db=$(echo "${owner_db}" | cut -d "," -f 2);
# check if we exclude this db
exclude=0;
@@ -615,10 +617,11 @@ if [ ${GLOBALS} -eq 1 ]; then
# build dump parms
_PG_PARAMS_DUMP=("${PG_PARAMS[@]}");
_PG_PARAMS_DUMP+=("--globals-only")
PG_COMMAND=("${PG_DUMPALL}" "${_PG_PARAMS_DUMP[@]}");
if [ ${TEST} -eq 0 ]; then
${PG_DUMPALL} "${_PG_PARAMS_DUMP[@]}" > "${filename}";
"${PG_COMMAND[@]}" > "${filename}";
else
echo "${PG_DUMPALL} ${_PG_PARAMS_DUMP[*]} > ${filename}";
echo "${PG_COMMAND[*]} > ${filename}";
fi;
echo "done";
else
@@ -641,7 +644,8 @@ fi;
filesize_sum=0;
_PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}");
_PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;");
for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS_SELECT[@]}");
for owner_db in $("${PG_COMMAND[@]}"); do
# get the user who owns the DB too
owner=$(echo "${owner_db}" | cut -d "," -f 1);
db=$(echo "${owner_db}" | cut -d "," -f 2);
@@ -673,10 +677,11 @@ for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do
# build dump parms
_PG_PARAMS_DUMP=("${PG_PARAMS[@]}");
_PG_PARAMS_DUMP+=("-c" "--format=c" "${db}")
PG_COMMAND=("${PG_DUMP}" "${_PG_PARAMS_DUMP[@]}");
if [ ${TEST} -eq 0 ]; then
${PG_DUMP} "${_PG_PARAMS_DUMP[@]}" > "${filename}";
"${PG_COMMAND[@]}" > "${filename}";
else
echo "${PG_DUMP} ${_PG_PARAMS_DUMP[*]} > ${filename}";
echo "${PG_COMMAND[*]} > ${filename}";
fi;
# get the file size for the dumped file and convert it to a human readable format
filesize=0;

138
pg_drop_restore.sh
View File

@@ -39,6 +39,8 @@ DRY_RUN=0;
BC='/usr/bin/bc';
PORT_REGEX="^[0-9]{4,5}$";
OPTARG_REGEX="^-";
# log path
LOG_PATH='';
MAX_JOBS='';
PG_PARAMS=();
PG_PARAM_ROLE=();
@@ -359,6 +361,18 @@ else
PG_PATH="${PG_BASE_PATH}${ident}/bin/'";
fi;
# set log path, this is the base path of the file + logs
LOG_PATH=$(dirname "${file}")"/logs/";
# create logs folder if missing
if [ ! -d "$LOG_PATH" ]; then
echo "+ Creating '$LOG_PATH' folder";
mkdir -p "$LOG_PATH";
if [ ! -d "$LOG_PATH" ]; then
echo "[!] Creation of '$LOG_PATH' folder failed";
exit 1;
fi;
fi;
PG_DROPDB="${PG_PATH}dropdb";
PG_CREATEDB="${PG_PATH}createdb";
PG_CREATELANG="${PG_PATH}createlang";
@@ -390,11 +404,11 @@ fi;
if [ $DRY_RUN -eq 1 ]; then
echo "**** [DRY RUN] ****";
fi;
echo "Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'";
echo "[.] Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'";
if [ $SCHEMA_ONLY -eq 1 ]; then
echo "!!!!!!! WILL ONLY RESTORE SCHEMA, NO DATA !!!!!!!";
fi;
if [ $NO_ASK -eq 1 ]; then
if [ $NO_ASK -eq 1 ] || [ $DRY_RUN -eq 1 ]; then
go='yes';
else
echo "Continue? type 'yes'";
@@ -406,75 +420,135 @@ if [ "$go" != 'yes' ]; then
else
start_time=$(date +"%F %T");
START=$(date +'%s');
echo "Drop DB $database [$_host:$_port] @ $start_time";
echo "[1] - Drop DB $database [$_host:$_port] @ $start_time";
# DROP DATABASE
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "${database}");
PG_COMMAND=("${PG_DROPDB}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_DROPDB}" "${_PG_PARAMS[@]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not drop database $database, aborting";
exit 1;
fi;
else
echo "${PG_DROPDB} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
# CREATE DATABASE
echo "Create DB $database with $owner and encoding $encoding on [$_host:$_port] @ $(date +"%F %T")";
echo "[2] + Create DB $database with $owner and encoding $encoding on [$_host:$_port] @ $(date +"%F %T")";
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-O" "${owner}" "-E" "${encoding}" "-T" "${TEMPLATEDB}" "${database}");
PG_COMMAND=("${PG_CREATEDB}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_CREATEDB}" "${_PG_PARAMS[@]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not create database $database, aborting";
exit 1;
fi;
else
echo "${PG_CREATEDB} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
# CREATE plpgsql LANG
if [ -f "$PG_CREATELANG" ]; then
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "plpgsql" "${database}");
echo "Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")";
PG_COMMAND=("${PG_CREATELANG}" "${_PG_PARAMS[@]}");
echo "[3] + Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")";
if [ $DRY_RUN -eq 0 ]; then
"${PG_CREATELANG}" "${_PG_PARAMS[@]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not create plgpgsql language in $database, aborting";
exit 1;
fi;
else
echo "${PG_CREATELANG} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
fi;
# RESTORE DATA
echo "Restore data from $file to DB $database on [$_host:$_port] with Jobs $MAX_JOBS @ $(date +"%F %T")";
echo "[4] % Restore data from $file to DB $database on [$_host:$_port] with Jobs $MAX_JOBS @ $(date +"%F %T")";
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("${PG_PARAM_ROLE[@]}")
_PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "${schema}" "-j" "${MAX_JOBS}" "${file}");
if [ -n "$schema" ]; then
_PG_PARAMS+=("${schema}");
fi;
_PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "-j" "${MAX_JOBS}" "${file}");
PG_COMMAND=("${PG_RESTORE}" "${_PG_PARAMS[@]}");
LOG_ERROR_FILE="${LOG_PATH}/restore_errors.${LOG_FILE_EXT}";
LOG_OUTPUT_FILE="${LOG_PATH}/restore_output.${LOG_FILE_EXT}";
if [ $DRY_RUN -eq 0 ]; then
"${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"restore_errors.${LOG_FILE_EXT}";
"${PG_COMMAND[@]}" 1>"${LOG_OUTPUT_FILE}" 2>"${LOG_ERROR_FILE}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not restore the database $database successfully, check ${LOG_ERROR_FILE} for details";
fi;
else
echo "${PG_RESTORE} ${_PG_PARAMS[*]} 2>restore_errors.${LOG_FILE_EXT}";
echo "${PG_COMMAND[*]} 1>${LOG_OUTPUT_FILE} 2>${LOG_ERROR_FILE}";
fi;
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data, check if the ACL dump is missing and do a basic restore
if ! "${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
echo "Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
echo "[5] ? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# grant usage on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
_PG_PARAMS+=("-U" "postgres" "-AtqX" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not fix usage access to basic public schema ACLs in $database";
fi;
else
echo "${PG_COMMAND[*]}";
fi;
# grant create on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
_PG_PARAMS+=("-U" "postgres" "-AtqX" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not fix create access to basic public schema ACLs in $database";
fi;
else
echo "${PG_COMMAND[*]}";
fi;
fi;
# SEQUENCE RESET DATA COLLECTION
echo "Resetting all sequences from DB $database [$_host:$_port] @ $(date +"%F %T")";
echo "[6] ? Resetting all sequences from DB $database [$_host:$_port] @ $(date +"%F %T")";
reset_query="SELECT 'SELECT SETVAL(' ||quote_literal(S.relname)|| ', MAX(' ||quote_ident(C.attname)|| ') ) FROM ' ||quote_ident(T.relname)|| ';' FROM pg_class AS S, pg_depend AS D, pg_class AS T, pg_attribute AS C WHERE S.relkind = 'S' AND S.oid = D.objid AND D.refobjid = T.oid AND D.refobjid = C.attrelid AND D.refobjsubid = C.attnum ORDER BY S.relname;";
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-o" "${TEMP_FILE}" "-c" "${reset_query}" "${database}");
_PG_PARAMS+=("-U" "postgres" "-AtqX" "-o" "${TEMP_FILE}" "-c" "${reset_query}" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
_PG_PARAMS_OUT=("${PG_PARAMS[@]}");
_PG_PARAMS_OUT+=("-U" "postgres" "-e" "-f" "${TEMP_FILE}" "${database}");
_PG_PARAMS_OUT+=("-U" "postgres" "-X" "-e" "-f" "${TEMP_FILE}" "${database}");
PG_COMMAND_OUT=("${PG_PSQL}" "${_PG_PARAMS_OUT[@]}");
LOG_OUTPUT_FILE="${LOG_PATH}/output_sequence.${LOG_FILE_EXT}";
LOG_ERROR_FILE="${LOG_PATH}/errors_sequence.${database}.${LOG_FILE_EXT}";
if [ $DRY_RUN -eq 0 ]; then
"${PG_PSQL}" "${_PG_PARAMS[@]}";
"${PG_PSQL}" "${_PG_PARAMS_OUT[@]}" 1>"output_sequence.${LOG_FILE_EXT}" 2>"errors_sequence.${database}.${LOG_FILE_EXT}";
rm "${TEMP_FILE}";
else
echo "${PG_PSQL}　${_PG_PARAMS[*]}";
echo "${PG_PSQL} ${_PG_PARAMS_OUT[*]} 1>output_sequence.${LOG_FILE_EXT} 2>errors_sequence.${database}.${LOG_FILE_EXT}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not create sequence reset query for database $database";
fi;
echo "Restore of data $file for DB $database [$_host:$_port] finished";
if [ -f "${TEMP_FILE}" ]; then
"${PG_COMMAND_OUT[@]}" 1>"${LOG_OUTPUT_FILE}" 2>"${LOG_ERROR_FILE}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not reset sequences for database $database";
fi;
rm "${TEMP_FILE}";
fi;
else
echo "${PG_COMMAND[*]}";
echo "${PG_COMMAND_OUT[*]} 1>${LOG_OUTPUT_FILE} 2>${LOG_ERROR_FILE}";
fi;
echo "[.] $ Restore of data $file for DB $database [$_host:$_port] finished";
DURATION=$(($(date "+%s")-START));
echo "Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION})";
echo "=== END RESTORE" >>"restore_errors.${LOG_FILE_EXT}";
echo "[.] * Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION})";
echo "=== END RESTORE" >>"${LOG_PATH}/restore_errors.${LOG_FILE_EXT}";
fi;

171
pg_restore_db_file.sh
View File

@@ -181,14 +181,14 @@ if [ ! -d "$DUMP_FOLDER" ]; then
exit;
fi;
LOGS=$DUMP_FOLDER'/logs/';
LOG_PATH=$DUMP_FOLDER'/logs/';
# create logs folder if missing
if [ ! -d "$LOGS" ]; then
echo "Creating '$LOGS' folder";
mkdir -p "$LOGS";
if [ ! -d "$LOGS" ]; then
echo "Creation of '$LOGS' folder failed";
exit;
if [ ! -d "$LOG_PATH" ]; then
echo "+ Creating '$LOG_PATH' folder";
mkdir -p "$LOG_PATH";
if [ ! -d "$LOG_PATH" ]; then
echo "[!] Creation of '$LOG_PATH' folder failed";
exit 1;
fi;
fi;
@@ -263,7 +263,7 @@ PG_PSQL="psql";
# default port and host
EXCLUDE_LIST="pg_globals"; # space separated
LOGFILE="tee -a "$LOGS/PG_RESTORE_DB_FILE.$(date +"%Y%m%d_%H%M%S").log"";
LOG_FILE="tee -a "$LOG_PATH/PG_RESTORE_DB_FILE.$(date +"%Y%m%d_%H%M%S").log"";
# get the count for DBs to import
db_count=$(find "${DUMP_FOLDER}" -name "*.sql" -print | wc -l);
@@ -286,11 +286,11 @@ fi;
if [ ${DRY_RUN} ]; then
echo "**** [DRY RUN] ****";
fi;
echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOGFILE;
echo "= into the DB server $_HOST:$_PORT" | $LOGFILE;
echo "= running $MAX_JOBS jobs" | $LOGFILE;
echo "= import logs: $LOGS" | $LOGFILE;
echo "" | $LOGFILE;
echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOG_FILE;
echo "= into the DB server $_HOST:$_PORT" | $LOG_FILE;
echo "= running $MAX_JOBS jobs" | $LOG_FILE;
echo "= import logs: $LOG_PATH" | $LOG_FILE;
echo "" | $LOG_FILE;
pos=1;
# go through all the files an import them into the database
MASTERSTART=$(date +"%s");
@@ -300,7 +300,7 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then
start_time=$(date +"%F %T");
START=$(date +"%s");
# get the pg_globals file
echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOGFILE;
echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOG_FILE;
# get newest and only the first one
file=$(find "$DUMP_FOLDER" -name "pg_global*" -type f -printf "%Ts\t%p\n" | sort -nr | head -1);
filename=$(basename "$file");
@@ -334,23 +334,24 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then
PG_PATH_VERSION_LOCAL="${PG_PATH_VERSION}";
fi;
PG_PATH="${PG_BASE_PATH}${PG_PATH_VERSION_LOCAL}${PG_PATH_BIN}";
echo "+ Restore globals file: $filename to [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
echo "+ Restore globals file: $filename to [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("-U" "postgres");
_PG_PARAMS+=("${PG_PARAM_HOST[@]}");
_PG_PARAMS+=("${PG_PARAM_PORT[@]}");
_PG_PARAMS+=("-f" "$file" "-e" "-q" "-X" "template1");
PG_COMMAND=("${PG_PATH}${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_PSQL}" "${_PG_PARAMS[@]}" | $LOGFILE;
"${PG_COMMAND[@]}" | $LOG_FILE;
else
echo "${PG_PATH}${PG_PSQL} ${_PG_PARAMS[*]}" | $LOGFILE;
echo "${PG_COMMAND[*]}" | $LOG_FILE;
fi;
DURATION=$(($(date +"%s")-START));
printf "=[Globals Restore]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOGFILE;
printf "=[Globals Restore]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE;
fi;
for file in "$DUMP_FOLDER/"*.sql; do
start_time=$(date +"%F %T");
START=$(date +"%s");
echo "=[$pos/$db_count]=START=[$start_time]==================================================>" | $LOGFILE;
echo "=[$pos/$db_count]=START=[$start_time]==================================================>" | $LOG_FILE;
# the encoding
set_encoding='';
# get the filename
@@ -358,6 +359,22 @@ for file in "$DUMP_FOLDER/"*.sql; do
# get the databse, user
# default file name is <database>.<owner>.<encoding>.<type>-<version>_<host>_<port>_<date>_<time>_<sequence>
database=$(echo "$filename" | cut -d "." -f 1);
# check this is skip or not
exclude=0;
for exclude_db in $EXCLUDE_LIST; do
if [ "$exclude_db" = "$database" ]; then
exclude=1;
break;
fi;
done;
if [ $exclude -eq 1 ]; then
DURATION=0;
echo "# Skipped DB '$database'" | $LOG_FILE;
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
# restore DB
owner=$(echo "$filename" | cut -d "." -f 2);
__encoding=$(echo "$filename" | cut -d "." -f 3);
# the last _ part if for version 10
@@ -403,14 +420,6 @@ for file in "$DUMP_FOLDER/"*.sql; do
PG_PATH_VERSION_LOCAL="${PG_PATH_VERSION}";
fi;
PG_PATH="${PG_BASE_PATH}${PG_PATH_VERSION_LOCAL}${PG_PATH_BIN}";
# check this is skip or not
exclude=0;
for exclude_db in $EXCLUDE_LIST; do
if [ "$exclude_db" = "$database" ]; then
exclude=1;
fi;
done;
if [ $exclude -eq 0 ]; then
# create user if not exist yet
# check query for user
# for all calls
@@ -422,49 +431,87 @@ for file in "$DUMP_FOLDER/"*.sql; do
_PG_PARAMS+=("-A" "-F" "," "-t" "-q" "-X" "-c" "SELECT oid FROM pg_roles WHERE rolname = '$owner';" "template1");
user_oid=$("$PG_PSQL" "${_PG_PARAMS[@]}");
if [ -z "$user_oid" ]; then
echo "+ Create USER '$owner' for DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
echo "+ Create USER '$owner' for DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-D" "-R" "-S" "$owner");
PG_COMMAND=("${PG_PATH}${PG_CREATEUSER}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_CREATEUSER}" "${_PG_PARAMS[@]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Creation of user '$owner' failed, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
else
echo "${PG_PATH}${PG_CREATEUSER} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
fi;
# before importing the data, drop this database
echo "- Drop DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
echo "- Drop DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("$database");
PG_COMMAND=("${PG_PATH}${PG_DROPDB}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_DROPDB}" "${_PG_PARAMS[@]}";
else
echo "${PG_PATH}${PG_DROPDB} ${_PG_PARAMS[*]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Could not drop database, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
echo "+ Create DB '$database' with '$owner' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
else
echo "${PG_COMMAND[*]}";
fi;
echo "+ Create DB '$database' with '$owner' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-O" "$owner" "-E" "$set_encoding" "-T" "$TEMPLATEDB" "$database");
PG_COMMAND=("${PG_PATH}${PG_CREATEDB}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_CREATEDB}" "${_PG_PARAMS[@]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Could not create database, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
else
echo "${PG_PATH}${PG_CREATEDB} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
if [ -f "${PG_PATH}${PG_CREATELANG}" ]; then
echo "+ Create plpgsql lang in DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
echo "+ Create plpgsql lang in DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("plpgsql" "$database");
PG_COMMAND=("${PG_PATH}${PG_CREATELANG}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_CREATELANG}" "${_PG_PARAMS[@]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Could not create plpgsql language, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
else
echo "${PG_PATH}${PG_CREATELANG} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
fi;
echo "% Restore data from '$filename' to DB '$database' using $MAX_JOBS jobs [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
echo "% Restore data from '$filename' to DB '$database' using $MAX_JOBS jobs [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-d" "$database" "-F" "c" "-v" "-c" "-j" "$MAX_JOBS" "$file");
PG_COMMAND=("${PG_PATH}${PG_RESTORE}" "${_PG_PARAMS[@]}");
LOG_ERROR_FILE="$LOG_PATH/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"$LOGS/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
"${PG_COMMAND[@]}" 2>"${LOG_ERROR_FILE}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Restore of database '$database' failed, see ${LOG_ERROR_FILE} for details";
fi;
else
echo "${PG_PATH}${PG_RESTORE} ${_PG_PARAMS[*]} 2>${LOGS}/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
echo "${PG_COMMAND[*]} 2>${LOG_ERROR_FILE}";
fi;
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data,
@@ -472,24 +519,30 @@ for file in "$DUMP_FOLDER/"*.sql; do
if ! "${PG_PATH}${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
echo "? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# grant usage on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
# grant create on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-Atq" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
fi;
echo "$ Restore of data '$filename' for DB '$database' [$_host:$_port] finished" | $LOGFILE;
DURATION=$(($(date "+%s")-START));
echo "* Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-AtqX" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
else
DURATION=0;
echo "# Skipped DB '$database'" | $LOGFILE;
echo "${PG_COMMAND[*]}";
fi;
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOGFILE;
# grant create on schema public to public;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-AtqX" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
else
echo "${PG_COMMAND[*]}";
fi;
fi;
echo "$ Restore of data '$filename' for DB '$database' [$_host:$_port] finished" | $LOG_FILE;
DURATION=$(($(date "+%s")-START));
echo "* Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds" | $LOG_FILE;
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE;
pos=$((pos+1));
done;
DURATION=$(($(date "+%s")-MASTERSTART));
echo "" | $LOGFILE;
echo "= Start at $master_start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds. Imported $db_count databases." | $LOGFILE;
echo "" | $LOG_FILE;
echo "= Start at $master_start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds. Imported $db_count databases." | $LOG_FILE;