481 lines
14 KiB
Bash
Executable File
481 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
# Author: Clemens Schwaighofer
|
||
# Description:
|
||
# Drop and restore one database from a dump created by created by pg_db_dump_file.sh
|
||
|
||
function usage ()
|
||
{
|
||
cat <<- EOT
|
||
Restores a single database dump to a database
|
||
|
||
Usage: ${0##/*/} -o <DB OWNER> -d <DB NAME> -f <FILE NAME> [-h <DB HOST>] [-p <DB PORT>] [-e <ENCODING>] [-i <POSTGRES VERSION>] [-j <JOBS>] [-s] [-r] [-n]
|
||
|
||
-o <DB OWNER>: The user who will be owner of the database to be restored
|
||
-d <DB NAME>: The database to restore the file to
|
||
-f <FILE NAME>: the data that should be loaded
|
||
-h <DB HOST>: optional hostname, if not given 'localhost' is used. Use 'local' to use unix socket
|
||
-p <DB PORT>: optional port number, if not given '5432' is used
|
||
-e <ENCODING>: optional encoding name, if not given 'UTF8' is used
|
||
-i <POSTGRES VERSION>: optional postgresql version in the format X.Y, if not given the default is used (current active)
|
||
-j <JOBS>: Run how many jobs Parallel. If not set, 2 jobs are run parallel
|
||
-s: Restore only schema, no data
|
||
-r: use redhat base paths instead of debian
|
||
-n: dry run, do not do anything, just test flow
|
||
EOT
|
||
}
|
||
|
||
_port=5432
|
||
_host='local';
|
||
_encoding='UTF8';
|
||
# role='';
|
||
schema='';
|
||
NO_ASK=0;
|
||
TEMPLATEDB='template0';
|
||
SCHEMA_ONLY=0;
|
||
ERROR=0;
|
||
REDHAT=0;
|
||
DRY_RUN=0;
|
||
BC='/usr/bin/bc';
|
||
PORT_REGEX="^[0-9]{4,5}$";
|
||
OPTARG_REGEX="^-";
|
||
MAX_JOBS='';
|
||
PG_PARAMS=();
|
||
PG_PARAM_ROLE=();
|
||
# if we have options, set them and then ignore anything below
|
||
while getopts ":o:d:h:f:p:e:i:j:rqnms" opt; do
|
||
# pre test for unfilled
|
||
if [ "${opt}" = ":" ] || [[ "${OPTARG-}" =~ ${OPTARG_REGEX} ]]; then
|
||
if [ "${opt}" = ":" ]; then
|
||
CHECK_OPT=${OPTARG};
|
||
else
|
||
CHECK_OPT=${opt};
|
||
fi;
|
||
case ${CHECK_OPT} in
|
||
o)
|
||
echo "-o needs an owner name";
|
||
ERROR=1;
|
||
;;
|
||
d)
|
||
echo "-d needs a database name";
|
||
ERROR=1;
|
||
;;
|
||
h)
|
||
echo "-h needs a host name";
|
||
ERROR=1;
|
||
;;
|
||
f)
|
||
echo "-f needs a file name";
|
||
ERROR=1;
|
||
;;
|
||
p)
|
||
echo "-h needs a port number";
|
||
ERROR=1;
|
||
;;
|
||
e)
|
||
echo "-e needs an encoding";
|
||
ERROR=1;
|
||
;;
|
||
i)
|
||
echo "-i needs a postgresql version";
|
||
ERROR=1;
|
||
;;
|
||
j)
|
||
echo "-j needs a numeric value for parallel jobs";
|
||
ERROR=1;
|
||
;;
|
||
esac
|
||
fi;
|
||
case $opt in
|
||
# o|owner)
|
||
o)
|
||
if [ -z "$owner" ]; then
|
||
owner=$OPTARG;
|
||
# if not standard user we need to set restore role
|
||
# so tables/etc get set to new user
|
||
# role="--no-owner --role $owner";
|
||
PG_PARAM_ROLE=("--no-owner" "--role" "$owner");
|
||
fi;
|
||
;;
|
||
# d|database)
|
||
d)
|
||
if [ -z "$database" ]; then
|
||
database=$OPTARG;
|
||
fi;
|
||
;;
|
||
# e|encoding)
|
||
e)
|
||
if [ -z "$encoding" ]; then
|
||
encoding=$OPTARG;
|
||
fi;
|
||
;;
|
||
# f|file)
|
||
f)
|
||
if [ -z "$file" ]; then
|
||
file=$OPTARG;
|
||
fi;
|
||
;;
|
||
# h|hostname)
|
||
h)
|
||
if [ -z "$_host" ]; then
|
||
# if local it is socket
|
||
if [ "$OPTARG" != "local" ]; then
|
||
PG_PARAMS+=("-h" "${OPTARG}");
|
||
fi;
|
||
_host=$OPTARG;
|
||
fi;
|
||
;;
|
||
# p|port)
|
||
p)
|
||
if [ -z "$port" ]; then
|
||
PG_PARAMS+=("-p" "${OPTARG}");
|
||
_port=$OPTARG;
|
||
fi;
|
||
;;
|
||
# i|ident)
|
||
i)
|
||
if [ -z "$ident" ]; then
|
||
ident=$OPTARG;
|
||
fi;
|
||
;;
|
||
# j|jobs)
|
||
j)
|
||
MAX_JOBS=${OPTARG};
|
||
;;
|
||
# q|quiet)
|
||
q)
|
||
NO_ASK=1;
|
||
;;
|
||
# r|redhat)
|
||
r)
|
||
REDHAT=1;
|
||
;;
|
||
# n|dry-run)
|
||
n)
|
||
DRY_RUN=1;
|
||
;;
|
||
# s|schema-only)
|
||
s)
|
||
SCHEMA_ONLY=1
|
||
schema='-s';
|
||
;;
|
||
# m|help)
|
||
m)
|
||
usage;
|
||
exit 0;
|
||
;;
|
||
\?)
|
||
echo -e "\n Option does not exist: $OPTARG\n";
|
||
usage;
|
||
exit 1;
|
||
;;
|
||
esac;
|
||
done;
|
||
|
||
if [ "${ERROR}" -eq 1 ]; then
|
||
exit 0;
|
||
fi;
|
||
|
||
# check that the port is a valid number
|
||
if ! [[ "$_port" =~ $PORT_REGEX ]]; then
|
||
echo "The port needs to be a valid number: $_port";
|
||
exit 1;
|
||
fi;
|
||
NUMBER_REGEX="^[0-9]{1,}$";
|
||
# find the max allowed jobs based on the cpu count
|
||
# because setting more than this is not recommended
|
||
_max_jobs=$(nproc --all);
|
||
# if the MAX_JOBS is not number or smaller 1 or greate _max_jobs
|
||
if [ -n "${MAX_JOBS}" ]; then
|
||
# check that it is a valid number
|
||
if ! [[ "$MAX_JOBS" =~ $NUMBER_REGEX ]]; then
|
||
echo "Please enter a number for the -j option";
|
||
exit 1;
|
||
fi;
|
||
if [ "${MAX_JOBS}" -lt 1 ] || [ "${MAX_JOBS}" -gt "${_max_jobs}" ]; then
|
||
echo "The value for the jobs option -j cannot be smaller than 1 or bigger than ${_max_jobs}";
|
||
exit 1;
|
||
fi;
|
||
else
|
||
# auto set the MAX_JOBS based on the cpu count
|
||
MAX_JOBS=${_max_jobs};
|
||
fi;
|
||
|
||
# check if we have the 'bc' command available or not
|
||
if [ -f "${BC}" ]; then
|
||
BC_OK=1;
|
||
else
|
||
BC_OK=0;
|
||
fi;
|
||
|
||
if [ ! -f "${file}" ]; then
|
||
echo "File name needs to be provided or file could not be found";
|
||
exit 1;
|
||
fi;
|
||
|
||
# METHOD: convert_time
|
||
# PARAMS: timestamp in seconds or with milliseconds (nnnn.nnnn)
|
||
# RETURN: formated string with human readable time (d/h/m/s)
|
||
# CALL : var=$(convert_time $timestamp);
|
||
# DESC : converts a timestamp or a timestamp with float milliseconds to a human readable format
|
||
# output is in days/hours/minutes/seconds
|
||
function convert_time
|
||
{
|
||
timestamp=${1};
|
||
# round to four digits for ms
|
||
timestamp=$(printf "%1.4f" "$timestamp");
|
||
# get the ms part and remove any leading 0
|
||
ms=$(echo "${timestamp}" | cut -d "." -f 2 | sed -e 's/^0*//');
|
||
timestamp=$(echo "${timestamp}" | cut -d "." -f 1);
|
||
timegroups=(86400 3600 60 1); # day, hour, min, sec
|
||
timenames=("d" "h" "m" "s"); # day, hour, min, sec
|
||
output=( );
|
||
time_string='';
|
||
for timeslice in "${timegroups[@]}"; do
|
||
# floor for the division, push to output
|
||
if [ ${BC_OK} -eq 1 ]; then
|
||
output[${#output[*]}]=$(echo "${timestamp}/${timeslice}" | bc);
|
||
timestamp=$(echo "${timestamp}%${timeslice}" | bc);
|
||
else
|
||
output[${#output[*]}]=$(awk "BEGIN {printf \"%d\", ${timestamp}/${timeslice}}");
|
||
timestamp=$(awk "BEGIN {printf \"%d\", ${timestamp}%${timeslice}}");
|
||
fi;
|
||
done;
|
||
|
||
for ((i=0; i<${#output[@]}; i++)); do
|
||
if [ "${output[$i]}" -gt 0 ] || [ -n "$time_string" ]; then
|
||
if [ -n "${time_string}" ]; then
|
||
time_string=${time_string}" ";
|
||
fi;
|
||
time_string=${time_string}${output[$i]}${timenames[$i]};
|
||
fi;
|
||
done;
|
||
# milliseconds must be filled, but we also check that they are non "nan" string
|
||
# that can appear in the original value
|
||
if [ -n "${ms}" ] && [ "${ms}" != "nan" ]; then
|
||
if [ "${ms}" -gt 0 ]; then
|
||
time_string="${time_string} ${ms}ms";
|
||
fi;
|
||
fi;
|
||
# just in case the time is 0
|
||
if [ -z "${time_string}" ]; then
|
||
time_string="0s";
|
||
fi;
|
||
echo -n "${time_string}";
|
||
}
|
||
|
||
# for the auto find, we need to get only the filename, and therefore remove all path info
|
||
db_file=$(basename "$file");
|
||
# if file is set and exist, but no owner or database are given, use the file name data to get user & database
|
||
if [ -r "$file" ] && { [ ! "$owner" ] || [ ! "$database" ] || [ ! "$encoding" ]; }; then
|
||
# file name format is
|
||
# <database>.<owner>.<encoding>.<db type>-<version>_<host>_<port>_<date>_<time>_<sequence>
|
||
# we only are interested in the first two
|
||
_database=$(echo "${db_file}" | cut -d "." -f 1);
|
||
_owner=$(echo "${db_file}" | cut -d "." -f 2);
|
||
__encoding=$(echo "${db_file}" | cut -d "." -f 3);
|
||
# set the others as optional
|
||
# the last _ is for version 10 or higher
|
||
# db version, without prefix of DB type
|
||
_ident=$(echo "${db_file}" | cut -d "." -f 4 | cut -d "-" -f 2 | cut -d "_" -f 1);
|
||
cut_pos=4;
|
||
# if this is < 10 then we need the second part too
|
||
if [ "${_ident}" -lt 10 ]; then
|
||
# db version, second part (after .)
|
||
_ident=$_ident'.'$(echo "$db_file" | cut -d "." -f 5 | cut -d "_" -f 1);
|
||
cut_pos=5;
|
||
fi;
|
||
__host=$(echo "${db_file}" | cut -d "." -f ${cut_pos} | cut -d "_" -f 2);
|
||
__port=$(echo "${db_file}" | cut -d "." -f ${cut_pos} | cut -d "_" -f 3);
|
||
# if any of those are not set, override by the file name settings
|
||
if [ ! "$owner" ]; then
|
||
owner=$_owner;
|
||
fi;
|
||
if [ ! "$database" ]; then
|
||
database=$_database;
|
||
fi;
|
||
# port hast to be a valid number, at least 4 digits long and maximum 5 digits
|
||
if [ ! "$port" ] && [[ $__port =~ $PORT_REGEX ]] ; then
|
||
port='-p '$__port;
|
||
_port=$__port;
|
||
fi;
|
||
# unless it is local and no command line option is set, set the target connection host
|
||
if [ ! "$host" ] && [ "$__host" != "local" ] && [ "$_host" != "local" ]; then
|
||
host='-h '$__host;
|
||
_host=$__host;
|
||
fi;
|
||
if [ ! "$encoding" ]; then
|
||
if [ -n "$__encoding" ]; then
|
||
encoding=$__encoding;
|
||
else
|
||
encoding=$_encoding;
|
||
fi;
|
||
fi;
|
||
if [ ! "$ident" ]; then
|
||
ident=$_ident;
|
||
fi;
|
||
fi;
|
||
|
||
# if no user or database, exist
|
||
if [ ! "$file" ] || [ ! -f "$file" ]; then
|
||
echo "The file has not been set or the file given could not be found.";
|
||
exit 1;
|
||
fi;
|
||
if [ ! "$owner" ] || [ ! "$encoding" ] || [ ! "$database" ]; then
|
||
echo "The Owner, database name and encoding could not be set automatically, the have to be given as command line options.";
|
||
exit 1;
|
||
fi;
|
||
|
||
if [ "$REDHAT" -eq 1 ]; then
|
||
# Debian base path
|
||
PG_BASE_PATH="/usr/pgsql-";
|
||
else
|
||
# Redhat base path (for non official ones would be '/usr/pgsql-'
|
||
PG_BASE_PATH="/usr/lib/postgresql/";
|
||
fi;
|
||
|
||
# if no ident is given, try to find the default one, if not fall back to pre set one
|
||
if [ -n "$ident" ]; then
|
||
PG_PATH="${PG_BASE_PATH}${ident}/bin/";
|
||
if [ ! -d "$PG_PATH" ]; then
|
||
ident='';
|
||
fi;
|
||
else
|
||
# try to run psql from default path and get the version number
|
||
ident=$(
|
||
pgv=$(
|
||
"pg_dump" --version | grep "pg_dump" | cut -d " " -f 3
|
||
);
|
||
if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then
|
||
echo "${pgv}" | cut -d "." -f 1;
|
||
else
|
||
echo "${pgv}" | cut -d "." -f 1,2;
|
||
fi
|
||
);
|
||
if [ -z "$ident" ]; then
|
||
# hard setting
|
||
ident='15';
|
||
fi;
|
||
PG_PATH="${PG_BASE_PATH}${ident}/bin/'";
|
||
fi;
|
||
|
||
PG_DROPDB="${PG_PATH}dropdb";
|
||
PG_CREATEDB="${PG_PATH}createdb";
|
||
PG_CREATELANG="${PG_PATH}createlang";
|
||
PG_RESTORE="${PG_PATH}pg_restore";
|
||
PG_PSQL="${PG_PATH}psql";
|
||
TEMP_FILE="temp";
|
||
LOG_FILE_EXT="${database}.$(date +"%Y%m%d_%H%M%S").log";
|
||
|
||
# core abort if no core files found
|
||
if [ ! -f "$PG_PSQL" ] || [ ! -f "$PG_DROPDB" ] || [ ! -f "$PG_CREATEDB" ] || [ ! -f "$PG_RESTORE" ]; then
|
||
echo "One of the core binaries (psql, pg_dump, createdb, pg_restore) could not be found.";
|
||
echo "Search Path: ${PG_PATH}";
|
||
echo "Perhaps manual ident set with -i is necessary";
|
||
echo "Backup aborted";
|
||
exit 0;
|
||
fi;
|
||
|
||
# check if port / host settings are OK
|
||
# if I cannot connect with user postgres to template1, the restore won't work
|
||
_PG_PARAMS=("${PG_PARAMS[@]}");
|
||
_PG_PARAMS+=("-U" "postgres" "template1" "-q" "-t" "-X" "-A" "-F" "," "-c" "SELECT version();");
|
||
_output=$("${PG_PSQL}" "${_PG_PARAMS[@]}" 2>&1);
|
||
found=$(echo "$_output" | grep "PostgreSQL");
|
||
# if the output does not have the PG version string, we have an error and abort
|
||
if [ -z "$found" ]; then
|
||
echo "Cannot connect to the database: $_output";
|
||
exit 1;
|
||
fi;
|
||
if [ $DRY_RUN -eq 1 ]; then
|
||
echo "**** [DRY RUN] ****";
|
||
fi;
|
||
echo "Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'";
|
||
if [ $SCHEMA_ONLY -eq 1 ]; then
|
||
echo "!!!!!!! WILL ONLY RESTORE SCHEMA, NO DATA !!!!!!!";
|
||
fi;
|
||
if [ $NO_ASK -eq 1 ]; then
|
||
go='yes';
|
||
else
|
||
echo "Continue? type 'yes'";
|
||
read -r go;
|
||
fi;
|
||
if [ "$go" != 'yes' ]; then
|
||
echo "Aborted";
|
||
exit;
|
||
else
|
||
start_time=$(date +"%F %T");
|
||
START=$(date +'%s');
|
||
echo "Drop DB $database [$_host:$_port] @ $start_time";
|
||
# DROP DATABASE
|
||
_PG_PARAMS=("${PG_PARAMS[@]}");
|
||
_PG_PARAMS+=("-U" "postgres" "${database}");
|
||
if [ $DRY_RUN -eq 0 ]; then
|
||
"${PG_DROPDB}" "${_PG_PARAMS[@]}";
|
||
else
|
||
echo "${PG_DROPDB} ${_PG_PARAMS[*]}";
|
||
fi;
|
||
# CREATE DATABASE
|
||
echo "Create DB $database with $owner and encoding $encoding on [$_host:$_port] @ $(date +"%F %T")";
|
||
_PG_PARAMS=("${PG_PARAMS[@]}");
|
||
_PG_PARAMS+=("-U" "postgres" "-O" "${owner}" "-E" "${encoding}" "-T" "${TEMPLATEDB}" "${database}");
|
||
if [ $DRY_RUN -eq 0 ]; then
|
||
"${PG_CREATEDB}" "${_PG_PARAMS[@]}";
|
||
else
|
||
echo "${PG_CREATEDB} ${_PG_PARAMS[*]}";
|
||
fi;
|
||
# CREATE plpgsql LANG
|
||
if [ -f "$PG_CREATELANG" ]; then
|
||
_PG_PARAMS=("${PG_PARAMS[@]}");
|
||
_PG_PARAMS+=("-U" "postgres" "plpgsql" "${database}");
|
||
echo "Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")";
|
||
if [ $DRY_RUN -eq 0 ]; then
|
||
"${PG_CREATELANG}" "${_PG_PARAMS[@]}";
|
||
else
|
||
echo "${PG_CREATELANG} ${_PG_PARAMS[*]}";
|
||
fi;
|
||
fi;
|
||
# RESTORE DATA
|
||
echo "Restore data from $file to DB $database on [$_host:$_port] with Jobs $MAX_JOBS @ $(date +"%F %T")";
|
||
_PG_PARAMS=("${PG_PARAMS[@]}");
|
||
_PG_PARAMS+=("${PG_PARAM_ROLE[@]}")
|
||
_PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "${schema}" "-j" "${MAX_JOBS}" "${file}");
|
||
if [ $DRY_RUN -eq 0 ]; then
|
||
"${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"restore_errors.${LOG_FILE_EXT}";
|
||
else
|
||
echo "${PG_RESTORE} ${_PG_PARAMS[*]} 2>restore_errors.${LOG_FILE_EXT}";
|
||
fi;
|
||
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump
|
||
# it does not dump the default public ACL so the owner of the DB cannot access the data, check if the ACL dump is missing and do a basic restore
|
||
if ! "${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
|
||
echo "Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
|
||
# grant usage on schema public to public;
|
||
_PG_PARAMS=("${PG_PARAMS[@]}");
|
||
_PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
|
||
"${PG_PSQL}" "${_PG_PARAMS[@]}";
|
||
# grant create on schema public to public;
|
||
_PG_PARAMS=("${PG_PARAMS[@]}");
|
||
_PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
|
||
"${PG_PSQL}" "${_PG_PARAMS[@]}";
|
||
fi;
|
||
# SEQUENCE RESET DATA COLLECTION
|
||
echo "Resetting all sequences from DB $database [$_host:$_port] @ $(date +"%F %T")";
|
||
reset_query="SELECT 'SELECT SETVAL(' ||quote_literal(S.relname)|| ', MAX(' ||quote_ident(C.attname)|| ') ) FROM ' ||quote_ident(T.relname)|| ';' FROM pg_class AS S, pg_depend AS D, pg_class AS T, pg_attribute AS C WHERE S.relkind = 'S' AND S.oid = D.objid AND D.refobjid = T.oid AND D.refobjid = C.attrelid AND D.refobjsubid = C.attnum ORDER BY S.relname;";
|
||
_PG_PARAMS=("${PG_PARAMS[@]}");
|
||
_PG_PARAMS+=("-U" "postgres" "-Atq" "-o" "${TEMP_FILE}" "-c" "${reset_query}" "${database}");
|
||
_PG_PARAMS_OUT=("${PG_PARAMS[@]}");
|
||
_PG_PARAMS_OUT+=("-U" "postgres" "-e" "-f" "${TEMP_FILE}" "${database}");
|
||
if [ $DRY_RUN -eq 0 ]; then
|
||
"${PG_PSQL}" "${_PG_PARAMS[@]}";
|
||
"${PG_PSQL}" "${_PG_PARAMS_OUT[@]}" 1>"output_sequence.${LOG_FILE_EXT}" 2>"errors_sequence.${database}.${LOG_FILE_EXT}";
|
||
rm "${TEMP_FILE}";
|
||
else
|
||
echo "${PG_PSQL} ${_PG_PARAMS[*]}";
|
||
echo "${PG_PSQL} ${_PG_PARAMS_OUT[*]} 1>output_sequence.${LOG_FILE_EXT} 2>errors_sequence.${database}.${LOG_FILE_EXT}";
|
||
fi;
|
||
echo "Restore of data $file for DB $database [$_host:$_port] finished";
|
||
DURATION=$(($(date "+%s")-START));
|
||
echo "Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION})";
|
||
echo "=== END RESTORE" >>"restore_errors.${LOG_FILE_EXT}";
|
||
fi;
|