initial uploads

This commit is contained in:
Matthew Saunders Brown 2021-01-21 13:58:30 -08:00
parent 6504bd0e0f
commit 2b232c41f9
5 changed files with 678 additions and 0 deletions

165
bashup-backup-mysql.sh Normal file
View File

@ -0,0 +1,165 @@
#!/bin/bash
#
# Bashup - A set of bash scripts for managing backups.
# https://git.stack-source.com/msb/bashup
# MIT License Copyright (c) 2021 Matthew Saunders Brown
# retention vars
retention_years=0;
retention_months=3;
retention_weeks=5;
retention_days=7;
# backup storage directory
backup_storage_dir='/mnt/backups';
# mysql config file that contains 'host' 'user' 'password' vars
defaults_extra_file='/etc/mysql/debian.cnf';
# list of databases to skip
exclusions=('information_schema' 'performance_schema' 'sys' 'wsrep');
# check for local config, which can be used to override any of the above
if [[ -f /usr/local/etc/bashup.cnf ]]; then
source /usr/local/etc/bashup.cnf
fi
if [ ! -d $backup_storage_dir ]; then
echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist."
exit 1
fi
# check if backup_storage_dir is a mount in fstab
grep -qs " $backup_storage_dir " /etc/fstab
if [ $? -eq 0 ]; then
# check if backup_storage_dir is already mounted
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
# attempt to mount backups
mount $backup_storage_dir
# re-check for backups mount
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
echo "ERROR: failed to mount $backup_storage_dir"
exit 1
fi
fi
fi
# get todays date (backup dir name)
today=$(date +%Y%m%d)
# get list of existing backups
existing_backups=($(ls $backup_storage_dir|grep -v lost+found))
# remove today from existing backups, if it exists. we do other checks later to avoid re-doing backups
if [[ " ${existing_backups[@]} " =~ " ${today} " ]]; then
unset 'existing_backups[-1]';
fi
# create retention array
declare -a retention_array
# set retention days
if [ $retention_days -gt 0 ]; then
i="0"
while [ $i -lt $retention_days ]; do
DATE=`date --date="$i day ago" +%Y%m%d`
retention_array[$DATE]="$DATE"
i=$[$i+1]
done
fi
# set retention weeks
if [ $retention_weeks -gt 0 ]; then
i="0"
while [ $i -lt $retention_weeks ]; do
i=$[$i+1]
DATE=`date --date="sunday - $i week" +%Y%m%d`
retention_array[$DATE]="$DATE"
done
fi
# set retention months
if [ $retention_months -gt 0 ]; then
i="0"
while [ $i -lt $retention_months ]; do
DATE=`date --date="$i month ago" +%Y%m01`
retention_array[$DATE]="$DATE"
i=$[$i+1]
done
fi
# set retention years
if [ $retention_years -gt 0 ]; then
i="0"
while [ $i -lt $retention_years ]; do
DATE=`date --date="$i year ago" +%Y0101`
retention_array[$DATE]="$DATE"
i=$[$i+1]
done
fi
# create backup for today
if [[ " ${retention_array[@]} " =~ " ${today} " ]]; then
# create backup (date) dir if it doesn't already exist
if [ ! -d "$backup_storage_dir/$today" ]; then
mkdir $backup_storage_dir/$today
fi
# only proceed if mysql has not already been backed up
if [[ ! -d $backup_storage_dir/$today/mysql ]]; then
mkdir $backup_storage_dir/$today/mysql
mysqladmin --defaults-extra-file=$defaults_extra_file refresh
# create array of all existing databases
databases=($(mysql --defaults-extra-file=$defaults_extra_file -E -e 'show databases;'|grep : |awk '{ print $2 }' |tr '\n' ' '));
for database in "${databases[@]}"; do
if [[ " ${exclusions[@]} " =~ " ${database} " ]]; then
# do nothing, db is in exclusions array
one=1;
else
mysqldump --defaults-extra-file=$defaults_extra_file --opt --quote-names --events --databases $database | gzip > $backup_storage_dir/$today/mysql/$database.sql.gz
fi
done
fi
fi
# remove expired backups
for existing_backup in "${existing_backups[@]}"; do
if [[ " ${retention_array[@]} " =~ " ${existing_backup} " ]]; then
# keep $existing_backup, do nothing
one=1;
else
if [[ -d $backup_storage_dir/$existing_backup/mysql ]]; then
rm -r $backup_storage_dir/$existing_backup/mysql
fi
if [ -z "$(ls -A $backup_storage_dir/$existing_backup)" ]; then
rm -r $backup_storage_dir/$existing_backup
fi
fi
done
# check if backup_storage_dir is mounted and unmount if so
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -eq 0 ]; then
umount $backup_storage_dir
fi
exit 0;

154
bashup-backup-pdns.sh Normal file
View File

@ -0,0 +1,154 @@
#!/bin/bash
#
# Bashup - A set of bash scripts for managing backups.
# https://git.stack-source.com/msb/bashup
# MIT License Copyright (c) 2021 Matthew Saunders Brown
# retention vars
retention_years=0;
retention_months=3;
retention_weeks=5;
retention_days=7;
# backup storage directory
backup_storage_dir='/mnt/backups';
# check for local config, which can be used to override any of the above
if [[ -f /usr/local/etc/bashup.cnf ]]; then
source /usr/local/etc/bashup.cnf
fi
if [ ! -d $backup_storage_dir ]; then
echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist."
exit 1
fi
# check if backup_storage_dir is a mount in fstab
grep -qs " $backup_storage_dir " /etc/fstab
if [ $? -eq 0 ]; then
# check if backup_storage_dir is already mounted
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
# attempt to mount backups
mount $backup_storage_dir
# re-check for backups mount
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
echo "ERROR: failed to mount $backup_storage_dir"
exit 1
fi
fi
fi
# get todays date (backup dir name)
today=$(date +%Y%m%d)
# get list of existing backups
existing_backups=($(ls $backup_storage_dir|grep -v lost+found))
# remove today from existing backups, if it exists. we do other checks later to avoid re-doing backups
if [[ " ${existing_backups[@]} " =~ " ${today} " ]]; then
unset 'existing_backups[-1]';
fi
# create retention array
declare -a retention_array
# set retention days
if [ $retention_days -gt 0 ]; then
i="0"
while [ $i -lt $retention_days ]; do
DATE=`date --date="$i day ago" +%Y%m%d`
retention_array[$DATE]="$DATE"
i=$[$i+1]
done
fi
# set retention weeks
if [ $retention_weeks -gt 0 ]; then
i="0"
while [ $i -lt $retention_weeks ]; do
i=$[$i+1]
DATE=`date --date="sunday - $i week" +%Y%m%d`
retention_array[$DATE]="$DATE"
done
fi
# set retention months
if [ $retention_months -gt 0 ]; then
i="0"
while [ $i -lt $retention_months ]; do
DATE=`date --date="$i month ago" +%Y%m01`
retention_array[$DATE]="$DATE"
i=$[$i+1]
done
fi
# set retention years
if [ $retention_years -gt 0 ]; then
i="0"
while [ $i -lt $retention_years ]; do
DATE=`date --date="$i year ago" +%Y0101`
retention_array[$DATE]="$DATE"
i=$[$i+1]
done
fi
# create backup for today
if [[ " ${retention_array[@]} " =~ " ${today} " ]]; then
# create backup (date) dir if it doesn't already exist
if [ ! -d "$backup_storage_dir/$today" ]; then
mkdir $backup_storage_dir/$today
fi
# only proceed if pdns zones have not already been backed up
if [[ ! -d $backup_storage_dir/$today/pdns ]]; then
mkdir $backup_storage_dir/$today/pdns
zones=(`/usr/bin/pdnsutil list-all-zones`)
for zone in "${zones[@]}"; do
/usr/bin/pdnsutil list-zone $zone > "$backup_storage_dir/$today/pdns/$zone.zone"
done
fi
fi
# remove expired backups
for existing_backup in "${existing_backups[@]}"; do
if [[ " ${retention_array[@]} " =~ " ${existing_backup} " ]]; then
# keep $existing_backup, do nothing
one=1;
else
if [[ -d $backup_storage_dir/$existing_backup/pdns ]]; then
rm -r $backup_storage_dir/$existing_backup/pdns
fi
if [ -z "$(ls -A $backup_storage_dir/$existing_backup)" ]; then
rm -r $backup_storage_dir/$existing_backup
fi
fi
done
# check if backup_storage_dir is mounted and unmount if so
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -eq 0 ]; then
umount $backup_storage_dir
fi
exit 0;

46
bashup-list-backups.sh Normal file
View File

@ -0,0 +1,46 @@
#!/bin/bash
#
# Bashup - A set of bash scripts for managing backups.
# https://git.stack-source.com/msb/bashup
# MIT License Copyright (c) 2021 Matthew Saunders Brown
# backups are stored here
backup_storage_dir='/mnt/backups';
# check for local config, which can be used to override any of the above
if [[ -f /usr/local/etc/bashup.cnf ]]; then
source /usr/local/etc/bashup.cnf
fi
if [ ! -d $backup_storage_dir ]; then
echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist."
exit 1
fi
# check if backup_storage_dir is a mount in fstab
grep -qs " $backup_storage_dir " /etc/fstab
if [ $? -eq 0 ]; then
# check if backup_storage_dir is already mounted
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
# attempt to mount backups
mount $backup_storage_dir
# re-check for backups mount
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
echo "ERROR: failed to mount $backup_storage_dir"
exit 1
fi
fi
fi
# output list of backups (dates)
ls -1 $backup_storage_dir | grep -v lost+found
# check if backup_storage_dir is mounted and unmount if so
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -eq 0 ]; then
umount $backup_storage_dir
fi
exit 0

164
bashup-restore-files.sh Normal file
View File

@ -0,0 +1,164 @@
#!/bin/bash
#
# Bashup - A set of bash scripts for managing backups.
# https://git.stack-source.com/msb/bashup
# MIT License Copyright (c) 2021 Matthew Saunders Brown
# backup storage directory
backup_storage_dir='/mnt/backups';
# directories to be backed up
backup_dirs=('/etc' '/home' '/srv' '/root' '/usr/local' '/var/www');
# check for local config, which can be used to override any of the above
if [[ -f /usr/local/etc/bashup.cnf ]]; then
source /usr/local/etc/bashup.cnf
fi
help()
{
thisfilename=$(basename -- "$0")
echo "$thisfilename"
echo "Restore file(s) from backup."
echo ""
echo "usage: $thisfilename [-b BACKUPDATE] [-f PATH]"
echo ""
echo " -b BACKUPDATE Backup date/archive to restore from."
echo " -p PATH Path to file or directory to restore."
echo " -h Print this help."
echo " You will be prompted to select backup date and file"
echo " or directory if not specified on the command line"
echo " with the above options."
exit
}
if [ ! -d $backup_storage_dir ]; then
echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist."
exit 1
fi
# check if backup_storage_dir is a mount in fstab
grep -qs " $backup_storage_dir " /etc/fstab
if [ $? -eq 0 ]; then
# check if backup_storage_dir is already mounted
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
# attempt to mount backups
mount $backup_storage_dir
# re-check for backups mount
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
echo "ERROR: failed to mount $backup_storage_dir"
exit 1
fi
fi
fi
# set any options that were passed
while getopts "b:p:h" opt; do
case "${opt}" in
h )
help
exit;;
b )
backup=${OPTARG}
;;
p )
pathtorestore=${OPTARG}
;;
\? )
echo "Invalid option: $OPTARG" 1>&2
exit 1;;
: )
echo "Invalid option: $OPTARG requires an argument" 1>&2
exit 1;;
esac
done
# get list of backups (dates)
existing_backups=($(ls $backup_storage_dir|grep -v lost+found))
# prompt if backup was not set on command line
if [ -z "$backup" ] ; then
# select backup to restore from
PS3="Select number of backup to restore from: "
existing_backups_len=${#existing_backups[@]}
select backup in ${existing_backups[@]}
do
if [[ $REPLY -gt 0 ]] && [[ $REPLY -le $existing_backups_len ]]; then
echo $backup
break
else
echo "ERROR: Invalid entry, try again."
fi
done
fi
# set path to file or directory to restore
if [ -z "$pathtorestore" ] ; then
if [ -d $backup_storage_dir/$backup/files ]; then
# select backup file or directory to restore
PS3="Select number of backup directory to restore from: "
backup_dirs_len=${#backup_dirs[@]}
select backup_dir in ${backup_dirs[@]}
do
if [[ $REPLY -gt 0 ]] && [[ $REPLY -le $backup_dirs_len ]]; then
echo "Complete path to specific file or directory to restore."
read -e -p "Just press enter to recursively restore entire dir: " -i $backup_storage_dir/$backup/files$backup_dir pathtobackup
pathtorestore=${pathtobackup#"$backup_storage_dir/$backup/files"}
break
else
echo "ERROR: Invalid entry, try again."
fi
done
else
echo "ERROR: Backup dir for $backup/files/ does not exist."
exit 1
fi
else
pathtobackup="$backup_storage_dir/$backup/files$pathtorestore"
fi
if [ -d $pathtobackup ]; then
# restore is a direcotry, make sure paths end in a slash (/) for proper rsync
if [[ ! "$pathtobackup" == */ ]]; then
pathtobackup="$pathtobackup/"
fi
if [[ ! "$pathtorestore" == */ ]]; then
pathtorestore="$pathtorestore/"
fi
elif [ ! -f $pathtobackup ]; then
# restore is not a file nor a direcotry, does not exist
echo "ERROR: Backup file/directory ($pathtobackup) does not exist."
exit 1
fi
# make sure parent dir of destination exists
if [ ! -d $(/usr/bin/dirname $pathtorestore) ]; then
echo "ERROR: Parent directory missing for restore location."
exit 1
fi
# make sure pathtorestore is for a file/dir *within* one of the backup_dirs
verify_back_dir=FALSE
for dir in "${backup_dirs[@]}"; do
if [[ $pathtorestore == $dir ]] || [[ $pathtorestore == $dir* ]]; then
verify_back_dir=TRUE
fi
done
if [[ $verify_back_dir == FALSE ]]; then
echo "ERROR: File or directory to restore does not exist within one of the backed up directories."
exit 1
fi
# perform restore
# Still in testing mode, display command instead of running it
"To restore $pathtorestore from $backup run this command:"
echo "/usr/bin/rsync -vn --archive --numeric-ids --one-file-system --delete $pathtobackup $pathtorestore"
# check if backup_storage_dir is mounted and unmount if so
/usr/bin/grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -eq 0 ]; then
/usr/bin/umount $backup_storage_dir
fi
exit 0

149
bashup-restore-mysql.sh Normal file
View File

@ -0,0 +1,149 @@
#!/bin/bash
#
# Bashup - A set of bash scripts for managing backups.
# https://git.stack-source.com/msb/bashup
# MIT License Copyright (c) 2021 Matthew Saunders Brown
# backup directory
backup_storage_dir='/mnt/backups';
# config file that contains [client] config with 'host' 'user' 'password' settings
defaults_extra_file='/etc/mysql/debian.cnf';
# check for local config, which can be used to override any of the above
if [[ -f /usr/local/etc/bashup.cnf ]]; then
source /usr/local/etc/bashup.cnf
fi
help()
{
thisfilename=$(basename -- "$0")
echo "$thisfilename"
echo "Restore database from backup."
echo ""
echo "usage: $thisfilename [-b BACKUPDATE] [-d DATABASE]"
echo ""
echo " -b BACKUPDATE Backup date/archive to restore from."
echo " -d DATABASE Database to restore."
echo " -h Print this help."
echo " You will be prompted to select backup date and/or"
echo " database name from a list of available options"
echo " if they are not specified on the command line."
exit
}
if [ ! -d $backup_storage_dir ]; then
echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist."
exit 1
fi
# check if backup_storage_dir is a mount in fstab
grep -qs " $backup_storage_dir " /etc/fstab
if [ $? -eq 0 ]; then
# check if backup_storage_dir is already mounted
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
# attempt to mount backups
mount $backup_storage_dir
# re-check for backups mount
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -ne 0 ]; then
echo "ERROR: failed to mount $backup_storage_dir"
exit 1
fi
fi
fi
# set any options that were passed
while getopts "b:d:h" opt; do
case "${opt}" in
h )
help
exit;;
b )
backup=${OPTARG}
;;
d )
database=${OPTARG}
;;
\? )
echo "Invalid option: $OPTARG" 1>&2
exit 1;;
: )
echo "Invalid option: $OPTARG requires an argument" 1>&2
exit 1;;
esac
done
# get list of backups (dates)
existing_backups=($(ls $backup_storage_dir|grep -v lost+found))
# prompt if backup was not set on command line
if [ -z "$backup" ] ; then
# select backup to restore from
PS3="Select number of backup to restore from: "
existing_backups_len=${#existing_backups[@]}
select backup in ${existing_backups[@]}
do
if [[ $REPLY -gt 0 ]] && [[ $REPLY -le $existing_backups_len ]]; then
echo $backup
break
else
echo "ERROR: Invalid entry, try again."
fi
done
fi
# set dump if database was set via command option
if [ ! -z "$database" ] ; then
dump="$database.sql.gz"
else
if [ -d $backup_storage_dir/$backup/mysql ]; then
# get list of dbs in backup
existing_dumps=($(ls $backup_storage_dir/$backup/mysql))
# set array with names of dbs (without .sql.gz suffix)
declare -a existing_dumps_db_names
for existing_dump in "${existing_dumps[@]}"; do
existing_dumps_db_name="$(basename $existing_dump .sql.gz)"
existing_db_names+=("$existing_dumps_db_name")
done
# select database dump to restore
PS3="Select number of database to restore: "
existing_db_names_len=${#existing_db_names[@]}
select database in ${existing_db_names[@]}
do
if [[ $REPLY -gt 0 ]] && [[ $REPLY -le $existing_db_names_len ]]; then
dump=$database.sql.gz
break
else
echo "ERROR: Invalid entry, try again."
fi
done
else
echo "ERROR: Backup dir for $backup/mysql/ does not exist."
exit 1
fi
fi
# check that dump exists and restore it now
if [ -d $backup_storage_dir/$backup ]; then
if [ -f $backup_storage_dir/$backup/mysql/$dump ]; then
# Still in testing mode, display command instead of running it
echo "To restore database $database from backup $backup run this command:"
echo "/usr/bin/zcat $backup_storage_dir/$backup/mysql/$dump | mysql --defaults-extra-file=$defaults_extra_file $database"
# echo "SUCCESS: Database $database from backup $backup has been restored."
else
echo "ERROR: Dump for database $database does not exist in the $backup backup dir."
exit 1
fi
else
echo "ERROR: Backup dir for $backup does not exist."
exit 1
fi
# check if backup_storage_dir is mounted and unmount if so
grep -qs " $backup_storage_dir " /proc/mounts
if [ $? -eq 0 ]; then
umount $backup_storage_dir
fi
exit