From 2b232c41f9ce3506e976a42e039e496d34035999 Mon Sep 17 00:00:00 2001 From: Matthew Saunders Brown Date: Thu, 21 Jan 2021 13:58:30 -0800 Subject: [PATCH] initial uploads --- bashup-backup-mysql.sh | 165 ++++++++++++++++++++++++++++++++++++++++ bashup-backup-pdns.sh | 154 +++++++++++++++++++++++++++++++++++++ bashup-list-backups.sh | 46 +++++++++++ bashup-restore-files.sh | 164 +++++++++++++++++++++++++++++++++++++++ bashup-restore-mysql.sh | 149 ++++++++++++++++++++++++++++++++++++ 5 files changed, 678 insertions(+) create mode 100644 bashup-backup-mysql.sh create mode 100644 bashup-backup-pdns.sh create mode 100644 bashup-list-backups.sh create mode 100644 bashup-restore-files.sh create mode 100644 bashup-restore-mysql.sh diff --git a/bashup-backup-mysql.sh b/bashup-backup-mysql.sh new file mode 100644 index 0000000..92ec937 --- /dev/null +++ b/bashup-backup-mysql.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# +# Bashup - A set of bash scripts for managing backups. +# https://git.stack-source.com/msb/bashup +# MIT License Copyright (c) 2021 Matthew Saunders Brown + +# retention vars +retention_years=0; +retention_months=3; +retention_weeks=5; +retention_days=7; +# backup storage directory +backup_storage_dir='/mnt/backups'; +# mysql config file that contains 'host' 'user' 'password' vars +defaults_extra_file='/etc/mysql/debian.cnf'; +# list of databases to skip +exclusions=('information_schema' 'performance_schema' 'sys' 'wsrep'); + +# check for local config, which can be used to override any of the above +if [[ -f /usr/local/etc/bashup.cnf ]]; then + source /usr/local/etc/bashup.cnf +fi + +if [ ! -d $backup_storage_dir ]; then + echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist." + exit 1 +fi + +# check if backup_storage_dir is a mount in fstab +grep -qs " $backup_storage_dir " /etc/fstab +if [ $? -eq 0 ]; then + # check if backup_storage_dir is already mounted + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + # attempt to mount backups + mount $backup_storage_dir + # re-check for backups mount + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + echo "ERROR: failed to mount $backup_storage_dir" + exit 1 + fi + fi +fi + +# get todays date (backup dir name) +today=$(date +%Y%m%d) + +# get list of existing backups +existing_backups=($(ls $backup_storage_dir|grep -v lost+found)) +# remove today from existing backups, if it exists. we do other checks later to avoid re-doing backups +if [[ " ${existing_backups[@]} " =~ " ${today} " ]]; then + unset 'existing_backups[-1]'; +fi + +# create retention array +declare -a retention_array + +# set retention days +if [ $retention_days -gt 0 ]; then + i="0" + while [ $i -lt $retention_days ]; do + DATE=`date --date="$i day ago" +%Y%m%d` + retention_array[$DATE]="$DATE" + i=$[$i+1] + done +fi + +# set retention weeks +if [ $retention_weeks -gt 0 ]; then + i="0" + while [ $i -lt $retention_weeks ]; do + i=$[$i+1] + DATE=`date --date="sunday - $i week" +%Y%m%d` + retention_array[$DATE]="$DATE" + done +fi + +# set retention months +if [ $retention_months -gt 0 ]; then + i="0" + while [ $i -lt $retention_months ]; do + DATE=`date --date="$i month ago" +%Y%m01` + retention_array[$DATE]="$DATE" + i=$[$i+1] + done +fi + +# set retention years +if [ $retention_years -gt 0 ]; then + i="0" + while [ $i -lt $retention_years ]; do + DATE=`date --date="$i year ago" +%Y0101` + retention_array[$DATE]="$DATE" + i=$[$i+1] + done +fi + +# create backup for today +if [[ " ${retention_array[@]} " =~ " ${today} " ]]; then + + # create backup (date) dir if it doesn't already exist + if [ ! -d "$backup_storage_dir/$today" ]; then + + mkdir $backup_storage_dir/$today + + fi + + # only proceed if mysql has not already been backed up + if [[ ! -d $backup_storage_dir/$today/mysql ]]; then + + mkdir $backup_storage_dir/$today/mysql + mysqladmin --defaults-extra-file=$defaults_extra_file refresh + + # create array of all existing databases + databases=($(mysql --defaults-extra-file=$defaults_extra_file -E -e 'show databases;'|grep : |awk '{ print $2 }' |tr '\n' ' ')); + + for database in "${databases[@]}"; do + + if [[ " ${exclusions[@]} " =~ " ${database} " ]]; then + # do nothing, db is in exclusions array + one=1; + else + mysqldump --defaults-extra-file=$defaults_extra_file --opt --quote-names --events --databases $database | gzip > $backup_storage_dir/$today/mysql/$database.sql.gz + fi + + done + + fi + +fi + +# remove expired backups +for existing_backup in "${existing_backups[@]}"; do + + if [[ " ${retention_array[@]} " =~ " ${existing_backup} " ]]; then + + # keep $existing_backup, do nothing + one=1; + + else + + if [[ -d $backup_storage_dir/$existing_backup/mysql ]]; then + + rm -r $backup_storage_dir/$existing_backup/mysql + + fi + + if [ -z "$(ls -A $backup_storage_dir/$existing_backup)" ]; then + + rm -r $backup_storage_dir/$existing_backup + + fi + + fi + +done + +# check if backup_storage_dir is mounted and unmount if so +grep -qs " $backup_storage_dir " /proc/mounts +if [ $? -eq 0 ]; then + umount $backup_storage_dir +fi + +exit 0; diff --git a/bashup-backup-pdns.sh b/bashup-backup-pdns.sh new file mode 100644 index 0000000..e7642e1 --- /dev/null +++ b/bashup-backup-pdns.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# +# Bashup - A set of bash scripts for managing backups. +# https://git.stack-source.com/msb/bashup +# MIT License Copyright (c) 2021 Matthew Saunders Brown + +# retention vars +retention_years=0; +retention_months=3; +retention_weeks=5; +retention_days=7; +# backup storage directory +backup_storage_dir='/mnt/backups'; + +# check for local config, which can be used to override any of the above +if [[ -f /usr/local/etc/bashup.cnf ]]; then + source /usr/local/etc/bashup.cnf +fi + +if [ ! -d $backup_storage_dir ]; then + echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist." + exit 1 +fi + +# check if backup_storage_dir is a mount in fstab +grep -qs " $backup_storage_dir " /etc/fstab +if [ $? -eq 0 ]; then + # check if backup_storage_dir is already mounted + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + # attempt to mount backups + mount $backup_storage_dir + # re-check for backups mount + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + echo "ERROR: failed to mount $backup_storage_dir" + exit 1 + fi + fi +fi + +# get todays date (backup dir name) +today=$(date +%Y%m%d) + +# get list of existing backups +existing_backups=($(ls $backup_storage_dir|grep -v lost+found)) +# remove today from existing backups, if it exists. we do other checks later to avoid re-doing backups +if [[ " ${existing_backups[@]} " =~ " ${today} " ]]; then + unset 'existing_backups[-1]'; +fi + +# create retention array +declare -a retention_array + +# set retention days +if [ $retention_days -gt 0 ]; then + i="0" + while [ $i -lt $retention_days ]; do + DATE=`date --date="$i day ago" +%Y%m%d` + retention_array[$DATE]="$DATE" + i=$[$i+1] + done +fi + +# set retention weeks +if [ $retention_weeks -gt 0 ]; then + i="0" + while [ $i -lt $retention_weeks ]; do + i=$[$i+1] + DATE=`date --date="sunday - $i week" +%Y%m%d` + retention_array[$DATE]="$DATE" + done +fi + +# set retention months +if [ $retention_months -gt 0 ]; then + i="0" + while [ $i -lt $retention_months ]; do + DATE=`date --date="$i month ago" +%Y%m01` + retention_array[$DATE]="$DATE" + i=$[$i+1] + done +fi + +# set retention years +if [ $retention_years -gt 0 ]; then + i="0" + while [ $i -lt $retention_years ]; do + DATE=`date --date="$i year ago" +%Y0101` + retention_array[$DATE]="$DATE" + i=$[$i+1] + done +fi + +# create backup for today +if [[ " ${retention_array[@]} " =~ " ${today} " ]]; then + + # create backup (date) dir if it doesn't already exist + if [ ! -d "$backup_storage_dir/$today" ]; then + + mkdir $backup_storage_dir/$today + + fi + + # only proceed if pdns zones have not already been backed up + if [[ ! -d $backup_storage_dir/$today/pdns ]]; then + + mkdir $backup_storage_dir/$today/pdns + + zones=(`/usr/bin/pdnsutil list-all-zones`) + + for zone in "${zones[@]}"; do + + /usr/bin/pdnsutil list-zone $zone > "$backup_storage_dir/$today/pdns/$zone.zone" + + done + + fi + +fi + +# remove expired backups +for existing_backup in "${existing_backups[@]}"; do + + if [[ " ${retention_array[@]} " =~ " ${existing_backup} " ]]; then + + # keep $existing_backup, do nothing + one=1; + + else + + if [[ -d $backup_storage_dir/$existing_backup/pdns ]]; then + + rm -r $backup_storage_dir/$existing_backup/pdns + + fi + + if [ -z "$(ls -A $backup_storage_dir/$existing_backup)" ]; then + + rm -r $backup_storage_dir/$existing_backup + + fi + + fi + +done + +# check if backup_storage_dir is mounted and unmount if so +grep -qs " $backup_storage_dir " /proc/mounts +if [ $? -eq 0 ]; then + umount $backup_storage_dir +fi + +exit 0; diff --git a/bashup-list-backups.sh b/bashup-list-backups.sh new file mode 100644 index 0000000..1391ef9 --- /dev/null +++ b/bashup-list-backups.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Bashup - A set of bash scripts for managing backups. +# https://git.stack-source.com/msb/bashup +# MIT License Copyright (c) 2021 Matthew Saunders Brown + +# backups are stored here +backup_storage_dir='/mnt/backups'; + +# check for local config, which can be used to override any of the above +if [[ -f /usr/local/etc/bashup.cnf ]]; then + source /usr/local/etc/bashup.cnf +fi + +if [ ! -d $backup_storage_dir ]; then + echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist." + exit 1 +fi + +# check if backup_storage_dir is a mount in fstab +grep -qs " $backup_storage_dir " /etc/fstab +if [ $? -eq 0 ]; then + # check if backup_storage_dir is already mounted + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + # attempt to mount backups + mount $backup_storage_dir + # re-check for backups mount + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + echo "ERROR: failed to mount $backup_storage_dir" + exit 1 + fi + fi +fi + +# output list of backups (dates) +ls -1 $backup_storage_dir | grep -v lost+found + +# check if backup_storage_dir is mounted and unmount if so +grep -qs " $backup_storage_dir " /proc/mounts +if [ $? -eq 0 ]; then + umount $backup_storage_dir +fi + +exit 0 diff --git a/bashup-restore-files.sh b/bashup-restore-files.sh new file mode 100644 index 0000000..26de212 --- /dev/null +++ b/bashup-restore-files.sh @@ -0,0 +1,164 @@ +#!/bin/bash +# +# Bashup - A set of bash scripts for managing backups. +# https://git.stack-source.com/msb/bashup +# MIT License Copyright (c) 2021 Matthew Saunders Brown + +# backup storage directory +backup_storage_dir='/mnt/backups'; +# directories to be backed up +backup_dirs=('/etc' '/home' '/srv' '/root' '/usr/local' '/var/www'); + +# check for local config, which can be used to override any of the above +if [[ -f /usr/local/etc/bashup.cnf ]]; then + source /usr/local/etc/bashup.cnf +fi + +help() +{ + thisfilename=$(basename -- "$0") + echo "$thisfilename" + echo "Restore file(s) from backup." + echo "" + echo "usage: $thisfilename [-b BACKUPDATE] [-f PATH]" + echo "" + echo " -b BACKUPDATE Backup date/archive to restore from." + echo " -p PATH Path to file or directory to restore." + echo " -h Print this help." + echo " You will be prompted to select backup date and file" + echo " or directory if not specified on the command line" + echo " with the above options." + exit +} + +if [ ! -d $backup_storage_dir ]; then + echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist." + exit 1 +fi + +# check if backup_storage_dir is a mount in fstab +grep -qs " $backup_storage_dir " /etc/fstab +if [ $? -eq 0 ]; then + # check if backup_storage_dir is already mounted + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + # attempt to mount backups + mount $backup_storage_dir + # re-check for backups mount + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + echo "ERROR: failed to mount $backup_storage_dir" + exit 1 + fi + fi +fi + +# set any options that were passed +while getopts "b:p:h" opt; do + case "${opt}" in + h ) + help + exit;; + b ) + backup=${OPTARG} + ;; + p ) + pathtorestore=${OPTARG} + ;; + \? ) + echo "Invalid option: $OPTARG" 1>&2 + exit 1;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + exit 1;; + esac +done + +# get list of backups (dates) +existing_backups=($(ls $backup_storage_dir|grep -v lost+found)) + +# prompt if backup was not set on command line +if [ -z "$backup" ] ; then + # select backup to restore from + PS3="Select number of backup to restore from: " + existing_backups_len=${#existing_backups[@]} + select backup in ${existing_backups[@]} + do + if [[ $REPLY -gt 0 ]] && [[ $REPLY -le $existing_backups_len ]]; then + echo $backup + break + else + echo "ERROR: Invalid entry, try again." + fi + done +fi + +# set path to file or directory to restore +if [ -z "$pathtorestore" ] ; then + if [ -d $backup_storage_dir/$backup/files ]; then + # select backup file or directory to restore + PS3="Select number of backup directory to restore from: " + backup_dirs_len=${#backup_dirs[@]} + select backup_dir in ${backup_dirs[@]} + do + if [[ $REPLY -gt 0 ]] && [[ $REPLY -le $backup_dirs_len ]]; then + echo "Complete path to specific file or directory to restore." + read -e -p "Just press enter to recursively restore entire dir: " -i $backup_storage_dir/$backup/files$backup_dir pathtobackup + pathtorestore=${pathtobackup#"$backup_storage_dir/$backup/files"} + break + else + echo "ERROR: Invalid entry, try again." + fi + done + else + echo "ERROR: Backup dir for $backup/files/ does not exist." + exit 1 + fi +else + pathtobackup="$backup_storage_dir/$backup/files$pathtorestore" +fi + +if [ -d $pathtobackup ]; then + # restore is a direcotry, make sure paths end in a slash (/) for proper rsync + if [[ ! "$pathtobackup" == */ ]]; then + pathtobackup="$pathtobackup/" + fi + if [[ ! "$pathtorestore" == */ ]]; then + pathtorestore="$pathtorestore/" + fi +elif [ ! -f $pathtobackup ]; then + # restore is not a file nor a direcotry, does not exist + echo "ERROR: Backup file/directory ($pathtobackup) does not exist." + exit 1 +fi + +# make sure parent dir of destination exists +if [ ! -d $(/usr/bin/dirname $pathtorestore) ]; then + echo "ERROR: Parent directory missing for restore location." + exit 1 +fi + +# make sure pathtorestore is for a file/dir *within* one of the backup_dirs +verify_back_dir=FALSE +for dir in "${backup_dirs[@]}"; do + if [[ $pathtorestore == $dir ]] || [[ $pathtorestore == $dir* ]]; then + verify_back_dir=TRUE + fi +done +if [[ $verify_back_dir == FALSE ]]; then + echo "ERROR: File or directory to restore does not exist within one of the backed up directories." + exit 1 +fi + +# perform restore +# Still in testing mode, display command instead of running it +"To restore $pathtorestore from $backup run this command:" +echo "/usr/bin/rsync -vn --archive --numeric-ids --one-file-system --delete $pathtobackup $pathtorestore" + +# check if backup_storage_dir is mounted and unmount if so +/usr/bin/grep -qs " $backup_storage_dir " /proc/mounts +if [ $? -eq 0 ]; then + /usr/bin/umount $backup_storage_dir +fi + +exit 0 diff --git a/bashup-restore-mysql.sh b/bashup-restore-mysql.sh new file mode 100644 index 0000000..a6758b3 --- /dev/null +++ b/bashup-restore-mysql.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# +# Bashup - A set of bash scripts for managing backups. +# https://git.stack-source.com/msb/bashup +# MIT License Copyright (c) 2021 Matthew Saunders Brown + +# backup directory +backup_storage_dir='/mnt/backups'; +# config file that contains [client] config with 'host' 'user' 'password' settings +defaults_extra_file='/etc/mysql/debian.cnf'; + +# check for local config, which can be used to override any of the above +if [[ -f /usr/local/etc/bashup.cnf ]]; then + source /usr/local/etc/bashup.cnf +fi + +help() +{ + thisfilename=$(basename -- "$0") + echo "$thisfilename" + echo "Restore database from backup." + echo "" + echo "usage: $thisfilename [-b BACKUPDATE] [-d DATABASE]" + echo "" + echo " -b BACKUPDATE Backup date/archive to restore from." + echo " -d DATABASE Database to restore." + echo " -h Print this help." + echo " You will be prompted to select backup date and/or" + echo " database name from a list of available options" + echo " if they are not specified on the command line." + exit +} + +if [ ! -d $backup_storage_dir ]; then + echo "ERROR: Backup storage dir ($backup_storage_dir) does not exist." + exit 1 +fi + +# check if backup_storage_dir is a mount in fstab +grep -qs " $backup_storage_dir " /etc/fstab +if [ $? -eq 0 ]; then + # check if backup_storage_dir is already mounted + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + # attempt to mount backups + mount $backup_storage_dir + # re-check for backups mount + grep -qs " $backup_storage_dir " /proc/mounts + if [ $? -ne 0 ]; then + echo "ERROR: failed to mount $backup_storage_dir" + exit 1 + fi + fi +fi + +# set any options that were passed +while getopts "b:d:h" opt; do + case "${opt}" in + h ) + help + exit;; + b ) + backup=${OPTARG} + ;; + d ) + database=${OPTARG} + ;; + \? ) + echo "Invalid option: $OPTARG" 1>&2 + exit 1;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + exit 1;; + esac +done + +# get list of backups (dates) +existing_backups=($(ls $backup_storage_dir|grep -v lost+found)) + +# prompt if backup was not set on command line +if [ -z "$backup" ] ; then + # select backup to restore from + PS3="Select number of backup to restore from: " + existing_backups_len=${#existing_backups[@]} + select backup in ${existing_backups[@]} + do + if [[ $REPLY -gt 0 ]] && [[ $REPLY -le $existing_backups_len ]]; then + echo $backup + break + else + echo "ERROR: Invalid entry, try again." + fi + done +fi + +# set dump if database was set via command option +if [ ! -z "$database" ] ; then + dump="$database.sql.gz" +else + if [ -d $backup_storage_dir/$backup/mysql ]; then + # get list of dbs in backup + existing_dumps=($(ls $backup_storage_dir/$backup/mysql)) + # set array with names of dbs (without .sql.gz suffix) + declare -a existing_dumps_db_names + for existing_dump in "${existing_dumps[@]}"; do + existing_dumps_db_name="$(basename $existing_dump .sql.gz)" + existing_db_names+=("$existing_dumps_db_name") + done + # select database dump to restore + PS3="Select number of database to restore: " + existing_db_names_len=${#existing_db_names[@]} + select database in ${existing_db_names[@]} + do + if [[ $REPLY -gt 0 ]] && [[ $REPLY -le $existing_db_names_len ]]; then + dump=$database.sql.gz + break + else + echo "ERROR: Invalid entry, try again." + fi + done + else + echo "ERROR: Backup dir for $backup/mysql/ does not exist." + exit 1 + fi +fi + +# check that dump exists and restore it now +if [ -d $backup_storage_dir/$backup ]; then + if [ -f $backup_storage_dir/$backup/mysql/$dump ]; then + # Still in testing mode, display command instead of running it + echo "To restore database $database from backup $backup run this command:" + echo "/usr/bin/zcat $backup_storage_dir/$backup/mysql/$dump | mysql --defaults-extra-file=$defaults_extra_file $database" +# echo "SUCCESS: Database $database from backup $backup has been restored." + else + echo "ERROR: Dump for database $database does not exist in the $backup backup dir." + exit 1 + fi +else + echo "ERROR: Backup dir for $backup does not exist." + exit 1 +fi + +# check if backup_storage_dir is mounted and unmount if so +grep -qs " $backup_storage_dir " /proc/mounts +if [ $? -eq 0 ]; then + umount $backup_storage_dir +fi + +exit