From 7b16d0971965dbb01e333325df58cc6fdc8406d0 Mon Sep 17 00:00:00 2001 From: nono Date: Sun, 19 Mar 2023 10:21:49 +0100 Subject: [PATCH] cretae readme content --- README.md | 50 ++++++++++ pg_backup.sh.old | 250 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 pg_backup.sh.old diff --git a/README.md b/README.md index e69de29..db58f54 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,50 @@ +# Backup utilities for databases + +## Summary +Linux backup utilities for the following databases. +- MariaDB / MySQL + - https://git.gresse.net/Ops/Backup/raw/branch/main/mysql_backup.sh +- PostgreSQL + - https://git.gresse.net/Ops/Backup/raw/branch/main/pg_backup.sh + +## How to install + +### Install online + +Install Utility to '/tmp/backup' direcotry with curl +```bash +curl https://git.gresse.net/Ops/Backup/raw/branch/main/mysql_backup.sh | sh -s installonline +``` + +Or install Utility to '/tmp/backup' direcotry with wget +```bash +wget -O - https://raw.githubusercontent.com/PierNola/BackupPg/main/backup.sh | sh -s installonline +``` + +### Install online in a specific directory + +```bash +curl https://raw.githubusercontent.com/PierNola/BackupPg/main/backup.sh | sh -s installonline +``` + +Or: + +```bash +wget -O - https://raw.githubusercontent.com/PierNola/BackupPg/main/backup.sh | sh -s installonline +``` + +### Install description +The installer will perform 3 actions: + +1. Create and copy `backup.sh` to `/opt/backup` or to a specific directory +2. Create 3 subdirectory : + - `archive` which contains archived dumps + - `data` which contains the dumps of the day + - `log` which contains the logs +3. Create daily cron job to for daily backup. + +Cron entry example: + +```bash +30 22 * * * /opt/backup/backup.sh --run > /dev/null +``` diff --git a/pg_backup.sh.old b/pg_backup.sh.old new file mode 100644 index 0000000..67646c6 --- /dev/null +++ b/pg_backup.sh.old @@ -0,0 +1,250 @@ +#!/bin/bash + +SCRIPT_NAME=${0##*/} + +#LOGIN +PGHOST=localhost +PGUSER= +PGPASSWD= + +# Number of monthly, weekly and daily backups +KEEP_DAILY_BCK=1 +KEEP_WEEKLY_BCK=4 +KEEP_MONTHLY_BCK=6 + +# Target weekday to take weekly and monthly backups (Monday is 1 and Sunday is 7 - O to disable) +TARGET_WEEKDAY=1 + +# Ex : "|mydb001|mydb002" +EGREP_EXCLUDE_DB="" + +BACKUP_DIR=$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P ) + +LATEST_TIMESTAMP=$(date +"%Y%m%d-%HH%M") +DAILY_TIMESTAMP=$(date +"%Y%m%d") + +LOG_FILE=${BACKUP_DIR}/logs/${SCRIPT_NAME}-${LATEST_TIMESTAMP}.log +ERR_FILE=${BACKUP_DIR}/logs/${SCRIPT_NAME}.err + +CMD_MYSQL=/usr/local/mysql/bin/mysql +CMD_MYSQLDUMP=/usr/local/mysql/bin/mysqldump +CMD_GZIP=gzip + +function backup() { + DAY_OF_MONTH=$(date +%d) + WEEKDAY=$(date +%u) + + + + + + + +########################### +###### FULL BACKUPS ####### +########################### + +for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ } +do + EXCLUDE_SCHEMA_ONLY_CLAUSE="$EXCLUDE_SCHEMA_ONLY_CLAUSE and datname !~ '$SCHEMA_ONLY_DB'" +done + +FULL_BACKUP_QUERY="select datname from pg_database where not datistemplate and datallowconn $EXCLUDE_SCHEMA_ONLY_CLAUSE order by datname;" + +echo -e "\n\nPerforming full backups" +echo -e "--------------------------------------------\n" + +for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres` +do + if [ $ENABLE_PLAIN_BACKUPS = "yes" ] + then + echo "Plain backup of $DATABASE" + + set -o pipefail + if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then + echo "[!!ERROR!!] Failed to produce plain backup database $DATABASE" 1>&2 + else + mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz + fi + set +o pipefail + fi + + if [ $ENABLE_CUSTOM_BACKUPS = "yes" ] + then + echo "Custom backup of $DATABASE" + + if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" -f $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress; then + echo "[!!ERROR!!] Failed to produce custom backup database $DATABASE" 1>&2 + else + mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom + fi + fi + +done + +echo -e "\nAll database backups complete!" + + echo_date "Starting Backup" >>${LOG_FILE} + DATABASES=`echo "show databases" | ${CMD_MYSQL} -h ${MYSQL_SERVER} -u ${MYSQL_USER} -p${MYSQL_PASSWD} | egrep -v "Database|information_schema|performance_schema${EGREP_EXCLUDE_DB}"` + for DB in ${DATABASES}; do + + echo_date "Remove latest backup of database '${DB}'" >>${LOG_FILE} + rm -f ${BACKUP_DIR}/latest/${DB}--* >>${LOG_FILE} 2>&1 + + echo_date "Starting backup of database '${DB}'" >>${LOG_FILE} + ${CMD_MYSQLDUMP} -h ${MYSQL_SERVER} -u ${MYSQL_USER} -p${MYSQL_PASSWD} ${DB} | ${CMD_GZIP} > ${BACKUP_DIR}/latest/${DB}--${LATEST_TIMESTAMP}.dmp.gz 2>>${LOG_FILE} + + echo_date "Copy to daily backup of database '${DB}' " >>${LOG_FILE} + cp ${BACKUP_DIR}/latest/${DB}--${LATEST_TIMESTAMP}.dmp.gz ${BACKUP_DIR}/daily/${DB}--${DAILY_TIMESTAMP}.dmp.gz + + echo_date "Keep only the last ${KEEP_DAILY_BCK} daily backup of database '${DB}'" >>${LOG_FILE} + ls -td ${BACKUP_DIR}/daily/${DB}--* | tail -n +$((${KEEP_DAILY_BCK}+1)) | xargs rm -f >>${LOG_FILE} 2>&1 + + if [ ${WEEKDAY} = ${TARGET_WEEKDAY} ]; then + echo_date "Keep only the last ${KEEP_WEEKLY_BCK} weekly backup of database '${DB}'" >>${LOG_FILE} + ls -td ${BACKUP_DIR}/weekly/${DB}--* | tail -n +$((${KEEP_WEEKLY_BCK}+1)) | xargs rm -f >>${LOG_FILE} 2>&1 + + echo_date "Copy to weekly backup of database '${DB}'" >>${LOG_FILE} + cp ${BACKUP_DIR}/latest/${DB}--${LATEST_TIMESTAMP}.dmp.gz ${BACKUP_DIR}/weekly/${DB}--${DAILY_TIMESTAMP}.dmp.gz + + if [ ${DAY_OF_MONTH} -lt 8 ]; then + echo_date "Keep only the last ${KEEP_MONTHLY_BCK} monthly backup of database '${DB}'" >>${LOG_FILE} + ls -td ${BACKUP_DIR}/monthly/${DB}--* | tail -n +$((${KEEP_MONTHLY_BCK}+1)) | xargs rm -f >>${LOG_FILE} 2>&1 + + echo_date "Copy to monthly backup of database '${DB}'" >>${LOG_FILE} + cp ${BACKUP_DIR}/latest/${DB}--${LATEST_TIMESTAMP}.dmp.gz ${BACKUP_DIR}/monthly/${DB}--${DAILY_TIMESTAMP}.dmp.gz + fi + fi + + done + + echo_date "Backup finished" >>${LOG_FILE} +} + +function usage() { + echo "./mysql_backup.sh" + echo " -h --help : Show help" + echo " -i : Install" + echo " -l : List" + echo " -u : Uninstall" + echo "" +} + +function install() { + echo Install ${SCRIPT_NAME} to ${BACKUP_DIR} + + echo -n " -> Create directory ${BACKUP_DIR}/daily" + mkdir -p ${BACKUP_DIR}/daily + echo ": Done" + + echo -n " -> Create directory ${BACKUP_DIR}/weekly" + mkdir -p ${BACKUP_DIR}/weekly + echo ": Done" + + echo -n " -> Create directory ${BACKUP_DIR}/monthly" + mkdir -p ${BACKUP_DIR}/monthly + echo ": Done" + + echo -n " -> Create directory ${BACKUP_DIR}/latest" + mkdir -p ${BACKUP_DIR}/latest + echo ": Done" + + echo -n " -> Create directory ${BACKUP_DIR}/logs" + mkdir -p ${BACKUP_DIR}/logs + echo ": Done" + + echo -n " -> Create directory ${BACKUP_DIR}/conf" + mkdir -p ${BACKUP_DIR}/conf + echo ": Done" + + echo " -> Install Finished :)" +} + +function list() { + echo "=== === === Listing all backups === === ===" + + echo " -> List ${BACKUP_DIR}/latest" + ls -alth ${BACKUP_DIR}/latest/* 2>/dev/null + + echo " -> List ${BACKUP_DIR}/daily" + ls -alth ${BACKUP_DIR}/daily/* 2>/dev/null + + echo " -> List ${BACKUP_DIR}/weekly" + ls -alth ${BACKUP_DIR}/weekly/* 2>/dev/null + + echo " -> List ${BACKUP_DIR}/monthly" + ls -alth ${BACKUP_DIR}/monthly/* 2>/dev/null +} + +function uninstall() { + read -r -p "Are you sure to uninstall? [y/N]" confirm_uninstall + case ${confirm_uninstall} in + [yY]) + echo "=== === === Uninstall all === === ===" + + echo -n " -> Remove directory ${BACKUP_DIR}/daily" + rm -Rf ${BACKUP_DIR}/daily + echo ": Done" + + echo -n " -> Remove directory ${BACKUP_DIR}/weekly" + rm -Rf ${BACKUP_DIR}/weekly + echo ": Done" + + echo -n " -> Remove directory ${BACKUP_DIR}/monthly" + rm -Rf ${BACKUP_DIR}/monthly + echo ": Done" + + echo -n " -> Remove directory ${BACKUP_DIR}/latest" + rm -Rf ${BACKUP_DIR}/latest + echo ": Done" + + echo -n " -> Remove directory ${BACKUP_DIR}/logs" + rm -Rf ${BACKUP_DIR}/logs + echo ": Done" + + echo -n " -> Remove directory ${BACKUP_DIR}/conf" + rm -Rf ${BACKUP_DIR}/conf + echo ": Done" + + echo " -> Uninstall Finished :)" + ;; + *) + echo "=== === === Abort Uninstall === === ===" + exit + ;; + esac +} + +function echo_date() { + date +"[%Y%m%d-%H:%M:%S] ${1}" +} + +if [ "$#" -ge 1 ]; then + case $1 in + -h | --help) + usage + exit + ;; + -i) + install + exit + ;; + -u) + uninstall + exit + ;; + -l) + list + exit + ;; + *) + echo "ERROR: unknown parameter \"$PARAM\"" + usage + exit 1 + ;; + esac + shift +else + backup + exit +fi \ No newline at end of file