From ffffde0c751cc21d28e98e3b833c53587a4d4bf4 Mon Sep 17 00:00:00 2001 From: Holger Frey Date: Wed, 7 Aug 2019 14:23:09 +0200 Subject: [PATCH] first test --- .gitignore | 1 + old_backup_script | 22 ++++++++ zfs-snapshot-backup.py | 123 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 old_backup_script create mode 100644 zfs-snapshot-backup.py diff --git a/.gitignore b/.gitignore index 7f7cccc..031fad5 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ var/ *.egg-info/ .installed.cfg *.egg +.venv/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/old_backup_script b/old_backup_script new file mode 100644 index 0000000..fd133ed --- /dev/null +++ b/old_backup_script @@ -0,0 +1,22 @@ +#!/bin/sh + +previous_snapshot=$(zfs list -t snapshot -H -o name -r Datenspeicher | sort | tail -n2 | head -n1) +current_snapshot=$(zfs list -t snapshot -H -o name -r Datenspeicher | sort | tail -n1) + +echo "PREV: $previous_snapshot" +echo "CUR: $current_snapshot" + +file_name=$(echo "$current_snapshot" | cut -f2 -d"/") + +cd /mnt/Datenspeicher/snap-backup-dataset/temporary-backups/ + +zfs send -I $previous_snapshot $current_snapshot | gzip > $file_name.gz + +scp -i ../backup_key $file_name.gz zfs_snap_backup@etha.cpi.imtek.uni-freiburg.de:~/zfs-backups/ && rm $file_name.gz + +# Restoring Backups: +# +# cd /mnt/Datenspeicher/snap-backup-dataset/temporary-backups/ +# scp -i ../backup_key zfs_snap_backup@etha.cpi.imtek.uni-freiburg.de:~/zfs-backups/ tmp-backup-file.gz +# gunzip -c tmep-backup-file.gz | zfs receive -F Datenspeicher/test-backup +# rm tmp-backup-file.gz diff --git a/zfs-snapshot-backup.py b/zfs-snapshot-backup.py new file mode 100644 index 0000000..011d976 --- /dev/null +++ b/zfs-snapshot-backup.py @@ -0,0 +1,123 @@ +import pathlib +import subprocess + +SSH_KEY_FILE = "/mnt/Datenspeicher/snap-backup-dataset/backup_key" +SSH_REMOTE = "zfs_snap_backup@etha.cpi.imtek.uni-freiburg.de" + +REMOTE_PATH = "zfs-backups" +SCP_REMOTE_URL = f"{SSH_REMOTE}:~/{REMOTE_PATH}/" + +ZFS_POOL = "Datenspeicher" +ZFS_ELAB_PREFIX = "elabfs-" + +TMP_BACKUP_FOLDER = "/mnt/Datenspeicher/snap-backup-dataset/temporary-backups" + + +def call(arguments): + result = subprocess.run( + arguments, check=True, capture_output=True, text=True + ) + return result.stdout + + +def remote_call(arguments): + cmd = ["ssh", "-i", SSH_KEY_FILE, SSH_REMOTE] + cmd.extend(arguments) + return call(cmd) + + +def clean_split(text): + items = (item.strip() for item in text.split) + return [item for item in items if item] + + +def list_snapshots(): + cmd = ["zfs", "list", "-t", "snapshot", "-H", "-o", "name", "-r", ZFS_POOL] + return clean_split(call(cmd)) + + +def elab_snapshots(): + result = {} + for snapshot in list_snapshots(): + pool, snap_name = snapshot.split("/", 1) + if snap_name.startswith(ZFS_ELAB_PREFIX): + prefix_and_member, _ = snap_name.rsplit("@", 1) + member = prefix_and_member.replace(ZFS_ELAB_PREFIX, "") + if member not in result: + result[member] = [] + result[member].append(snapshot) + return result + + +def gzip_filename(name): + return f"{name}.gz" + + +def gzip_filepath(name): + return pathlib.Path(TMP_BACKUP_FOLDER) / gzip_filename(name) + + +def list_remote_backups(members): + result = {} + for member in members: + remote_sub_dir = f"{REMOTE_PATH}/{member}" + try: + backups = clean_split(remote_call(["ls", remote_sub_dir])) + result[member] = set(backups) + except CalledProcessError: + remote_call(["mkdir", remote_sub_dir]) + result[member] = set() + + +def backup_latest_snapshot(member, elab_snapshots, existing_backups): + print(f"backing up member {member}") + snapshots = sorted(elab_snapshots, reverse=True) + current_snapshot = snapshots[0] + latest_backup = None + for snapshot in snapshots: + if gzip_filename(snapshot) in existing_backups: + latest_backup = snapshot + break + if current_snapshot == latest_backup: + # nothing to back up + print("- nothing to backup, latest snapshot: {current_snapshot}") + return + elif latest_backup is None: + # no snapshot was found in backups, make a full backup for consistency + send_cmd = ["zfs", "send", current_snapshot] + print(" - full backup, latest snapshot: {current_snapshot}") + else: + # make an incremental backup + print( + " - incremental backup, from: {latest_backup} to: {current_snapshot}" + ) + send_cmd = ["zfs", "send", "-I", latest_backup, current_snapshot] + + # create the backup + print(" - generating temporary backup file") + tmp_gzip_filepath = gzip_filepath(current_snapshot) + with open(tmp_gzip_filepath, "wb") as file_handle: + gzip_in = subprocess.Popen( + "gzip", stdin=subprocess.PIPE, stdout=file_handle, check=True + ).stdin + subprocess.call(send_cmd, stdout=gzip_in, check=True) + + # copy the backup to the remote server + print(" - copying temporary backup file") + remote_url = ( + f"{SSH_REMOTE}:~/{REMOTE_PATH}/{member}/{tmp_gzip_filepath.name}" + ) + copy_cmd = ["scp", "-i", SSH_KEY_FILE, str(tmp_gzip_filepath), remote_url] + call(copy_cmd) + + # remove the temporary file + print(" - removing temporary backup file") + tmp_gzip_filepath.unlink() + + +def create_backups(): + elab_snapshots = list_snapshots() + existing_backups = list_remote_backups(elab_snapshots.keys()) + for member, snapshots in elab_snapshots.items(): + members_backups = existing_backups.get(member, []) + backup_latest_snapshot(member, snapshots, members_backups)