fixed weird bug on scp

copy_snapshots_to_remote.py

@@ -0,0 +1,67 @@
+#!/usr/local/bin/python
+
+# simple python module to copy incremental backups of zfs snapshots to a
+# remote server
+
+import pathlib
+import subprocess
+
+
+SSH_KEY_FILE = "/mnt/Datenspeicher/snap-backup-dataset/backup_key"
+SSH_REMOTE = "zfs_snap_backup@etha.cpi.imtek.uni-freiburg.de"
+
+REMOTE_PATH = "zfs-backups"
+SCP_REMOTE_URL = f"{SSH_REMOTE}:~/{REMOTE_PATH}/"
+
+ZFS_POOL = "Datenspeicher"
+ZFS_ELAB_PREFIX = "elabfs-"
+
+TMP_BACKUP_FOLDER = "/mnt/Datenspeicher/snap-backup-dataset/temporary-backups"
+
+
+def call(arguments, as_text=False):
+    """ run a command line argument
+
+    simple wrapper around subprocess.run() with some sensible defaults
+
+    :params arguments: list of command line arguments and parameters
+    :params as_text: should the output treated as text
+    :returns: bytesarray or string (if as_text is trueish)
+    :raises subprocess.CalledProcessError: if command has not an exit value of 0
+    """
+    result = subprocess.run(
+        arguments,
+        check=True,
+        stdout=subprocess.PIPE,
+        universal_newlines=as_text,
+    )
+    return result.stdout
+
+def get_member_name(filename):
+    #elabfs-LukasMetzler@auto-20190809.0200-1w.gz
+    if not filename.startswith(ZFS_ELAB_PREFIX):
+        raise ValueError(f"not an elabfs snapshot: {filename}")
+    prefix, rest = filename.split("@", 1)
+    return prefix.replace(ZFS_ELAB_PREFIX, "")
+
+
+def copy_snapshot(filepath):
+    filename = filepath.name
+    member = get_member_name(filename)
+    print(f" - copying {filename}")
+    remote_url = (
+        f"{SSH_REMOTE}:~/{REMOTE_PATH}/{member}/{filename}"
+    )
+    copy_cmd = ["scp", "-i", SSH_KEY_FILE, str(filepath), remote_url]
+    call(copy_cmd)
+
+
+def batch_copy_snapshots():
+    tmp_folder = pathlib.Path(TMP_BACKUP_FOLDER)
+    gzip_items = (i for i in tmp_folder.iterdir() if i.suffix==".gz")
+    for path in gzip_items:
+        copy_snapshot(path)
+
+
+if __name__ == "__main__":
+    batch_copy_snapshots()

create_snapshot_backup.py

@@ -206,24 +206,13 @@ def backup_latest_snapshot(member, elab_snapshots, existing_backups):
     # create the backup
     tmp_gzip_filepath = pathlib.Path(TMP_BACKUP_FOLDER) / gzip_tmp_filename
     print(f" - generating temporary backup file {tmp_gzip_filepath.name}")
+    #with open(tmp_gzip_filepath, "wb") as file_handle:
     with open(tmp_gzip_filepath, "wb") as file_handle:
         gzip_in = subprocess.Popen(
             "gzip", stdin=subprocess.PIPE, stdout=file_handle
         ).stdin
         subprocess.run(send_cmd, stdout=gzip_in, check=True)
 
-    # copy the backup to the remote server
-    print(" - copying temporary backup file")
-    remote_url = (
-        f"{SSH_REMOTE}:~/{REMOTE_PATH}/{member}/{tmp_gzip_filepath.name}"
-    )
-    copy_cmd = ["scp", "-i", SSH_KEY_FILE, str(tmp_gzip_filepath), remote_url]
-    call(copy_cmd)
-
-    # remove the temporary file
-    print(" - removing temporary backup file")
-    tmp_gzip_filepath.unlink()
-
 
 def create_backups():
     """ batch create backups for all available elab snapshots"""

run_snapshot_backups.py

@@ -0,0 +1,4 @@
+#!/bin/csh
+rm temporary-backups/*.gz
+./create_snapshot_backup.py
+./copy_snapshots_to_remote.py