updates

authorized_keys/atc-docker01.dell-atc.lan

@@ -0,0 +1 @@
+command="borg serve --restrict-to-path /var/borg/atc-docker01.dell-atc.lan" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQClDrplpfAMxQJByLR0lSVDpuJeXKoexJwj3oiea16ixtllJFpD+aozi5HQfhckw/ynI5AMFCt69b1EVR9YkS1lTqku5rAAWen1tQuaR4afkxuLXr/3T11mQHz6nxy9/rTfZyxGZA32AuGav9bmdjrCz/y7AKT1OnXaol8b6XPfxHgxvRl779gLaVz3s0xbzo4v9diPcdTy4NSNsOjynJaGj6ReMV6qFIxbZeW3gX18TcMw8pwsD4Qog1LRr6hzU96aV04Y6lkxeuYiVEQlcrdTrlrelPSFu9gtOCMJ7ZMA1JFx2sYf5o8+b1v+hqekOK/1+8TuflThFkHga35u4jwwCYKKm+E0HwcF90n3lXTWunhum02N/C8z2DHaqqykaNeb5WxKQ62SXYFskPlsD8/bXHeGuQS+RuqvXnHg0M8LpsQql9rmG+Eps180tfpY9ozBYe0kaI9TurTXWwwmAyyz5kjgfJmR4UYNjlYkE9VGuA7/JT0OR6sYoD2DVLpH1VznvqhK904JZ/8Hbuwnr3TzAJCF/aRc9EEtiUOG9q6BwPQpDv87WLnSJYElpktAu7q0UXdvWLKckC7Bp6osNtiZUuiT3oLIy5sms+8nv/PWPWY51mbXXEuvE2hhL6A8X9tjp4GUCaPOfKMriHmSirvc/cnwhr5giiZXcsqM0Eu/ow== 

authorized_keys/atc-docker02.dell-atc.lan

@@ -0,0 +1 @@
+command="borg serve --restrict-to-path /var/borg/atc-docker02.dell-atc.lan" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCeoG+ZXWY7WFXVVbTGY/1I9aRXFT0gjFAvvB/ke1SV2ia4sbrLlcGLfH/jeM5xoYeUkSxvJenmhppaRDbjlGUO2VJPufWwbIkjKdTdxuHGs86ccLdScOafJt+HQK+ii6NGuWLSKOF1eNqaxLQUN7ySZY68Ny1Diqj59KJ/fue/37BSMF8XBRCwWjrpBBG+hiHcVBO84xYLWy1fnuIEi/Ui3PxL2dmOpAmHJKrsVVDnuUyKpkZAUL/fZeQ7fiejVpSYGg3PNaa5lcqwKt0ZEHEFH9qxkzfkYCax5ztxl3xFVlkeiFv3BSr0kbfacq3kOrFPwayTfsUgiZ10XsZc3p/91DWU43kfuTpb6o3pIo0u2hhSo18tWUbtvejtVNB0SUWG1Wg7WPT2wM7jqmv8ScJk9Zzrnii4bLq4uyO2Qg5jFv8D5OeyYAgQggTcE/CBrGlQ9SjqzIyFp03hX0cY1SN0DgdIHGAXRVKy9Uh6IiRkQLyDf0Xz8d2EnEtwRwuyi1bjjLoD54cqWICDV0yMdPL/tJKv7fxjJvjMZvNdqsKi3rOmz0kyBP5p9KPZhbm7VT1uCDhhv06ngERQgTOyvs1JUuej3qass/DPuLnbmyN4jZl6NcF/kPxKcl8uc6xN1QS+sCwnaPrNmbP0+0S4SUKQDlwBynAVm1NC2EqoqhZ+oQ== 

authorized_keys/atc-gpu-bart.dell-atc.lan

@@ -0,0 +1 @@
+command="borg serve --restrict-to-path /var/borg/atc-gpu-bart.dell-atc.lan" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCrMrYRn9i4rW2AKdRniP4GVUaAJ1vw1RZLBGS7qYq68aqcH6/3QtEkfUUs+ZKkzRcy7jGOsn/n5QHe+tQTycEUDznx20UtSpB1ArwltWHOuJNAmtxBaR5gxfOGgjHgxyiOSEW0qXV0+hIfHNZRKOpxlj2M3zlCbd+osegW3WeUXZpkPttjwUIxxGkvYxmAt7Au4VegVxu+gTVrDrc5z/iV7JUtSBwjpVXE/giVYW2Nu6z/9ZP6/SXn9/misn+PAGEWlDFO1BRCoB3fMIkDIooCdz+rcZ1zReYzKamFtTKlKBWQHzgRV7CAka4HNodjtxwEys1CRjtkpUuZR1WNmEQrrhl/Y+Jh+aVmA4Fbz/NAY4DR3ivGApxv3j3e2ucHOPEA6dH/Nacxzshs8V23dNFZgxTd8eTiMqR74dpNHov9y9RiwbE5qSTxTgoy87yyosgR1MVzPlVeb5MYnpOASLtd/EdDlpY8gM3r7wWKkelhEDwAofoe9DoSRvmPBaS+GFSFkblwRlcrymrZgzZVT9rPGclS9JEBfzwp8TGlq1fr7BoAvA8yuw478GfbwxkSn7YmT7CtlIkyS1U6cWTy/kYG7HFY1T2ryW25XAHJbrXS4xyXiSEKcegu/r/y4/FVIK7NYU0FEGfPLfAnSTFP9fiMsoEcHo4xdzORf8Y3Yl6RtQ== 

authorized_keys/atc-mgt01.dell-atc.lan

@@ -0,0 +1 @@
+command="borg serve --restrict-to-path /var/borg/atc-mgt01.dell-atc.lan" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCkvGaWb4zIg2oJqXSJwFETaMlJZmJbA+2/Ej411J5napb1vr0t7vobDtuo70u8z77w9UnZg2ES9lwJvYUaCxeTj1dsDRP7KJOGJayBVxrjjI7QY7OpF1TzP7BF8YPDWvx21RQq7dHWMofPl3z66KHTLQTBMiKjscX/V+/J3FN3o9z1aqxQ7Qj7DQfVtpvacufWChg0+uZRS2/r58M5dcWMssMiPLNh0NQJQZuP5aNy0Wj7VXJYQPdCwbdIvZnEVxucKisoHvYzoJYp196Wj71MUHkLo+eJnJcnivn5JGEpJGe7lxplHq+D1HhOeQU0yiHvjYX8QYwqc9Nw4Sb2Oknxu54dS8qbPtIIePsFU8bTpuIdxY+UUN6i+Z455ElbT1tDPMJI2A9iAxXJGHIdnH9/dUZNzMS7oFQ8Itf8dihSQWbs5H297nNN/f48RPsE+hJmBK49tq0fqjXJZ3dz9AzD8TQXMaAr72Vhk9lyomrWq5empBCFuz+6myMOaMXj2XrVbEi2MRiIYE56UWGqCIFCHo0BswrPh9T/GxAO8P3CfebjWCFOLl+v+w13MvX8KummEI8nfiiFO8AdvrWEbGXJE1x7BY9BDa0jt68AIakKHFtWLU0ghwks2w4kTsfoYtlFZ52oSU5uDLezddgmkob7fyEuhysKi84xcTVoPJasMw== 

borg-clients.yml

@@ -0,0 +1,13 @@
+---
+- name: Setup Borg Backups
+  hosts: borg_clients
+  tasks:
+    - name: Install Borgmatic
+      ansible.builtin.package:
+        name: "borgmatic"
+        state: present
+
+    - name: Borgmatic
+      include_tasks: tasks/borgmatic.yml
+
+

borg-server.yml

@@ -0,0 +1,54 @@
+---
+- name: Sync Configuration Files
+  hosts: borg_servers
+  become: yes
+
+  tasks:
+  - name: Borg group
+    ansible.builtin.group:
+      name: borg
+      state: present
+      system: true
+
+  - name: Borg User
+    ansible.builtin.user:
+      name: borg
+      group: borg
+      comment: Borg Service
+      create_home: yes
+      system: true
+
+  - name: Install borg
+    package:
+      name: "borgbackup"
+      state: present
+
+  - name: Ensure borg .ssh directory
+    ansible.builtin.file:
+      path: /home/borg/.ssh
+      state: directory
+      owner: borg
+      group: borg
+      mode: '0700'
+
+  - name: concat
+    ansible.builtin.assemble:
+      src: authorized_keys
+      dest: /home/borg/.ssh/authorized_keys
+      owner: borg
+      group: borg
+      mode: '0600'
+      remote_src: false
+
+  - name: Make backup directories
+    file:
+      path: /var/borg/{{ item | basename }}
+      state: directory
+      owner: borg
+      group: borg
+      mode: '0700'
+
+    with_fileglob:
+      - "authorized_keys/*"
+
+

config.yml

@@ -1,6 +1,6 @@
 ---
 - name: Sync Configuration Files
-  hosts: rocky
+  hosts: rhel
   become: yes
   vars:
     worker_limit: 1024  # Global variable used in the template

etc/profile.d/aliases.sh

@@ -11,6 +11,7 @@ alias cdlab='cd /opt/atclab/'
 alias lowercase='mmv "*" "#l1"'
 alias lowerext='mmv "*.*" "#1.#l2"'
 alias varlog='cd /var/log'
+alias borgmatic='borgmatic -nc '
 
 #alias showlog='sudo cat /var/log/messages | tail -1000'
 #alias trlog='sudo tail -fn1000 /var/log/messages'

inventory.ini

@@ -33,6 +33,16 @@ atc-mgt01.dell-atc.lan
 atc-gpu-bart.dell-atc.lan
 atc-nas.dell-atc.lan
 atc-gpu-bart.dell-atc.lan
+atc-backup.dell-atc.lan
+
+[borg_servers]
+atc-backup.dell-atc.lan
+
+[borg_clients]
+atc-docker01.dell-atc.lan
+atc-docker02.dell-atc.lan
+atc-gpu-bart.dell-atc.lan
+atc-mgt01.dell-atc.lan
 
 [nvidia]
 atc-gpu-bart.dell-atc.lan

tasks/borgmatic.yml

@@ -0,0 +1,55 @@
+---
+- name: Install packages
+  ansible.builtin.package:
+    name: "borgmatic"
+    state: present
+
+- name: Deploy borgmatic config
+  template:
+    src: templates/borgmatic-system.yaml
+    dest: /etc/borgmatic.d/system.yaml
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Deploy borgmatic excludes
+  template:
+    src: templates/borgmatic-excludes
+    dest: /etc/borgmatic/excludes
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: SSH Keypair
+  openssh_keypair:
+    path: /etc/borgmatic/borg_key
+    owner: root
+    group: root
+
+- name: Get SSH key
+  slurp:
+    src: /etc/borgmatic/borg_key.pub
+  register: remote_key
+
+- name: Deploy borgmatic authorized_key
+  template:
+    src: templates/borgmatic-auth-key
+    dest: /etc/borgmatic/authorized_keys
+    owner: root
+    group: root
+    mode: '0644'
+  vars:
+    included_content: "{{ remote_key.content | b64decode }}"
+
+- name: Pickup authorized_key
+  fetch:
+    src: /etc/borgmatic/authorized_keys
+    dest: authorized_keys/{{ ansible_fqdn }}
+    flat: yes
+
+- name: Setup Backup Cron Job
+  ansible.builtin.cron:
+    name: "borgmatic"
+    minute: 0
+    hour: 1
+    job: /usr/bin/borgmatic

templates/borgmatic-auth-key

@@ -0,0 +1 @@
+command="borg serve --restrict-to-path /var/borg/{{ ansible_fqdn }}" {{ included_content }}

templates/borgmatic-excludes

@@ -0,0 +1,24 @@
+*/venv
+*/.cache
+*/.java
+*/.git
+*/.venv
+*/.mozilla
+*/__pycache__
+*/.Trash*
+*/.vscode-server
+*.pyc
+*/mirrors
+*/repodata
+*/blobs
+/etc/selinux
+/etc/ld.so.cache
+/root/.borgmatic/checks
+*/.esmtp_queue
+*/mirrors
+*/macbook
+*.swp
+*/.gnupg
+*/.ssh
+*.pem
+*id_*

templates/borgmatic-system.yaml

@@ -0,0 +1,51 @@
+constants:
+    user: borg # borg_{{ ansible_nodename }}
+    local: {{ ansible_fqdn }}
+    remote: atc-backup.dell-atc.lan
+    base: /var/borg
+
+source_directories:
+    - /etc
+    - /home
+    - /usr/local
+    - /opt
+    - /var/backups
+    - /var/log
+    - /var/opt
+    {% if 'docker' in group_names -%}
+    - /var/lib/docker/volumes/
+    {% endif -%}
+    {% if 'webservers' in group_names -%}
+    - /var/www
+    {% endif -%}
+    {% if 'mailservers' in group_names -%}
+    - /var/dovecot
+    {% endif %}
+
+repositories:
+    - path: ssh://{user}@{remote}{base}/{local}
+      label: system
+
+exclude_from:
+    - /etc/borgmatic/excludes
+
+color: false
+ssh_command: ssh -i /etc/borgmatic/borg_key -oBatchMode=yes -o StrictHostKeychecking=no
+archive_name_format: 'system-{now:%Y-%m-%d_%H%M}'
+
+# borg_base_directory: /path/to/base
+# $borg_base_directory/.config/borg
+# borg_config_directory: /path/to/base/config
+
+# $borg_base_directory/.cache/borg
+# borg_cache_directory: /path/to/base/cache
+
+# keep_within: 3H
+# keep_secondly: 60
+# keep_minutely: 60
+# keep_hourly: 24
+keep_daily: 7
+keep_weekly: 4
+keep_monthly: 6
+keep_yearly: 1
+