Enhanced the repository sync
This commit is contained in:
parent
8fb0c1393a
commit
0117713934
4 changed files with 55 additions and 157 deletions
|
|
@ -1,122 +1,31 @@
|
|||
from pathlib import Path
|
||||
from subprocess import run
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
from logging import getLogger
|
||||
import shutil
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from .models import BorgRepository
|
||||
from django.conf import settings
|
||||
|
||||
|
||||
logger = getLogger("deployments")
|
||||
|
||||
# sync users
|
||||
|
||||
|
||||
def get_users(user_prefix: str = "u-") -> dict[str, dict[str, Any]]:
|
||||
def sync_repos(dry_run=False):
|
||||
"""Synchronize the repos"""
|
||||
from .models import BorgRepository
|
||||
|
||||
passwd = Path("/etc/passwd").read_text()
|
||||
output = dict()
|
||||
for line in passwd.splitlines():
|
||||
name, _, _, _, _, homedir, _ = line.split(":")
|
||||
if not name.startswith(user_prefix):
|
||||
continue
|
||||
|
||||
output[name] = {"homedir": Path(homedir)}
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def cleanup_users(
|
||||
usernames: set[str],
|
||||
system_users: dict[str, dict[str, Any]],
|
||||
dry_run=False,
|
||||
):
|
||||
"""Cleans up all unused user accounts and directories on the system.
|
||||
|
||||
usernames is the set of users that should exist.
|
||||
system_users contains all the users that exist (with the correct prefix)"""
|
||||
|
||||
base_dir = Path("/home/")
|
||||
|
||||
# cleanup users
|
||||
for user in system_users.keys():
|
||||
if user not in usernames:
|
||||
logger.info("Deleting user '%(user)s'.")
|
||||
print(f"Deleting user '{user}'.")
|
||||
if not dry_run:
|
||||
run(["userdel", user])
|
||||
pass
|
||||
|
||||
# cleanup directories
|
||||
for dir in base_dir.iterdir():
|
||||
if dir.is_dir() and dir.name.startswith("u-") and dir.name not in usernames:
|
||||
logger.info("Deleting home directory '%(dir)s'")
|
||||
print(f"Deleting home directory '{dir}'")
|
||||
if not dry_run:
|
||||
shutil.rmtree(dir)
|
||||
pass
|
||||
|
||||
|
||||
def create_users(
|
||||
usernames: set[str],
|
||||
system_users: dict[str, dict[str, Any]],
|
||||
dry_run=False,
|
||||
):
|
||||
|
||||
for username in usernames:
|
||||
if username not in system_users.keys():
|
||||
logger.info("Adding user '%(username)s'")
|
||||
print(f"Adding user '{username}'")
|
||||
if not dry_run:
|
||||
add_user(username)
|
||||
|
||||
|
||||
def add_user(username: str):
|
||||
|
||||
users = get_users()
|
||||
assert username not in users.keys(), "User already exists"
|
||||
|
||||
run(["useradd", "--no-user-group", "--create-home", username], check=True)
|
||||
|
||||
|
||||
def sync_users(dry_run=False):
|
||||
|
||||
# gather data
|
||||
users = User.objects.all()
|
||||
usernames = {f"u-{user.pk}" for user in users}
|
||||
system_users = get_users()
|
||||
|
||||
# clean up old users
|
||||
cleanup_users(usernames=usernames, system_users=system_users, dry_run=dry_run)
|
||||
|
||||
# create the new users
|
||||
create_users(usernames=usernames, system_users=system_users, dry_run=dry_run)
|
||||
|
||||
for user in users:
|
||||
username = f"u-{user.pk}"
|
||||
sync_repos(
|
||||
username=username,
|
||||
user=user,
|
||||
system_user=system_users[username],
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
|
||||
# sync authorized_keys
|
||||
def sync_repos(username: str, user: User, system_user: dict[str, Any], dry_run=False):
|
||||
|
||||
repos = BorgRepository.objects.filter(user=user)
|
||||
repos = BorgRepository.objects.all()
|
||||
print(repos)
|
||||
|
||||
repos_by_key = defaultdict(list)
|
||||
for repo in repos:
|
||||
repos_by_key[repo.key].append(repo)
|
||||
|
||||
homedir = system_user["homedir"]
|
||||
|
||||
print(repos)
|
||||
# create .ssh directory
|
||||
ssh_dir = homedir / ".ssh"
|
||||
ssh_dir = settings.BACKUP_AUTHORIZED_KEYS.parent
|
||||
if not dry_run:
|
||||
ssh_dir.mkdir(
|
||||
mode=0o750,
|
||||
exist_ok=True,
|
||||
|
|
@ -124,34 +33,34 @@ def sync_repos(username: str, user: User, system_user: dict[str, Any], dry_run=F
|
|||
)
|
||||
|
||||
# create authorized_keys file
|
||||
authorized_keys = ssh_dir / "authorized_keys"
|
||||
authorized_keys = settings.BACKUP_AUTHORIZED_KEYS
|
||||
|
||||
commands = []
|
||||
for key, repos in repos_by_key.items():
|
||||
repo_paths = [
|
||||
f"--restrict-to-repository {str(system_user['homedir'] / repo.name)}"
|
||||
for repo in repos
|
||||
]
|
||||
for key, repositories in repos_by_key.items():
|
||||
repo_paths = [f"--restrict-to-repository {repo.path}" for repo in repositories]
|
||||
commands.append(
|
||||
f"""command="borg serve {" ".join(repo_paths)} --quota=500G",restrict ssh-rsa {key}"""
|
||||
f"""command="cd {str(settings.BACKUP_BORG_DIR)}; borg serve {" ".join(repo_paths)} --storage-quota=500G",restrict {key}"""
|
||||
)
|
||||
|
||||
print("\n".join(commands))
|
||||
|
||||
if not dry_run:
|
||||
authorized_keys.write_text("\n".join(commands) + "\n")
|
||||
|
||||
print(repos)
|
||||
# remove repositories that do no longer exist
|
||||
repo_names = {repo.name for repo in repos}
|
||||
print(repo_names)
|
||||
for dir in homedir.iterdir():
|
||||
print(dir.name)
|
||||
repo_paths = {repo.path for repo in repos}
|
||||
print(repo_paths)
|
||||
for user_dir in settings.BACKUP_BORG_DIR.iterdir():
|
||||
print(user_dir)
|
||||
for dir in user_dir.iterdir():
|
||||
if dir.is_dir() and not dir.name.startswith("."):
|
||||
if dir.name not in repo_names:
|
||||
if dir not in repo_paths:
|
||||
print(f"removing unused repo '{dir}'")
|
||||
if not dry_run:
|
||||
shutil.rmtree(dir)
|
||||
|
||||
# create the repositories
|
||||
for repo in repos:
|
||||
path = system_user["homedir"] / repo.name
|
||||
path.mkdir(mode=0o750, exist_ok=True, parents=True)
|
||||
shutil.chown(path, user=username)
|
||||
print(f"creating repo {repo}")
|
||||
if not dry_run:
|
||||
repo.path.mkdir(mode=0o750, exist_ok=True, parents=True)
|
||||
shutil.chown(repo.path, user=settings.BACKUP_USER)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,9 @@
|
|||
from django.core.management.base import BaseCommand, CommandError
|
||||
from ...models import BorgRepository
|
||||
from ...deployments import sync_users
|
||||
from django.core.management.base import BaseCommand
|
||||
from ...deployments import sync_repos
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Synchronized users on the host with the database."
|
||||
|
||||
def handle(self, *args, **options):
|
||||
|
||||
repos = BorgRepository.objects.all()
|
||||
|
||||
for repo in repos:
|
||||
self.stdout.write(f"{repo}")
|
||||
self.stdout.write(self.style.SUCCESS("This is a test"))
|
||||
|
||||
sync_users(dry_run=False)
|
||||
sync_repos(dry_run=False)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
from django.db import models
|
||||
from django.contrib.auth.models import User
|
||||
from django.core.validators import RegexValidator
|
||||
from django.conf import settings
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
from .tasks import update_user
|
||||
|
||||
|
||||
# Create your models here.
|
||||
|
|
@ -34,25 +39,22 @@ class BorgRepository(models.Model):
|
|||
return f"BorgRepository '{self.user.username}' - '{self.name}' ({self.pk})"
|
||||
|
||||
@property
|
||||
def username(self) -> str:
|
||||
"""Returns the username of the linux user account for this user."""
|
||||
return f"u-{self.pk}"
|
||||
def path(self) -> Path:
|
||||
return settings.BACKUP_BORG_DIR / str(self.user.pk) / str(self.pk)
|
||||
|
||||
@property
|
||||
def repo_url(self) -> str:
|
||||
return f"{settings.BACKUP_USER}@{settings.BACKUP_REPO_HOST}:{self.user.pk}/{self.pk}"
|
||||
|
||||
def save(self):
|
||||
from .tasks import update_user
|
||||
|
||||
update_user.enqueue(user_pk=self.user.pk)
|
||||
super().save()
|
||||
update_user.enqueue()
|
||||
|
||||
def delete(self):
|
||||
from .tasks import update_user
|
||||
|
||||
super().delete()
|
||||
update_user.enqueue(user_pk=self.user.pk)
|
||||
update_user.enqueue()
|
||||
|
||||
|
||||
class Voucher(models.Model):
|
||||
|
|
|
|||
|
|
@ -1,12 +1,7 @@
|
|||
from django.tasks import task
|
||||
from .deployments import sync_repos, get_users
|
||||
from django.contrib.auth.models import User
|
||||
from .deployments import sync_repos
|
||||
|
||||
|
||||
@task
|
||||
def update_user(user_pk):
|
||||
user = User.objects.get(pk=user_pk)
|
||||
username = f"u-{user.pk}"
|
||||
system_user = get_users()[username]
|
||||
|
||||
sync_repos(username=username, user=user, system_user=system_user)
|
||||
def update_user():
|
||||
sync_repos()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue