From 22a3a6fbed7d4088c43aafd50ededf99f46e7bf0 Mon Sep 17 00:00:00 2001 From: apb Date: Fri, 12 Nov 2021 16:14:45 +0000 Subject: [PATCH] [add] gitea --- gerbera/docker-compose.yml | 2 +- gitea/README.md | 4 +- gitea/backup_gitea.sh | 58 ++++++++ gitea/data/.gitignore | 8 ++ gitea/docker-compose.yml | 22 +++ gitea/reset-gitea-mysql.sql | 5 + gitea/restore_gitea_container.sh | 182 +++++++++++++++++++++++ gitea/restore_gitea_data.sh | 238 +++++++++++++++++++++++++++++++ 8 files changed, 517 insertions(+), 2 deletions(-) create mode 100644 gitea/backup_gitea.sh create mode 100644 gitea/data/.gitignore create mode 100644 gitea/docker-compose.yml create mode 100644 gitea/reset-gitea-mysql.sql create mode 100644 gitea/restore_gitea_container.sh create mode 100644 gitea/restore_gitea_data.sh diff --git a/gerbera/docker-compose.yml b/gerbera/docker-compose.yml index a487247..2ac9a5a 100644 --- a/gerbera/docker-compose.yml +++ b/gerbera/docker-compose.yml @@ -1,4 +1,4 @@ - +--- version: '2.0' services: diff --git a/gitea/README.md b/gitea/README.md index e7202cc..e33d255 100644 --- a/gitea/README.md +++ b/gitea/README.md @@ -1,3 +1,5 @@ # Gitea -@TODO \ No newline at end of file +##backup scripts + +from https://gist.github.com/sinbad/4bb771b916fa8facaf340af3fc49ee43 \ No newline at end of file diff --git a/gitea/backup_gitea.sh b/gitea/backup_gitea.sh new file mode 100644 index 0000000..4486a1f --- /dev/null +++ b/gitea/backup_gitea.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# `gitea dump` doesn't currently back up LFS data as well, only git repos +# It primarily backs up the SQL DB, and also the config / logs +# We'll backup like this: +# * "gitea dump" to backup the DB and config etc +# * tar / bzip all the repos since they will be skipped +# * Not rotated because git data is immutable (normally) so has all data +# * rsync LFS data directly from /volume/docker/gitea/git/lfs +# * No need for rotation since all files are immutable +# +# This means our backup folder will contain: +# * /gitea_data.zip - containing the gitea data +# * /repositories/ - containing the bundles, structured owner/name.bundle +# * /lfs/ - containing all the direct LFS data +# +# Stop on errors +set -e + +# Gitea config / SQL DB backup rotation +CONTAINER=gitea_server_1 +# Backup dir from our perspective +HOST_BACKUP_DIR="/volume1/backups/gitea" +# Git repo dir from our perspective (it's outside container) +HOST_GIT_REPO_DIR="/volume1/docker/gitea/git/repositories" +# Git LFS dir from our perspective (it's outside container) +HOST_GIT_LFS_DIR="/volume1/docker/gitea/git/lfs" +# Where we work on things (host and container) +TEMP_DIR="/tmp" + +GITEA_DATA_FILENAME="gitea_backup.zip" +HOST_BACKUP_FILE="$HOST_BACKUP_DIR/$GITEA_DATA_FILENAME" + +# Back up to temp files then copy on success to prevent syncing incomplete/bad files +CONTAINER_BACKUP_FILE_TEMP="$TEMP_DIR/gitea_dump_temp.zip" +docker exec -u git -i $(docker ps -qf "name=$CONTAINER") bash -c "rm -f $CONTAINER_BACKUP_FILE_TEMP" + +echo Backing up Gitea data to $HOST_BACKUP_FILE via $CONTAINER:$CONTAINER_BACKUP_FILE_TEMP +docker exec -u git -i $(docker ps -qf "name=$CONTAINER") bash -c "/app/gitea/gitea dump --skip-repository --skip-log --file $CONTAINER_BACKUP_FILE_TEMP" +# copy this into backup folder (in container) +docker cp $CONTAINER:$CONTAINER_BACKUP_FILE_TEMP $HOST_BACKUP_FILE + +echo Backing up git repositories +# Git repos are in 2-level structure, owner/repository +# Again we MUST tar to a TEMP file and move into place when successful +GITREPO_BACKUP_FILE="$HOST_BACKUP_DIR/gitrepos_backup.tar.bz2" +GITREPO_BACKUP_FILE_TEMP=`mktemp -p $TEMP_DIR gitrepos_backup.tar.bz2.XXXXXX` +tar cjf $GITREPO_BACKUP_FILE_TEMP -C $HOST_GIT_REPO_DIR . +mv -f $GITREPO_BACKUP_FILE_TEMP $GITREPO_BACKUP_FILE + +echo Backing up LFS data +# This syncs path/to/lfs to backup/dir/ +# This will then be replicated directly to B2 +# Yes this means we're storing LFS data twice but I prefer this to syncing to B2 +# directly from the data dir, it makes the B2 sync simpler (just the whole folder) +rsync -rLptgo $HOST_GIT_LFS_DIR $HOST_BACKUP_DIR/ + +echo Gitea backup completed successfully diff --git a/gitea/data/.gitignore b/gitea/data/.gitignore new file mode 100644 index 0000000..1e04ba0 --- /dev/null +++ b/gitea/data/.gitignore @@ -0,0 +1,8 @@ +# .gitignore sample +################### + +# Ignore all files in this dir... +* + +# ... except for this one. +!.gitignore diff --git a/gitea/docker-compose.yml b/gitea/docker-compose.yml new file mode 100644 index 0000000..e9cf2a6 --- /dev/null +++ b/gitea/docker-compose.yml @@ -0,0 +1,22 @@ +--- +version: "2.1" +services: + gitea: + image: gitea/gitea:latest + container_name: gitea + environment: + - USER_UID=1000 + - USER_GID=1000 + volumes: + - ./data:/data + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + ports: + - "8300:3000" + - "222:22" + restart: unless-stopped + +networks: + default: + external: + name: swag_default diff --git a/gitea/reset-gitea-mysql.sql b/gitea/reset-gitea-mysql.sql new file mode 100644 index 0000000..3979a50 --- /dev/null +++ b/gitea/reset-gitea-mysql.sql @@ -0,0 +1,5 @@ +DROP DATABASE IF EXISTS gitea; +CREATE DATABASE gitea CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_unicode_ci'; +GRANT ALL PRIVILEGES ON gitea.* TO 'gitea'; +FLUSH PRIVILEGES; +exit \ No newline at end of file diff --git a/gitea/restore_gitea_container.sh b/gitea/restore_gitea_container.sh new file mode 100644 index 0000000..b2dbbd9 --- /dev/null +++ b/gitea/restore_gitea_container.sh @@ -0,0 +1,182 @@ +#!/bin/bash + +usage () { + echo "Re-create a Gitea Docker container from a backup" + echo "Run this inside a docker-compose config folder!" + echo "If it's the test container (gitea_test) it'll be automated" + echo "If not, it will give you commands to run to complete the restore" + echo "Usage:" + echo " restore_gitea_container.sh [--unsafe] [--dry-run]" + echo "" + echo "Options:" + echo " --unsafe : Perform actions EVEN OUTSIDE gitea_test container" + echo " : BE CAREFUL with this, it will stomp your container" + echo " --dry-run : Only list actions, don't perform them" + echo "" +} +if [[ "$1" == "--help" ]]; then + usage + exit 0 +fi + +# MAKE SURE we're in the test dir +PWD=`pwd` +MANUAL_GUIDE=0 +IS_TEST=1 + +if [[ ! -f "$PWD/docker-compose.yml" ]]; then + echo "You must run this inside a folder containing docker-compose.yml! Aborting" + exit 1 +fi + +if [[ ! "$PWD" == */docker-compose/gitea_test ]]; then + echo "HEY! You're not running this in docker-compose/gitea_test" + echo "So we're assuming this is a live instance and will not execute anything automatically" + echo "Instead, we'll print the commands you need to run." + MANUAL_GUIDE=1 + IS_TEST=0 +fi + +while (( "$#" )); do + if [[ "$1" == "--dry-run" ]]; then + MANUAL_GUIDE=1 + elif [[ "$1" == "--unsafe" ]]; then + while true; do + echo "Using --unsafe will destroy & re-create this container in all cases" + echo " It will probably get the ports wrong, be ready to edit app.ini afterwards!" + read -p "Are you SURE this is what you want? (y/n)" yn + case $yn in + [Yy]* ) break;; + [Nn]* ) exit;; + * ) echo "Please answer yes or no.";; + esac + done + MANUAL_GUIDE=0 + else + if [[ "$1" == -* ]]; then + echo Unrecognised option $1 + usage + exit 3 + fi + fi + shift # $2 becomes $1.. +done + + +# Generate all paths based on the last path component +# Root of all the host versions of what gets mapped to /data in container +DATAROOT="/volume1/docker" +# Backup source folder, contains gitea_backup.zip, gitrepos_backup.tar.bz2 & lfs +BACKUPSRC="/volume1/backups/gitea" +CONFIG=$(basename $PWD) +DATADIR="/volume1/docker/$CONFIG" + +echo "Removing container" +# Use --all in case it's been run manually +if (( $MANUAL_GUIDE )); then + echo "Run this:" + echo " > docker-compose stop" + echo " > docker-compose rm" +else + docker-compose stop + docker-compose rm +fi +echo "Removing old Gitea data" +if (( $MANUAL_GUIDE )); then + echo "Run this:" + echo " > rm -r $DATADIR/*" +else + rm -r $DATADIR/* +fi + +echo "Re-creating container" +if (( $MANUAL_GUIDE )); then + echo "Run this:" + echo " > docker-compose up --no-start" +else + docker-compose up --no-start +fi + +# We now need to bring up the database server to restore the MySQL data +# Bring it up early so that it's got time to start while we do the data copying +echo "Bringing up database to restore" +if (( $MANUAL_GUIDE )); then + echo "Run this:" + echo " > docker-compose start db" +else + docker-compose start db +fi + +# Run the restore script to get back the contents of docker/gitea data folder +echo Restoring Gitea, Git and Git-LFS data +# Copy SQL to MySQL's folder (from host perspective) +MYSQLDATADIR="/volume1/docker/${CONFIG}_db" +MYSQLFILE=`mktemp -t gitea-db.sql.XXXXXX` +if (( $MANUAL_GUIDE )); then + echo "Run this:" + echo " > ../../backups/restore_gitea_data.sh $BACKUPSRC $DATADIR $MYSQLFILE" +else + ../../backups/restore_gitea_data.sh $BACKUPSRC $DATADIR $MYSQLFILE +fi + +# Restore DB +# We need to make sure it's up, it can take a little time before connections +# are allowed +MYSQL_ATTEMPTS=3 +while [[ $MYSQL_ATTEMPTS -gt 0 ]] ; do + echo "Testing if MySQL is up" + let MYSQL_ATTEMPTS-- + if docker-compose exec db mysqladmin -uroot -pDB_ROOT_PASSWORD status; then + break + fi + sleep 2 +done + +# Our docker container creates the gitea user and gitea DB in all cases +# Drop all tables first +# Can't just pipe in data to docker-compose exec because bug https://github.com/docker/compose/issues/3352 +# Fixed but not in the Synology version +# We can use main docker but need to parse out the ID for alias 'db' +DB_DOCKER_ID=$(docker-compose ps -q db) +if (( $MANUAL_GUIDE )); then + echo "Run this:" + echo " > docker exec -i $DB_DOCKER_ID mysql -uroot -pDB_ROOT_PASSWORD < ../../sql/reset-gitea-mysql.sql" + echo "> docker exec -i $DB_DOCKER_ID mysql -ugitea -pDB_GITEA_PASSWORD gitea < $MYSQLFILE" +else + echo "Restoring database....be patient!" + docker exec -i $DB_DOCKER_ID mysql -uroot -pDB_ROOT_PASSWORD < ../../sql/reset-gitea-mysql.sql + docker exec -i $DB_DOCKER_ID mysql -ugitea -pDB_GITEA_PASSWORD gitea < $MYSQLFILE +fi + +# Clean up +rm -f $MYSQLFILE + + +# Need to modify app.ini to change ports on URLs +if (( $IS_TEST )); then + echo Fixing up ports + if (( $MANUAL_GUIDE )); then + echo "You need to edit $DATADIR/gitea/conf.app.ini, change:" + echo " - ROOT_URL = https://git.yourserver.com:9000/" + echo " + ROOT_URL = https://git.yourserver.com:10000/" + echo " - SSH_PORT = 9022" + echo " - SSH_PORT = 10022" + else + sed -i "s/\.com:9000/.com:10000" $DATADIR/gitea/conf/app.ini + sed -i "s/9022/10022" $DATADIR/gitea/conf/app.ini + fi +else + echo "Since this isn't the test environment, you'll need to check $DATADIR/gitea/conf.app.ini manually!" +fi +# This will have created the missing ssh folder w/ server config etc +echo "Restoration complete" +echo "Restarting Server" +if (( $MANUAL_GUIDE )); then + echo "Run this:" + echo " > docker-compose up -d" +else + docker-compose up -d +fi + +echo "NOTE: SSH keys will likely not work if you're restoring to another server" +echo " Users will probably have to remove & re-add their keys in Settings" \ No newline at end of file diff --git a/gitea/restore_gitea_data.sh b/gitea/restore_gitea_data.sh new file mode 100644 index 0000000..d3447f6 --- /dev/null +++ b/gitea/restore_gitea_data.sh @@ -0,0 +1,238 @@ +#!/bin/bash + +# See backup_gitea.sh +# Source to restore should include: +# - gitea_backup.zip +# - gitrepos_backup.tar.bz2 +# - lfs/*/*/* + + +# Exit on error +set -e + +usage () { + echo "Script for restoring Gitea data from a backup. Will REPLACE existing data!!" + echo "You probably DON'T WANT THIS SCRIPT directly, it only gets the data files back" + echo "Look at restore_gitea_container.sh for a fully automated Docker container & DB restore" + echo "Usage:" + echo " restore_gitea_data.sh [--dry-run] " + echo "" + echo "Params:" + echo " source_dir : Directory containing Gitea backup" + echo " dest_dir : Directory to write data to (host, mapped to /data in container)" + echo " sql_file_dest : Full file path of where to place gitea-db.sql from backup" + echo "Options:" + echo " --dry-run : Don't actually perform actions, just report" + echo "" +} +if [[ "$1" == "--help" ]]; then + usage + exit 0 +fi + +DRYRUN=0 +SOURCE="" +DATADIR="" +SQLDEST="" +USER_UID=1000 +GROUP_GID=1000 + +while (( "$#" )); do + if [[ "$1" == "--dry-run" ]]; then + DRYRUN=1 + else + if [[ "$1" == -* ]]; then + echo Unrecognised option $1 + usage + exit 3 + # Populate positional args + elif [[ "$SOURCE" == "" ]]; then + SOURCE=$1 + elif [[ "$DATADIR" == "" ]]; then + DATADIR=$1 + else + SQLDEST=$1 + fi + fi + shift # $2 becomes $1.. +done + +if [[ "$SOURCE" == "" ]]; then + echo "Required: source folder" + usage + exit 3 +fi + +if [[ "$DATADIR" == "" ]]; then + echo "Required: destination data dir" + usage + exit 3 +fi + +echo Checking required files exist in $SOURCE +if [[ ! -f "$SOURCE/gitea_backup.zip" ]]; then + echo "ERROR: Missing file in restore $SOURCE/gitea_backup.zip" + exit 5 +fi +if [[ ! -f "$SOURCE/gitrepos_backup.tar.bz2" ]]; then + echo "ERROR: Missing file in restore $SOURCE/gitrepos_backup.tar.bz2" + exit 5 +fi +if [[ ! -d "$SOURCE/lfs" ]]; then + echo "ERROR: Missing directory in restore $SOURCE/lfs" + exit 5 +fi + +# The only thing we can't restore is the gitea/ssh folder, which contains the +# server SSH keys. We leave that for Gitea to re-create on first start +echo Checking container data +if [[ ! -d "$DATADIR/gitea" ]]; then + if (( $DRYRUN )); then + echo "Would have created $DATADIR/gitea" + else + echo "Creating $DATADIR/gitea" + mkdir -p $DATADIR/gitea + chown -R $USER_UID:$GROUP_GID $DATADIR/gitea + chmod -R u+rwX,go+rX,go-w $DATADIR/gitea + fi +fi +if [[ ! -d "$DATADIR/git/repositories" ]]; then + if (( $DRYRUN )); then + echo "Would have created $DATADIR/git/repositories" + else + echo "Creating $DATADIR/git/repositories" + mkdir -p $DATADIR/git/repositories + chown -R $USER_UID:$GROUP_GID $DATADIR/git/repositories + chmod -R u+rwX,go+rX,go-w $DATADIR/git/repositories + fi +fi +if [[ ! -d "$DATADIR/git/lfs" ]]; then + if (( $DRYRUN )); then + echo "Would have created $DATADIR/git/lfs" + else + echo "Creating $DATADIR/git/lfs" + mkdir -p $DATADIR/git/lfs + chown -R $USER_UID:$GROUP_GID $DATADIR/git/lfs + chmod -R u+rwX,go+rX,go-w $DATADIR/git/lfs + fi +fi + + +GITEA_DATA_FILE="$SOURCE/gitea_backup.zip" +echo "** Step 1 of 3: Gitea data START **" +echo "Copying Gitea files back from $GITEA_DATA_FILE" +# gitea_backup.zip contains: + +# gitea-db.sql - database dump in SQL form +# app.ini - same as custom/conf/app.ini +# +# custom/ - All subfolders go back to data folder +# conf/ +# log/ +# queues/ +# gitea.db - The actual SQLite DB file, seems the same? +# indexers/ +# sessions/ +# avatars/ +# data/ - Again, all subfolders go back to data, same as custom?? +# conf/ +# log/ +# queues/ +# gitea.db - Actual sqlite data file +# indexers/ +# sessions/ +# avatars/ +# log/ - Log files, we don't need these (will remove from backup script) + +# So there seems to be a lot of duplication in this dump +# Even when I don't "gitea dump" with -C it still creates custom/ and data/ +# I think we only want the /data dir +# Extract all to temp then copy + +TEMPDIR=`mktemp -d` +# protect! because we're going to rm -rf this later +if [[ ! "$TEMPDIR" == /tmp/* ]]; then + echo Error: expected mktemp to give us a dir in /tmp, being careful & aborting +fi +echo "Extracting archive..." +# We have to use 7z because that comes pre-installed on Synology but unzip doesn't +# Send to /dev/null as 7z writes a bunch of crap that doesn't translate well to some terminals +7z x -o$TEMPDIR $GITEA_DATA_FILE > /dev/null 2>&1 +# Unfortunately 7z doesn't restore file permissions / ownership +# Docker Gitea has everything owned by 1000:1000 +echo "Fixing permissions" +chown -R $USER_UID:$GROUP_GID $TEMPDIR +# And permissions are 755/644 for dirs / files +# The capital X only sets x bit on dirs, not files, which gives us 755/644 +chmod -R u+rwX,go+rX,go-w $TEMPDIR +GITEA_DEST="$DATADIR/gitea" +if (( $DRYRUN )); then + echo "Would have copied data directory $TEMPDIR/data to $GITEA_DEST" + echo "This was the structure:" + ls -la $TEMPDIR/data +else + echo "Copying data directory $TEMPDIR/data to $GITEA_DEST" + # Note -p is ESSENTIAL to preserve ownership + cp -p -R -f $TEMPDIR/data/* $GITEA_DEST/ +fi + +if [[ ! "$SQLDEST" == "" ]]; then + if (( $DRYRUN )); then + echo "Would have copied $TEMPDIR/gitea-db.sql to $SQLDEST" + else + echo "Copying $TEMPDIR/gitea-db.sql to $SQLDEST" + # Note -p is ESSENTIAL to preserve ownership + cp -p -f $TEMPDIR/gitea-db.sql $SQLDEST + fi +fi + +rm -rf $TEMPDIR +echo "** Step 1 of 3: Gitea data DONE **" + +# Now do repositories +# tar preserves owner/permissions, and we tarred relative to data/git/repositories +# so we can do this direct +# remove all existing data so it's clean +GIT_REPO_FILE="$SOURCE/gitrepos_backup.tar.bz2" +GIT_REPO_DEST="$DATADIR/git/repositories" +echo "** Step 2 of 3: Git repository data START **" +echo "Restoring git repository data" +if (( $DRYRUN )); then + echo "Would have deleted $GIT_REPO_DEST/*" + echo "Would have extracted $GIT_REPO_FILE to $GIT_REPO_DEST" + echo "Repositories were: " + # equivalent of depth = 2 + tar --exclude="*/*/*/*" -tf $GIT_REPO_FILE + +else + echo "Cleaning existing repo data" + rm -rf $GIT_REPO_DEST/* + echo "Extracting $GIT_REPO_FILE to $GIT_REPO_DEST" + tar -xf $GIT_REPO_FILE -C $GIT_REPO_DEST +fi +echo "Git repositories done" +echo "** Step 2 of 3: Git repository data DONE **" + + +echo "** Step 3 of 3: Git-LFS data START **" +echo "Restoring Git-LFS data" +GIT_LFS_SRC="$SOURCE/lfs" +GIT_LFS_DEST="$DATADIR/git/lfs" +# There is no reason to delete LFS data since it's all immutable +# Instead rsync back like we did during backup +if (( $DRYRUN )); then + echo "Would have synced LFS data from $GIT_LFS_SRC to $GIT_LFS_DEST" +else + echo "Syncing LFS data from $GIT_LFS_SRC to $GIT_LFS_DEST" + rsync -rLptgo $GIT_LFS_SRC/* $GIT_LFS_DEST/ + + echo "Fixing LFS permissions" + chown -R $USER_UID:$GROUP_GID $GIT_LFS_DEST + # And permissions are 755/644 for dirs / files + # The capital X only sets x bit on dirs, not files, which gives us 755/644 + chmod -R u+rwX,go+rX,go-w $GIT_LFS_DEST + +fi + +echo "** Step 3 of 3: Git-LFS data DONE **" +echo "Gitea data restored successfully"