From 23cf397581cce7fc0a482ae529bb9db5e4fa6185 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 19 Feb 2025 06:02:46 +0000 Subject: [PATCH] S3 Data Storage (#1) Using the storage options [here](https://docs.gitea.com/administration/config-cheat-sheet#storage-storage), this commit pushes all packages, avatars, and LFS files to S3. - Uses AWS S3. - Frees up storage on EC2 instance, so packages do not eventually steal all of the free space. Reviewed-on: https://git.maximhutz.com/web/git/pulls/1 Co-authored-by: Max Co-committed-by: Max --- Taskfile.yml | 4 +++- compose.dev.yml | 10 ++++++++-- gitea/Dockerfile.dev | 2 ++ gitea/config/app.ini | 7 +++++++ gitea/config/dev.app.ini | 13 +++++++++++-- playbooks/fast.yml | 2 ++ playbooks/slow.yml | 15 ++++++--------- terraform/install.sh | 3 ++- terraform/main.tf | 2 +- 9 files changed, 42 insertions(+), 16 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index d118a19..f18c3ea 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -4,7 +4,9 @@ includes: tf: { taskfile: terraform, dir: terraform } tasks: - dev: docker compose -f compose.dev.yml up --build --force-recreate --no-deps + dev: + - docker compose -f compose.dev.yml rm -fsv + - docker compose -f compose.dev.yml up --build --force-recreate --no-deps deploy:fast: ansible-playbook playbooks/fast.yml deploy:slow: ansible-playbook playbooks/slow.yml diff --git a/compose.dev.yml b/compose.dev.yml index 5852bf1..19661db 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -3,8 +3,10 @@ services: # Gitea itself. gitea: container_name: web-git-instance + restart: unless-stopped depends_on: - backup + - bucket-script build: context: gitea dockerfile: Dockerfile.dev @@ -31,7 +33,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro environment: AWS_ENDPOINT: localstack:4566 - AWS_S3_BUCKET_NAME: test + AWS_S3_BUCKET_NAME: backup AWS_ACCESS_KEY_ID: _ AWS_SECRET_ACCESS_KEY: _ BACKUP_CRON_EXPRESSION: "* * * * *" @@ -59,7 +61,11 @@ services: AWS_ACCESS_KEY_ID: _ AWS_SECRET_ACCESS_KEY: _ AWS_ENDPOINT_URL: http://localstack:4566 - command: '"aws s3api create-bucket --bucket test"' + command: | + " + aws s3api create-bucket --bucket backup + aws s3api create-bucket --bucket storage + " volumes: data: diff --git a/gitea/Dockerfile.dev b/gitea/Dockerfile.dev index 1a43e50..7f6b027 100644 --- a/gitea/Dockerfile.dev +++ b/gitea/Dockerfile.dev @@ -3,6 +3,8 @@ FROM gitea/gitea:latest-rootless ADD --chown=git:git config /etc/gitea ADD --chown=git:git custom /etc/gitea-custom +ENV GITEA_CUSTOM=/etc/gitea-custom + RUN rm /etc/gitea/app.ini RUN mv /etc/gitea/dev.app.ini /etc/gitea/app.ini diff --git a/gitea/config/app.ini b/gitea/config/app.ini index 80fc288..8fb2a66 100644 --- a/gitea/config/app.ini +++ b/gitea/config/app.ini @@ -94,3 +94,10 @@ DEFAULT_MERGE_STYLE = merge [repository.signing] DEFAULT_TRUST_MODEL = committer + +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.us-east-1.amazonaws.com +MINIO_BUCKET = myrica-faya +MINIO_USE_SSL = true +MINIO_INSECURE_SKIP_VERIFY = false \ No newline at end of file diff --git a/gitea/config/dev.app.ini b/gitea/config/dev.app.ini index 194b865..f17a670 100644 --- a/gitea/config/dev.app.ini +++ b/gitea/config/dev.app.ini @@ -32,8 +32,8 @@ PROTOCOL = https ROOT_URL = https://localhost:443/ DOMAIN = localhost HTTP_PORT = 443 -CERT_FILE = cert.pem -KEY_FILE = key.pem +CERT_FILE = /etc/gitea-custom/cert.pem +KEY_FILE = /etc/gitea-custom/key.pem [database] DB_TYPE = sqlite3 @@ -95,3 +95,12 @@ DEFAULT_TRUST_MODEL = committer [oauth2] JWT_SECRET = x-----------------------------------------x + +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = localstack:4566 +MINIO_ACCESS_KEY_ID = test +MINIO_SECRET_ACCESS_KEY = test +MINIO_BUCKET = storage +MINIO_USE_SSL = false +MINIO_INSECURE_SKIP_VERIFY = true \ No newline at end of file diff --git a/playbooks/fast.yml b/playbooks/fast.yml index 6258bff..cbc46f7 100644 --- a/playbooks/fast.yml +++ b/playbooks/fast.yml @@ -63,6 +63,8 @@ GITEA__server__SSH_DOMAIN: "{{ full_domain.value }}" GITEA__server__DOMAIN: "{{ full_domain.value }}" GITEA__server__ROOT_URL: "https://{{ full_domain.value }}/" + GITEA__storage__MINIO_ACCESS_KEY_ID: "{{ minio_access_key }}" + GITEA__storage__MINIO_SECRET_ACCESS_KEY: "{{ minio_secret_key }}" labels: docker-volume-backup.stop-during-backup: "true" volumes: diff --git a/playbooks/slow.yml b/playbooks/slow.yml index a8b1966..f851fb0 100644 --- a/playbooks/slow.yml +++ b/playbooks/slow.yml @@ -11,22 +11,17 @@ rebuild: always pull: true - - name: Make temp file. - ansible.builtin.tempfile: - suffix: .tar - register: tar_file - - name: Push image to archive. community.docker.docker_image: name: "{{ image_name }}" - archive_path: "{{ tar_file.path }}" + archive_path: ../dist/image.tar source: local - name: Compress archive to artifact. register: compress_image community.general.archive: - path: "{{ tar_file.path }}" - dest: "{{ tar_file.path }}.xz" + path: ../dist/image.tar + dest: ../dist/image.tar.xz format: xz mode: "0644" @@ -34,7 +29,7 @@ amazon.aws.s3_object: bucket: "{{ image_bucket }}" object: "{{ image_key }}" - src: "{{ tar_file.path }}.xz" + src: ../dist/image.tar.xz mode: put region: "{{ aws_region }}" @@ -98,6 +93,8 @@ GITEA__server__SSH_DOMAIN: "{{ full_domain.value }}" GITEA__server__DOMAIN: "{{ full_domain.value }}" GITEA__server__ROOT_URL: "https://{{ full_domain.value }}/" + GITEA__storage__MINIO_ACCESS_KEY_ID: "{{ minio_access_key }}" + GITEA__storage__MINIO_SECRET_ACCESS_KEY: "{{ minio_secret_key }}" labels: docker-volume-backup.stop-during-backup: "true" volumes: diff --git a/terraform/install.sh b/terraform/install.sh index bcb2efd..895828d 100755 --- a/terraform/install.sh +++ b/terraform/install.sh @@ -1,5 +1,6 @@ #!/bin/sh +## Install extras. rpm --rebuilddb amazon-linux-extras install docker ansible2 python3.8 -y @@ -10,7 +11,7 @@ systemctl start docker # Set up the correct version of Python (for Ansible). ln -sf /usr/bin/python3.8 /usr/bin/python3 ln -sf /usr/bin/pip3.8 /usr/bin/pip3 -pip3 install botocore boto3 requests packaging +pip3 install botocore boto3 requests packaging --user ssm-user python3 -m pip install -U pip # Add some swap space. diff --git a/terraform/main.tf b/terraform/main.tf index 12a99ae..69704ec 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -18,7 +18,7 @@ resource "aws_instance" "this" { subnet_id = module.vpc.public_subnets[0] user_data = file("install.sh") - user_data_replace_on_change = false + user_data_replace_on_change = true iam_instance_profile = aws_iam_instance_profile.ssm.name vpc_security_group_ids = [aws_security_group.public_access.id]