From de96c5f06d0c92393e7953547e6b1e678105fbe0 Mon Sep 17 00:00:00 2001 From: Christian Winther Date: Mon, 15 Jan 2024 23:50:16 +0000 Subject: [PATCH] migration docs --- .env.docker | 360 +++++++++++------- .markdownlint.json | 4 + Dockerfile | 2 +- docker-compose.migrate.yml | 42 ++ docker-compose.yml | 35 +- docker/README.md | 1 + docker/customizing.md | 6 +- docker/faq.md | 2 + docker/migration.md | 326 ++++++++++++++++ .../docker/entrypoint.d/01-permissions.sh | 6 +- 10 files changed, 627 insertions(+), 157 deletions(-) create mode 100644 .markdownlint.json create mode 100644 docker-compose.migrate.yml create mode 100644 docker/migration.md diff --git a/.env.docker b/.env.docker index 5dfd766bb..5f5152968 100644 --- a/.env.docker +++ b/.env.docker @@ -1,60 +1,24 @@ -# shellcheck disable=SC2034,SC2148 - # -*- mode: bash -*- # vi: ft=bash -############################################################### -# Docker-wide configuration -############################################################### +# shellcheck disable=SC2034,SC2148 -# Path (relative to the docker-compose.yml) or absolute (/some/other/path) file where containers will store their data -DOCKER_DATA_ROOT="./docker-compose/data" - -# Path (relative to the docker-compose.yml) or absolute (/some/other/path) file where containers will store their confguration -DOCKER_CONFIG_ROOT="./docker-compose/config" - -# Image to pull the Pixelfed Docker images from -# -# Possible values: -# - "ghcr.io/pixelfed/pixelfed" to pull from GitHub -# - "pixelfed/pixelfed" to pull from DockerHub -# -DOCKER_IMAGE="ghcr.io/jippi/pixelfed" - -# Pixelfed version (image tag) to pull from the registry. -# -# See: https://github.com/pixelfed/pixelfed/pkgs/container/pixelfed -DOCKER_TAG="branch-jippi-fork-apache-8.1" - -# Set timezone used by *all* containers - these should be in sync. -# -# See: https://www.php.net/manual/en/timezones.php -TZ="UTC" - -# Automatically run "One-time setup tasks" commands. -# -# If you are migrating to this docker-compose setup or have manually run the "One time seutp" -# tasks (https://docs.pixelfed.org/running-pixelfed/installation/#setting-up-services) -# you can set this to "0" to prevent them from running. -# -# Otherwise, leave it at "1" to have them run *once*. -DOCKER_RUN_ONE_TIME_SETUP_TASKS="1" - -# The e-mail to use for Lets Encrypt certificate requests. -LETSENCRYPT_EMAIL="__CHANGE_ME__" - -# Lets Encrypt staging/test servers for certificate requests. -# -# Setting this to any value will change to letsencrypt test servers. -#LETSENCRYPT_TEST="1" - -# Enable Docker Entrypoint debug mode (will call [set -x] in bash scripts) -# by setting this to "1". -#ENTRYPOINT_DEBUG="0" - -############################################################### +################################################################################ # Pixelfed application configuration -############################################################### +################################################################################ + +# The docker tag prefix to use for pulling images, can be one of +# +# * latest +# * +# * staging +# * edge +# * branch- +# * pr- +# +# Combined with [DOCKER_RUNTIME] and [PHP_VERSION] configured +# elsewhere in this file, the final Docker tag is computed. +PIXELFED_RELEASE="branch-jippi-fork" # A random 32-character string to be used as an encryption key. # @@ -80,7 +44,7 @@ APP_DOMAIN="__CHANGE_ME__" # You should set this to the root of your application so that it is used when running Artisan tasks. # # See: https://docs.pixelfed.org/technical-documentation/config/#app_url -APP_URL=https://${APP_DOMAIN} +APP_URL="https://${APP_DOMAIN}" # Application domains used for routing. # @@ -104,7 +68,7 @@ ADMIN_DOMAIN="${APP_DOMAIN}" # Enable/disable new local account registrations. # # See: https://docs.pixelfed.org/technical-documentation/config/#open_registration -#OPEN_REGISTRATION=true +#OPEN_REGISTRATION="true" # Require email verification before a new user can do anything. # @@ -130,11 +94,11 @@ OAUTH_ENABLED="true" # Defaults to "UTC". # -# Do not edit your timezone or things will break! +# ! Do not edit your timezone once the service is running - or things will break! # # See: https://docs.pixelfed.org/technical-documentation/config/#app_timezone # See: https://www.php.net/manual/en/timezones.php -APP_TIMEZONE="${TZ}" +APP_TIMEZONE="UTC" # The application locale determines the default locale that will be used by the translation service provider. # You are free to set this value to any of the locales which will be supported by the application. @@ -277,11 +241,27 @@ INSTANCE_CONTACT_EMAIL="admin@${APP_DOMAIN}" # See: https://docs.pixelfed.org/technical-documentation/config/#restricted_instance #RESTRICTED_INSTANCE="false" -############################################################### -# Database configuration -############################################################### +################################################################################ +# Lets Encrypt configuration +################################################################################ -# Here you may specify which of the database connections below you wish to use as your default connection for all database work. +# The host to request LetsEncrypt certificate for +LETSENCRYPT_HOST="${APP_DOMAIN}" + +# The e-mail to use for Lets Encrypt certificate requests. +LETSENCRYPT_EMAIL="__CHANGE_ME__" + +# Lets Encrypt staging/test servers for certificate requests. +# +# Setting this to any value will change to letsencrypt test servers. +#LETSENCRYPT_TEST="1" + +################################################################################ +# Database configuration +################################################################################ + +# Here you may specify which of the database connections below +# you wish to use as your default connection for all database work. # # Of course you may use many connections at once using the database library. # @@ -313,11 +293,11 @@ DB_DATABASE="pixelfed_prod" DB_PORT="3306" # Automatically run [artisan migrate --force] if new migrations are detected. -DB_APPLY_NEW_MIGRATIONS_AUTOMATICALLY="0" +DB_APPLY_NEW_MIGRATIONS_AUTOMATICALLY="false" -############################################################### +################################################################################ # Mail configuration -############################################################### +################################################################################ # Laravel supports both SMTP and PHP’s “mail” function as drivers for the sending of e-mail. # You may specify which one you’re using throughout your application here. @@ -392,9 +372,9 @@ MAIL_FROM_NAME="Pixelfed @ ${APP_DOMAIN}" # See: https://docs.pixelfed.org/technical-documentation/config/#mail_encryption #MAIL_ENCRYPTION="tls" -############################################################### +################################################################################ # Redis configuration -############################################################### +################################################################################ # Defaults to "phpredis". # @@ -419,16 +399,16 @@ REDIS_HOST="redis" # Defaults to 6379. # # See: https://docs.pixelfed.org/technical-documentation/config/#redis_port -REDIS_PORT="6379" +#REDIS_PORT="6379" # Defaults to 0. # # See: https://docs.pixelfed.org/technical-documentation/config/#redis_database #REDIS_DATABASE="0" -############################################################### +################################################################################ # Cache settings -############################################################### +################################################################################ # This option controls the default cache connection that gets used while using this caching library. # @@ -448,11 +428,11 @@ CACHE_DRIVER="redis" # Defaults to ${APP_NAME}_cache, or laravel_cache if no APP_NAME is set. # # See: https://docs.pixelfed.org/technical-documentation/config/#cache_prefix -# CACHE_PREFIX="{APP_NAME}_cache" +#CACHE_PREFIX="{APP_NAME}_cache" -############################################################### +################################################################################ # Horizon settings -############################################################### +################################################################################ # This prefix will be used when storing all Horizon data in Redis. # @@ -498,9 +478,9 @@ CACHE_DRIVER="redis" # See: https://docs.pixelfed.org/technical-documentation/config/#horizon_supervisor_timeout #HORIZON_SUPERVISOR_TIMEOUT="300" -############################################################### +################################################################################ # Experiments -############################################################### +################################################################################ # Text only posts (alpha). # @@ -523,16 +503,16 @@ CACHE_DRIVER="redis" # See: https://docs.pixelfed.org/technical-documentation/config/#exp_cpt #EXP_CPT="false" -# Enforce Mastoapi Compatibility (alpha). +# Enforce Mastodon API Compatibility (alpha). # # Defaults to "true". # # See: https://docs.pixelfed.org/technical-documentation/config/#exp_emc -EXP_EMC="true" +#EXP_EMC="true" -############################################################### +################################################################################ # ActivityPub confguration -############################################################### +################################################################################ # Defaults to "false". # @@ -559,9 +539,9 @@ ACTIVITY_PUB="true" # See: https://docs.pixelfed.org/technical-documentation/config/#ap_outbox #AP_OUTBOX="true" -############################################################### +################################################################################ # Federation confguration -############################################################### +################################################################################ # Defaults to "true". # @@ -578,9 +558,9 @@ ACTIVITY_PUB="true" # See: https://docs.pixelfed.org/technical-documentation/config/#webfinger #WEBFINGER="true" -############################################################### +################################################################################ # Storage (cloud) -############################################################### +################################################################################ # Store media on object storage like S3, Digital Ocean Spaces, Rackspace # @@ -604,9 +584,9 @@ ACTIVITY_PUB="true" # See: https://docs.pixelfed.org/technical-documentation/config/#media_delete_local_after_cloud #MEDIA_DELETE_LOCAL_AFTER_CLOUD="true" -############################################################### +################################################################################ # Storage (cloud) - S3 andS S3 *compatible* providers (most of them) -############################################################### +################################################################################ # See: https://docs.pixelfed.org/technical-documentation/config/#aws_access_key_id #AWS_ACCESS_KEY_ID= @@ -765,10 +745,10 @@ LOG_CHANNEL="stderr" BROADCAST_DRIVER="redis" ############################################################### -# Other settings +# Sanitizing settings ############################################################### -# Defaults to true. +# Defaults to "true". # # See: https://docs.pixelfed.org/technical-documentation/config/#restrict_html_types #RESTRICT_HTML_TYPES="true" @@ -898,52 +878,122 @@ TRUST_PROXIES="*" # PHP configuration ############################################################### +# The PHP version to use for [web] and [worker] container +# +# Any version published on https://hub.docker.com/_/php should work +# +# Example: +# +# * 8.1 +# * 8.2 +# * 8.2.14 +# * latest +# +# Do *NOT* use the full Docker tag (e.g. "8.3.2RC1-fpm-bullseye") +# *only* the version part. The rest of the full tag is derived from +# the [DOCKER_RUNTIME] and [PHP_DEBIAN_RELEASE] settings +PHP_VERSION="8.1" + # See: https://www.php.net/manual/en/ini.core.php#ini.memory-limit #PHP_MEMORY_LIMIT="128M" -############################################################### -# MySQL DB container configuration (DO NOT CHANGE) -############################################################### +# The Debian release variant to use of the [php] Docker image +#PHP_DEBIAN_RELEASE="bullseye" + +# The [php] Docker image base type # -# See "Environment Variables" at https://hub.docker.com/_/mysql +# See: https://github.com/pixelfed/pixelfed/blob/dev/docker/runtimes.md +#PHP_BASE_TYPE="apache" -MYSQL_ROOT_PASSWORD="${DB_PASSWORD}" -MYSQL_USER="${DB_USERNAME}" -MYSQL_PASSWORD="${DB_PASSWORD}" -MYSQL_DATABASE="${DB_DATABASE}" - -############################################################### -# MySQL (MariaDB) DB container configuration (DO NOT CHANGE) -############################################################### +# List of extra APT packages (separated by space) to install when building +# locally using [docker compose build]. # -# See "Start a mariadb server instance with user, password and database" at https://hub.docker.com/_/mariadb +# See: https://github.com/pixelfed/pixelfed/blob/dev/docker/customizing.md +#APT_PACKAGES_EXTRA="" -MARIADB_ROOT_PASSWORD="${DB_PASSWORD}" -MARIADB_USER="${DB_USERNAME}" -MARIADB_PASSWORD="${DB_PASSWORD}" -MARIADB_DATABASE="${DB_DATABASE}" - -############################################################### -# PostgreSQL DB container configuration (DO NOT CHANGE) -############################################################### +# List of *extra* PECL extensions (separated by space) to install when +# building locally using [docker compose build]. # -# See "Environment Variables" at https://hub.docker.com/_/postgres +# See: https://github.com/pixelfed/pixelfed/blob/dev/docker/customizing.md +#PHP_PECL_EXTENSIONS_EXTRA="" -POSTGRES_USER="${DB_USERNAME}" -POSTGRES_PASSWORD="${DB_PASSWORD}" -POSTGRES_DB="${DB_DATABASE}" +# List of *extra* PHP extensions (separated by space) to install when +# building locally using [docker compose build]. +# +# See: https://github.com/pixelfed/pixelfed/blob/dev/docker/customizing.md +#PHP_EXTENSIONS_EXTRA="" -############################################################### -# Lets Encrypt configuration -############################################################### +################################################################################ +# Other configuration +################################################################################ -LETSENCRYPT_HOST="${APP_DOMAIN}" +# ? Add your own configuration here -############################################################### +################################################################################ +# Timezone configuration +################################################################################ + +# Set timezone used by *all* containers - these must be in sync. +# +# ! Do not edit your timezone once the service is running - or things will break! +# +# See: https://www.php.net/manual/en/timezones.php +TZ="${APP_TIMEZONE}" + +################################################################################ # Docker Specific configuration -############################################################### +################################################################################ + +# Image to pull the Pixelfed Docker images from. +# +# Example values: +# +# * "ghcr.io/pixelfed/pixelfed" to pull from GitHub +# * "pixelfed/pixelfed" to pull from DockerHub +# * "your/fork" to pull from a custom fork +# +DOCKER_IMAGE="ghcr.io/jippi/pixelfed" + +# The container runtime to use. +# +# See: https://github.com/jippi/pixelfed/blob/jippi-fork/docker/runtimes.md +DOCKER_RUNTIME="apache" + +# Pixelfed version (image tag) to pull from the registry. +# +# See: https://github.com/pixelfed/pixelfed/pkgs/container/pixelfed +DOCKER_TAG="${PIXELFED_RELEASE}-${DOCKER_RUNTIME}-${PHP_VERSION}" + +# Path (relative to the docker-compose.yml) or absolute (/some/other/path) where containers will store their data +DOCKER_DATA_ROOT="./docker-compose-state/data" + +# Path (relative to the docker-compose.yml) or absolute (/some/other/path) where containers will store their confguration +DOCKER_CONFIG_ROOT="./docker-compose-state/config" + +# Path (on host system) where the [db] container will store its data +# +# Path is relative (./some/other/path) to the docker-compose.yml or absolute (/some/other/path) +DOCKER_DB_DATA_PATH="${DOCKER_DATA_ROOT}/db" + +# Path (on host system) where the [redis] container will store its data +# +# Path is relative (./some/other/path) to the docker-compose.yml or absolute (/some/other/path) +DOCKER_REDIS_DATA_PATH="${DOCKER_DATA_ROOT}/redis" + +# Path (on host system) where the [app] + [worker] container will write +# its [storage] data (e.g uploads/images/profile pictures etc.). +# +# Path is relative (./some/other/path) to the docker-compose.yml or absolute (/some/other/path) +DOCKER_APP_STORAGE_PATH="${DOCKER_DATA_ROOT}/pixelfed/storage" + +# Path (on host system) where the [app] + [worker] container will write +# its [cache] data. +# +# Path is relative (./some/other/path) to the docker-compose.yml or absolute (/some/other/path) +DOCKER_APP_CACHE_PATH="${DOCKER_DATA_ROOT}/pixelfed/cache" + # Port that Redis will listen on *outside* the container (e.g. the host machine) -DOCKER_REDIS_PORT_EXTERNAL="${REDIS_PORT}" +DOCKER_REDIS_PORT_EXTERNAL="${REDIS_PORT:-6379}" # Port that the database will listen on *outside* the container (e.g. the host machine) # @@ -971,30 +1021,64 @@ DOCKER_PROXY_PROFILE="" # Set this to a non-empty value (e.g. "disabled") to disable the [proxy-acme] service DOCKER_PROXY_ACME_PROFILE="${DOCKER_PROXY_PROFILE}" -# Path (relative to the docker-compose.yml) or absolute (/some/other/path) where containers will store their data -DOCKER_DATA_ROOT="./docker-compose-state/data" - -# Path (relative to the docker-compose.yml) or absolute (/some/other/path) where containers will store their confguration -DOCKER_CONFIG_ROOT="./docker-compose-state/config" - -# Path (on host system) where the [db] container will store its data +# Automatically run "One-time setup tasks" commands. # -# Path is relative (./some/other/path) to the docker-compose.yml or absolute (/some/other/path) -DB_DATA_PATH="${DOCKER_DATA_ROOT}/db" - -# Path (on host system) where the [redis] container will store its data +# If you are migrating to this docker-compose setup or have manually run the "One time seutp" +# tasks (https://docs.pixelfed.org/running-pixelfed/installation/#setting-up-services) +# you can set this to "0" to prevent them from running. # -# Path is relative (./some/other/path) to the docker-compose.yml or absolute (/some/other/path) -REDIS_DATA_PATH="${DOCKER_DATA_ROOT}/redis" +# Otherwise, leave it at "1" to have them run *once*. +#DOCKER_RUN_ONE_TIME_SETUP_TASKS="1" -# Path (on host system) where the [app] + [worker] container will write -# its [storage] data (e.g uploads/images/profile pictures etc.). +# A space-seperated list of paths (inside the container) to *recursively* [chown] +# to the container user/group id (UID/GID) in case of permission issues. # -# Path is relative (./some/other/path) to the docker-compose.yml or absolute (/some/other/path) -APP_STORAGE_PATH="${DOCKER_DATA_ROOT}/pixelfed/storage" +# ! You should *not* leave this on permanently, at it can significantly slow down startup +# ! time for the container, and during normal operations there should never be permission +# ! issues. Please report a bug if you see behavior requiring this to be permanently on +# +# Example: "/var/www/storage /var/www/bootstrap/cache" +#DOCKER_ENSURE_OWNERSHIP_PATHS="" -# Path (on host system) where the [app] + [worker] container will write -# its [cache] data. +# Enable Docker Entrypoint debug mode (will call [set -x] in bash scripts) +# by setting this to "1". +#ENTRYPOINT_DEBUG="0" + +################################################################################ +# MySQL DB container configuration +################################################################################ # -# Path is relative (./some/other/path) to the docker-compose.yml or absolute (/some/other/path) -APP_CACHE_PATH="${DOCKER_DATA_ROOT}/pixelfed/cache" +# See "Environment Variables" at https://hub.docker.com/_/mysql +# +# ! DO NOT CHANGE unless you know what you are doing + +MYSQL_ROOT_PASSWORD="${DB_PASSWORD}" +MYSQL_USER="${DB_USERNAME}" +MYSQL_PASSWORD="${DB_PASSWORD}" +MYSQL_DATABASE="${DB_DATABASE}" + +################################################################################ +# MySQL (MariaDB) DB container configuration +################################################################################ +# +# See "Start a mariadb server instance with user, password and database" +# at https://hub.docker.com/_/mariadb +# +# ! DO NOT CHANGE unless you know what you are doing + +MARIADB_ROOT_PASSWORD="${DB_PASSWORD}" +MARIADB_USER="${DB_USERNAME}" +MARIADB_PASSWORD="${DB_PASSWORD}" +MARIADB_DATABASE="${DB_DATABASE}" + +################################################################################ +# PostgreSQL DB container configuration +################################################################################ +# +# See "Environment Variables" at https://hub.docker.com/_/postgres +# +# ! DO NOT CHANGE unless you know what you are doing + +POSTGRES_USER="${DB_USERNAME}" +POSTGRES_PASSWORD="${DB_PASSWORD}" +POSTGRES_DB="${DB_DATABASE}" diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 000000000..cf98a0902 --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,4 @@ +{ + "MD013": false, + "MD014": false +} diff --git a/Dockerfile b/Dockerfile index 6151d8758..3d12aba26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ ARG RUNTIME_UID=33 # often called 'www-data' ARG RUNTIME_GID=33 # often called 'www-data' # APT extra packages -ARG APT_PACKAGES_EXTRA="" +ARG APT_PACKAGES_EXTRA= # Extensions installed via [pecl install] ARG PHP_PECL_EXTENSIONS= diff --git a/docker-compose.migrate.yml b/docker-compose.migrate.yml new file mode 100644 index 000000000..b47abeb48 --- /dev/null +++ b/docker-compose.migrate.yml @@ -0,0 +1,42 @@ +--- +version: "3" + +services: + migrate: + image: "secoresearch/rsync" + entrypoint: "" + working_dir: /migrate + command: 'bash -c "exit 1"' + restart: never + volumes: + ################################ + # Storage volume + ################################ + # OLD + - "app-storage:/migrate/app-storage/old" + # NEW + - "${DOCKER_APP_STORAGE_PATH}:/migrate/app-storage/new" + + ################################ + # MySQL/DB volume + ################################ + # OLD + - "db-data:/migrate/db-data/old" + # NEW + - "${DOCKER_DB_DATA_PATH}:/migrate/db-data/new" + + ################################ + # Redis volume + ################################ + # OLD + - "redis-data:/migrate/redis-data/old" + # NEW + - "${DOCKER_REDIS_DATA_PATH}:/migrate/redis-data/new" + +# Volumes from the old [docker-compose.yml] file +# https://github.com/pixelfed/pixelfed/blob/b1ff44ca2f75c088a11576fb03b5bad2fbed4d5c/docker-compose.yml#L72-L76 +volumes: + db-data: + redis-data: + app-storage: + app-bootstrap: diff --git a/docker-compose.yml b/docker-compose.yml index 614d3954a..76aebf609 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,15 +65,22 @@ services: container_name: "${DOCKER_CONTAINER_NAME_PREFIX}-web" restart: unless-stopped build: - target: apache-runtime + target: ${DOCKER_RUNTIME}-runtime + args: + PHP_VERSION: "${PHP_VERSION:-8.1}" + PHP_BASE_TYPE: "${PHP_BASE_TYPE:-apache}" + PHP_DEBIAN_RELEASE: "${PHP_DEBIAN_RELEASE:-bullseye}" + APT_PACKAGES_EXTRA: "${APT_PACKAGES_EXTRA:-}" + PHP_PECL_EXTENSIONS_EXTRA: "${PHP_PECL_EXTENSIONS_EXTRA:-}" + PHP_EXTENSIONS_EXTRA: "${PHP_EXTENSIONS_EXTRA:-}" volumes: - "./.env:/var/www/.env" - - "${APP_CACHE_PATH}:/var/www/bootstrap/cache" - - "${APP_STORAGE_PATH}:/var/www/storage" + - "${DOCKER_APP_CACHE_PATH}:/var/www/bootstrap/cache" + - "${DOCKER_APP_STORAGE_PATH}:/var/www/storage" environment: LETSENCRYPT_HOST: "${LETSENCRYPT_HOST}" LETSENCRYPT_EMAIL: "${LETSENCRYPT_EMAIL}" - LETSENCRYPT_TEST: "${LETSENCRYPT_TEST}" + LETSENCRYPT_TEST: "${LETSENCRYPT_TEST:-}" VIRTUAL_HOST: "${APP_DOMAIN}" VIRTUAL_PORT: "80" labels: @@ -91,15 +98,19 @@ services: container_name: "${DOCKER_CONTAINER_NAME_PREFIX}-worker" command: gosu www-data php artisan horizon restart: unless-stopped - deploy: - mode: replicated - replicas: 1 build: - target: apache-runtime + target: ${DOCKER_RUNTIME}-runtime + args: + PHP_VERSION: "${PHP_VERSION:-8.1}" + PHP_BASE_TYPE: "${PHP_BASE_TYPE:-apache}" + PHP_DEBIAN_RELEASE: "${PHP_DEBIAN_RELEASE:-bullseye}" + APT_PACKAGES_EXTRA: "${APT_PACKAGES_EXTRA:-}" + PHP_PECL_EXTENSIONS_EXTRA: "${PHP_PECL_EXTENSIONS_EXTRA:-}" + PHP_EXTENSIONS_EXTRA: "${PHP_EXTENSIONS_EXTRA:-}" volumes: - "./.env:/var/www/.env" - - "${APP_CACHE_PATH}:/var/www/bootstrap/cache" - - "${APP_STORAGE_PATH}:/var/www/storage" + - "${DOCKER_APP_CACHE_PATH}:/var/www/bootstrap/cache" + - "${DOCKER_APP_STORAGE_PATH}:/var/www/storage" depends_on: - db - redis @@ -112,7 +123,7 @@ services: env_file: - ".env" volumes: - - "${DB_DATA_PATH}:/var/lib/mysql" + - "${DOCKER_DB_DATA_PATH}:/var/lib/mysql" ports: - "${DOCKER_DB_PORT_EXTERNAL}:3306" @@ -124,7 +135,7 @@ services: - ".env" volumes: - "${DOCKER_CONFIG_ROOT}/redis:/etc/redis" - - "${REDIS_DATA_PATH}:/data" + - "${DOCKER_REDIS_DATA_PATH}:/data" ports: - "${DOCKER_REDIS_PORT_EXTERNAL}:6399" healthcheck: diff --git a/docker/README.md b/docker/README.md index ea1870fd3..bf73662a8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,6 +1,7 @@ # Pixelfed + Docker + Docker Compose * [Setting up a new Pixelfed server with Docker Compose](new-server.md) +* [Migrating to the new `docker-compose.yml` setup](migration.md) * [Frequently Asked Question / FAQ](faq.md) * [How do I use my own Proxy server?](faq.md#how-do-i-use-my-own-proxy-server) * [How do I use my own SSL certificate?](faq.md#how-do-i-use-my-own-ssl-certificate) diff --git a/docker/customizing.md b/docker/customizing.md index 8dfc534dc..dcea45d3d 100644 --- a/docker/customizing.md +++ b/docker/customizing.md @@ -73,12 +73,12 @@ Please see the ## Fixing ownership on startup -You can set the environment variable `ENTRYPOINT_ENSURE_OWNERSHIP_PATHS` to a list of paths that should have their `$USER` and `$GROUP` ownership changed to the configured runtime user and group during container bootstrapping. +You can set the environment variable `DOCKER_ENSURE_OWNERSHIP_PATHS` to a list of paths that should have their `$USER` and `$GROUP` ownership changed to the configured runtime user and group during container bootstrapping. The variable is a space-delimited list shown below and accepts both relative and absolute paths: -* `ENTRYPOINT_ENSURE_OWNERSHIP_PATHS="./storage ./bootstrap"` -* `ENTRYPOINT_ENSURE_OWNERSHIP_PATHS="/some/other/folder"` +* `DOCKER_ENSURE_OWNERSHIP_PATHS="./storage ./bootstrap"` +* `DOCKER_ENSURE_OWNERSHIP_PATHS="/some/other/folder"` ## Build settings (arguments) diff --git a/docker/faq.md b/docker/faq.md index 3988be578..cc2495d15 100644 --- a/docker/faq.md +++ b/docker/faq.md @@ -10,6 +10,7 @@ No problem! All you have to do is: 1. Point your proxy upstream to the exposed `web` port (**Default**: `8080`). * The port is controlled by the `DOCKER_WEB_PORT_EXTERNAL_HTTP` key in `.env`. * The setting is near the bottom of the file. +1. Run `docker compose up -d --remove-orphans` to apply the configuration ## How do I use my own SSL certificate? @@ -26,6 +27,7 @@ No problem! All you have to do is: 1. `${APP_DOMAIN}.fullchain.pem` 1. `${APP_DOMAIN}.key.pem` * See the [`nginx-proxy` configuration file for name patterns](https://github.com/nginx-proxy/nginx-proxy/blob/main/nginx.tmpl#L659-L670) +1. Run `docker compose up -d --remove-orphans` to apply the configuration ## How do I change the container name prefix? diff --git a/docker/migration.md b/docker/migration.md new file mode 100644 index 000000000..76fb0904c --- /dev/null +++ b/docker/migration.md @@ -0,0 +1,326 @@ +# Migrating to the new Pixelfed Docker setup + +There is [*a lot* of changes](https://github.com/pixelfed/pixelfed/pull/4844) in how Pixelfed Docker/Docker Compose images work - it's a complete rewrite - with a bunch of breaking changes. + +## No more anonymous Docker Compose volumes + +The old `docker-compose.yml` configuration file [declared four anonymous volumes](https://github.com/pixelfed/pixelfed/blob/b1ff44ca2f75c088a11576fb03b5bad2fbed4d5c/docker-compose.yml#L72-L76) for storing Pixelfed related data within. + +These are no longer used, instead favoring a [Docker bind volume](https://docs.docker.com/storage/bind-mounts/) approach where content are stored directly on the server disk, outside +of a Docker volume. + +The consequence of this change is that *all* data stored in the - now unsupported - Docker volumes will no longer be accessible by Pixelfed. + +* The `db-data` volume *definitely* contain important data - it's your database after all! +* The `app-storage` volume *definitely* contain important data - it's files uploaded to - or seen by - your server! +* The `redis-data` volume *might* contain important data (depending on your configuration) +* The `app-bootstrap` volume does not contain any important data - all of it will be generated automatically in the new setup on startup. We will *not* be migrating this! + +### Migrating off anonymous Docker Compose volumes + +#### Caveats and warnings + +**NOTE**: This is a best-effort guide to help migrate off the old system, the operation is potentially rather complicated (and risky), so please do be careful! + +***PLEASE MAKE SURE TO BACKUP YOUR SERVER AND DATA BEFORE ATTEMPTING A MIGRATION*** + +We provide a "migration container" for your convenience that can access both the new and old volumes, allowing you to copy the data into the setup. + +**It's important to note that this is a *copy* operation - so disk usage will (temporarily) double while you migrate** + +**YOUR INSTANCE WILL BE *DOWN* WHILE DOING THE MIGRATION, PLEASE PLAN ACCORDINGLY, DEPENDING ON DATA SIZE IT COULD TAKE ANYWHERE FROM 5 *MINUTES* TO 5 *HOURS*** + +#### 0. Backup, rollout, and rollback plan + +1. Make sure to backup your server (ideally *after* step 1 below has completed, but *before* is better than not at all!) +1. Capture the current Git version / Pixelfed release you are on (e.g. `git --no-pager log -1` outputs the commit reference as the 2nd word in first line) +1. Backup your `.env` file (we will do this in step 3 as well) +1. Backup your `docker-compose.yml` file (`cp docker-compose.yml docker-compose.yml.old`) +1. Read through the *entire* document before starting + +#### 1. Migrate your ".env" file + +The new `.env` file for Docker is a bit different from the old one (many new settings!) so the easiest is to grab the new `.env.docker` file and modify it from scratch again. + +```shell +$ cp .env .env.old +$ wget -O .env.new https://raw.githubusercontent.com/pixelfed/pixelfed/dev/.env.docker +``` + +Then open your old `.env.old` configuration file, and for each of the key/value pairs within it, find and update the key in the new `.env.new` configuration file. + +Don't worry though, the file might *look* different (and significantly larger) but it behaves *exactly* the way the old file did, it just has way more options! + +If a key is missing in `.env.new`, don't worry, you can just add those key/value pairs back to the new file - ideally in the `Other configuration` section near the end of the file - but anywhere *should* be fine. + +This is a great time to review your settings and familiarize you with all the new settings. + +In *particular* the following sections and the [Frequently Asked Question / FAQ](faq.md) + +* `PHP configuration` section (near the end of the file) where + * The `PHP_VERSION` settings controls your PHP version + * The `PHP_MEMORY_LIMIT` settings controls your PHP memory limit +* `Docker Specific configuration` section (near the end of the file) where + * The `DOCKER_DATA_ROOT` setting dictate where the new migrated data will live. + * The `DOCKER_RUN_ONE_TIME_SETUP_TASKS` controls if the `One time setup tasks` should run or not. We do *not* want this, since your Pixelfed instance already is set up! + +#### 2. Stop all running containers + +Stop *all* running containers (web, worker, redis, db) + +```shell +$ docker compose down +``` + +#### 3. Pull down the new source code + +Update your project to the latest release of Pixelfed by running + +```shell +$ git pull origin $release +``` + +Where `$release` is either `dev`, `staging` or a [tagged release](https://github.com/pixelfed/pixelfed/releases) such as `v0.12.0`. + +#### 4. Run the migration container + +You can access the Docker container with both old and new volumes by running the following command + +```shell +$ docker compose -f docker-compose.migrate.yml run migrate bash +``` + +This will put you in the `/migrate` directory within the container, containing 8 directories like shown here + +```plain +|-- app-storage +| |-- new +| `-- old +|-- db-data +| |-- new +| `-- old +`-- redis-data + |-- new + `-- old +``` + +#### 5. Check the folders + +##### Old folders + +The following commands should all return *SOME* files and data - if they do not - then there might be an issue with the anonymous volume binding. + +```shell +$ ls app-storage/old +$ ls db-data/old +$ ls redis-data/old +``` + +##### New folders + +The following commands should all return *NO* files and data - if they contain data - you need to either delete it (backup first!) or skip that migration step. + +If you haven't run `docker compose up` since you updated your project in step (2) - they should be empty and good to go. + +```shell +$ ls app-storage/new +$ ls db-data/new +$ ls redis-data/new +``` + +#### 6. Copy the data + +Now we will copy the data from the old volumes, to the new ones. + +The migration container has [`rsync`](https://www.digitalocean.com/community/tutorials/how-to-use-rsync-to-sync-local-and-remote-directories) installed - which is perfect for that kind of work! + +**NOTE** It's important that the "source" (first path in the `rsync` command) has a trailing `/` - otherwise the directory layout will turn out wrong! + +**NOTE** Depending on your server, these commands might take some time to finish, each command should provide a progress bar with rough time estimation. + +**NOTE** `rsync` should preserve ownership, permissions, and symlinks correctly for you as well for all the files copied. + +Lets copy the data by running the following commands: + +```shell +$ rsync -avP app-storage/old/ app-storage/new +$ rsync -avP db-data/old/ db-data/new +$ rsync -avP redis-data/old/ redis-data/new +``` + +#### 7. Sanity checking + +Lets make sure everything copied over successfully! + +Each *new* directory should contain *something* like (but not always exactly) the following - **NO** directory should have a single folder called `old`, if they do, the `rsync` commands above didn't work correctly - and you need to move the content of the `old` folder into the "root" of the `new` folder like shown a bit in the following sections. + +The **redis-data/new** directory might also contain a `server.pid` + +```shell +$ ls redis-data/new +appendonlydir +``` + +The **app-storage/new** directory should look *something* like this + +```shell +$ ls app-storage/new +app debugbar docker framework logs oauth-private.key oauth-public.key purify +``` + +The **db-data/new** directory should look *something* like this. There might be a lot of files, or very few files, but there *must* be a `mysql`, `performance_schema`, and `${DB_DATABASE}` (e.g. `pixelfed_prod` directory) + +```shell +$ ls db-data/new +aria_log_control ddl_recovery-backup.log ib_buffer_pool ib_logfile0 ibdata1 mariadb_upgrade_info multi-master.info mysql performance_schema pixelfed_prod sys undo001 undo002 undo003 +``` + +If everything looks good, type `exit` to leave exit the migration container + +#### 6. Starting up the your Pixelfed server again + +With all an updated Pixelfed (step 2), updated `.env` file (step 3), migrated data (step 4, 5, 6 and 7) we're ready to start things back up again. + +But before we start your Pixelfed server back up again, lets put the new `.env` file we made in step 1 in its right place. + +```shell +$ cp .env.new .env +``` + +##### The Database + +First thing we want to try is to start up the database by running the following command and checking the logs + +```shell +$ docker compose up -d db +$ docker compose logs --tail 250 --follow db +``` + +if there are no errors and the server isn't crashing, great! If you have an easy way of connecting to the Database via a GUI or CLI client, do that as well and verify the database and tables are all there. + +##### Redis + +Next thing we want to try is to start up the Redis server by running the following command and checking the logs + +```shell +$ docker compose up -d redis +$ docker compose logs --tail 250 --follow redis +``` + +if there are no errors and the server isn't crashing, great! + +##### Worker + +Next thing we want to try is to start up the Worker server by running the following command and checking the logs + +```shell +$ docker compose up -d worker +$ docker compose logs --tail 250 --follow worker +``` + +The container should output a *lot* of logs from the [docker-entrypoint system](customizing.md#running-commands-on-container-start), but *eventually* you should see these messages + +* `Configuration complete; ready for start up` +* `Horizon started successfully.` + +If you see one or both of those messages, great, the worker seems to be running. + +If the worker is crash looping, inspect the logs and try to resolve the issues. + +You can consider the following additional steps: + +* Enabling `ENTRYPOINT_DEBUG` which will show even more log output to help understand whats going on +* Enabling `DOCKER_ENSURE_OWNERSHIP_PATHS` against the path(s) that might have permission issues +* Fixing permission issues directly on the host since your data should all be in the `${DOCKER_DATA_ROOT}` folder (`./docker-compose-state/data` by default) + +##### Web + +The final service, `web`, which will bring your site back online! What a journey it has been. + +Lets get to it, run these commands to start the `web` service and inspect the logs. + +```shell +$ docker compose up -d web +$ docker compose logs --tail 250 --follow web +``` + +The output should be pretty much identical to that of the `worker`, so please see that section for debugging tips if the container is crash looping. + +If the `web` service came online without issues, start the rest of the (optional) services, such as the `proxy`, if enabled, by running + +```shell +$ docker compose up -d +$ docker compose logs --tail 250 --follow +``` + +If you changed anything in the `.env` file while debugging, some containers might restart now, thats perfectly fine. + +#### 7. Verify + +With all services online, it's time to go to your browser and check everything is working + +1. Upload and post a picture +1. Comment on a post +1. Like a post +1. Check Horizon (`https://${APP_DOMAIN}/horizon`) for any errors +1. Check the Docker compose logs via `docker compose logs --follow` + +If everything looks fine, yay, you made it to the end! Lets do some cleanup + +#### 8. Final steps + cleanup + +With everything working, please take a new snapshot/backup of your server *before* we do any cleanup. A post-migration snapshot is incredibly useful, since it contains both the old and new configuration + data, making any recovery much easier in a rollback scenario later. + +Now, with all the data in the new folders, you can delete the old Docker Container volumes (if you want, completely optional) + +List all volumes, and give them a look: + +```shell +$ docker volume ls +``` + +The volumes we want to delete *ends* with the volume name (`db-data`, `app-storage`, `redis-data`, and `app-bootstrap`.) but has some prefix in front of them. + +Once you have found the volumes in in the list, delete each of them by running: + +```shell +$ docker volume rm $volume_name_in_column_two_of_the_output +``` + +You can also delete the `docker-compose.yml.old` and `.env.old` file since they are no longer needed + +```shell +$ rm docker-compose.yml.old +$ rm .env.old +``` + +### Rollback + +Oh no, something went wrong? No worries, we you got backups and a quick way back! + +#### Move `docker-compose.yml` back + +```shell +$ cp docker-compose.yml docker-compose.yml.new +$ cp docker-compose.yml.old docker-compose.yml +``` + +#### Move `.env` file back + +```shell +$ cp env.old .env +``` + +#### Go back to old source code version + +```shell +$ git checkout $commit_id_from_step_0 +``` + +#### Start things back up + +```shell +docker compose up -d +``` + +#### Verify it worked diff --git a/docker/shared/root/docker/entrypoint.d/01-permissions.sh b/docker/shared/root/docker/entrypoint.d/01-permissions.sh index 58b2f0bb8..287d708aa 100755 --- a/docker/shared/root/docker/entrypoint.d/01-permissions.sh +++ b/docker/shared/root/docker/entrypoint.d/01-permissions.sh @@ -13,13 +13,13 @@ run-as-current-user chown --verbose "${RUNTIME_UID}:${RUNTIME_GID}" "./bootstrap run-as-current-user chown --verbose "${RUNTIME_UID}:${RUNTIME_GID}" "./storage" # Optionally fix ownership of configured paths -: "${ENTRYPOINT_ENSURE_OWNERSHIP_PATHS:=""}" +: "${DOCKER_ENSURE_OWNERSHIP_PATHS:=""}" declare -a ensure_ownership_paths=() -IFS=' ' read -ar ensure_ownership_paths <<<"${ENTRYPOINT_ENSURE_OWNERSHIP_PATHS}" +IFS=' ' read -ar ensure_ownership_paths <<<"${DOCKER_ENSURE_OWNERSHIP_PATHS}" if [[ ${#ensure_ownership_paths[@]} == 0 ]]; then - log-info "No paths has been configured for ownership fixes via [\$ENTRYPOINT_ENSURE_OWNERSHIP_PATHS]." + log-info "No paths has been configured for ownership fixes via [\$DOCKER_ENSURE_OWNERSHIP_PATHS]." exit 0 fi