diff --git a/.dockerignore b/.dockerignore index 069a9248..2e6676cb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,3 +19,10 @@ node_modules .vscode .github .git +db_username.txt +db_password.txt +redis_password.txt +typesense_api_key.txt +.phpunit.result.cache +.env +docker-compose.yml diff --git a/.github/workflows/container-image-release.yml b/.github/workflows/container-image-release.yml index 271a1bf6..41c20a53 100644 --- a/.github/workflows/container-image-release.yml +++ b/.github/workflows/container-image-release.yml @@ -1,24 +1,34 @@ name: Container Image Release -concurrency: production +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true +permissions: + packages: write + contents: read on: release: types: [published] + workflow_dispatch: + inputs: + image_tag: + description: 'Image tag' + required: true + default: 'v4.0.0-nightly-' + +env: + REGISTRY_IMAGE: ghcr.io/jikan-me/jikan-rest jobs: - release-app-image: + build-app-image: runs-on: ubuntu-latest - name: Release App container image + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + name: Build App container image steps: - - name: Check if base container image exists - id: baseImageExists - run: | - GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64) - curl --fail -H "Authorization: Bearer ${GHCR_TOKEN}" https://ghcr.io/v2/jikan-me/jikan-rest-php/tags/list | grep -q latest - - - name: Base image existance check failed - if: ${{ always() && steps.baseImageExists.outcome == 'failure' }} - run: echo "Base image doesn't exist yet. Please run the base image creation workflow first." - - name: Checkout uses: actions/checkout@v4 with: @@ -29,18 +39,78 @@ jobs: - name: Set up docker buildx uses: docker/setup-buildx-action@v3 - with: - platforms: linux/amd64,linux/arm64 - name: Read metadata id: meta uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} + labels: | + org.opencontainers.image.title=Jikan REST API + org.opencontainers.image.description=REST API for Jikan + org.opencontainers.image.url=https://jikan.moe + org.opencontainers.image.source=https://github.com/jikan-me/jikan-rest + org.opencontainers.image.documentation=https://github.com/jikan-me/jikan-rest/blob/master/container_usage.md + org.opencontainers.image.revision=${{ github.sha }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v4 + with: + context: . + platforms: ${{ matrix.platform }} + # let's use github action cache storage + cache-from: type=gha + cache-to: type=gha,mode=max + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v3 + with: + name: digests + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + publish-app-image: + runs-on: ubuntu-latest + name: Publish app container image + needs: + - build-app-image + steps: + - name: Download digests + uses: actions/download-artifact@v3 + with: + name: digests + path: /tmp/digests + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Read metadata + id: meta + uses: docker/metadata-action@v4 with: images: | - ghcr.io/jikan-me/jikan-rest + ${{ env.REGISTRY_IMAGE }} jikanme/jikan-rest tags: | - type=raw,value=${{ github.ref_name }} + type=raw,value=${{ inputs.image_tag }} + type=raw,value=${{ github.ref_type == "tag" && 'latest' || 'latest-nightly' }} type=sha - name: Login to GitHub Container Registry @@ -56,16 +126,10 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v5 - with: - push: true - context: . - # let's use github action cache storage - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - GITHUB_PERSONAL_TOKEN=${{ secrets.GITHUB_TOKEN }} - BASE_IMAGE_VERSION=latest - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/.gitignore b/.gitignore index 4e303537..dfe0c39f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,9 @@ composer.phar /coverage .DS_Store *.cache +db_username.txt +db_admin_username.txt +db_password.txt +db_admin_password.txt +redis_password.txt +typesense_api_key.txt diff --git a/.rr.yaml b/.rr.yaml index b5d4e792..0f9a29ef 100644 --- a/.rr.yaml +++ b/.rr.yaml @@ -61,7 +61,7 @@ logs: # we want to use docker's log drivers, so push logs to stdout output: stdout - # we to use docker's log drivers, so push error logs to stdout + # we want to use docker's log drivers, so push error logs to stdout # this way it is possible for example to pipe logs to journald or to AWS Cloudwatch err_output: stdout diff --git a/Dockerfile b/Dockerfile index d00a7c1c..8e07b4cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,26 @@ -FROM spiralscout/roadrunner:2.12.2 as roadrunner -FROM composer:2.5.1 as composer -FROM mlocati/php-extension-installer:1.5.52 as php-ext-installer +FROM docker.io/spiralscout/roadrunner:2.12.2 as roadrunner +FROM docker.io/composer:2.5.1 as composer +FROM docker.io/mlocati/php-extension-installer:2.1.58 as php-ext-installer FROM php:8.1.16-bullseye -ARG GITHUB_PERSONAL_TOKEN COPY --from=composer /usr/bin/composer /usr/bin/composer COPY --from=php-ext-installer /usr/bin/install-php-extensions /usr/local/bin/ ENV COMPOSER_HOME="/tmp/composer" RUN set -x \ - && install-php-extensions gd exif intl bz2 gettext mongodb-stable redis opcache sockets pcntl \ + && install-php-extensions intl mbstring mongodb-stable redis opcache sockets pcntl \ # install xdebug (for testing with code coverage), but do not enable it && IPE_DONT_ENABLE=1 install-php-extensions xdebug-3.2.0 # install roadrunner COPY --from=roadrunner /usr/bin/rr /usr/bin/rr LABEL org.opencontainers.image.source=https://github.com/jikan-me/jikan-rest -# used only for supercronic atm. Supported values are: amd64, arm64 -ARG TARGET_ARCH="amd64" RUN set -ex \ && apt-get update && apt-get install -y --no-install-recommends \ openssl \ git \ - dos2unix \ + wget \ unzip \ - wget \ # install supercronic (for laravel task scheduling), project page: - && wget -q "https://github.com/aptible/supercronic/releases/download/v0.1.12/supercronic-linux-${TARGET_ARCH}" \ + && wget -q "https://github.com/aptible/supercronic/releases/download/v0.1.12/supercronic-linux-$(dpkg --print-architecture)" \ -O /usr/bin/supercronic \ && chmod +x /usr/bin/supercronic \ && mkdir /etc/supercronic \ @@ -51,11 +47,6 @@ WORKDIR /app # copy composer (json|lock) files for dependencies layer caching COPY --chown=jikanapi:jikanapi ./composer.* /app/ -# check if GITHUB_PERSONAL_TOKEN is set and configure it for composer -# it is recommended to set this for the build, otherwise the build might fail because of github's rate limits -RUN if [ -z "$GITHUB_PERSONAL_TOKEN" ]; then echo "** GITHUB_PERSONAL_TOKEN is not set. This build may fail due to github rate limits."; \ - else composer config github-oauth.github.com "$GITHUB_PERSONAL_TOKEN"; fi - # install composer dependencies (autoloader MUST be generated later!) RUN composer install -n --no-dev --no-cache --no-ansi --no-autoloader --no-scripts --prefer-dist diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 59f915af..22c9b19c 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,20 +2,8 @@ namespace App\Console; -use App\Console\Commands\ClearQueuedJobs; use App\Console\Commands\CacheRemove; -use App\Console\Commands\Indexer\AnimeIndexer; -use App\Console\Commands\Indexer\AnimeSweepIndexer; -use App\Console\Commands\Indexer\AnimeScheduleIndexer; -use App\Console\Commands\Indexer\CommonIndexer; -use App\Console\Commands\Indexer\CurrentSeasonIndexer; -use App\Console\Commands\Indexer\GenreIndexer; -use App\Console\Commands\Indexer\MangaIndexer; -use App\Console\Commands\Indexer\MangaSweepIndexer; -use App\Console\Commands\Indexer\ProducersIndexer; -use App\Console\Commands\ManageMicrocaching; -use App\Console\Commands\ModifyCacheDriver; -use App\Console\Commands\ModifyCacheMethod; +use App\Console\Commands\Indexer; use Illuminate\Console\Scheduling\Schedule; use Laravel\Lumen\Console\Kernel as ConsoleKernel; @@ -28,15 +16,15 @@ class Kernel extends ConsoleKernel */ protected $commands = [ CacheRemove::class, - CommonIndexer::class, - AnimeScheduleIndexer::class, - CurrentSeasonIndexer::class, - AnimeIndexer::class, - MangaIndexer::class, - GenreIndexer::class, - ProducersIndexer::class, - AnimeSweepIndexer::class, - MangaSweepIndexer::class, + Indexer\CommonIndexer::class, + Indexer\AnimeScheduleIndexer::class, + Indexer\CurrentSeasonIndexer::class, + Indexer\AnimeIndexer::class, + Indexer\MangaIndexer::class, + Indexer\GenreIndexer::class, + Indexer\ProducersIndexer::class, + Indexer\AnimeSweepIndexer::class, + Indexer\MangaSweepIndexer::class ]; /** diff --git a/container-setup.sh b/container-setup.sh new file mode 100755 index 00000000..49800107 --- /dev/null +++ b/container-setup.sh @@ -0,0 +1,175 @@ +#!/bin/bash + +_JIKAN_API_VERSION=v4.0.0 +SUBSTITUTE_VERSION=$_JIKAN_API_VERSION +if [ -x "$(command -v git)" ]; then + # check if we have checked out a tag or not + git symbolic-ref HEAD &> /dev/null + if [ $? -ne 0 ]; then + # if a tag is checked out then use the tag name as the version + SUBSTITUTE_VERSION=$(git describe --tags) + else + # this is used when building locally + SUBSTITUTE_VERSION=$(git describe --tags | sed -e "s/-[a-z0-9]\{8\}/-$(git rev-parse --short HEAD)/g") + fi +fi +# set JIKAN_API_VERSION env var to "latest" or a tag which exists in the container registry to use the remote image +# otherwise docker-compose will look for a locally builded image +export _JIKAN_API_VERSION=${JIKAN_API_VERSION:-$SUBSTITUTE_VERSION} + +DOCKER_COMPOSE_PROJECT_NAME=jikan-api +DOCKER_CMD="docker" +DOCKER_COMPOSE_CMD="docker-compose" + +display_help() { + echo "============================================================" + echo "Jikan API Container Setup CLI" + echo "============================================================" + echo "Syntax: ./container-setup.sh [command]" + echo "Jikan API Version: $_JIKAN_API_VERSION" + echo "---commands---" + echo "help Print CLI help" + echo "build-image Build Image Locally" + echo "start Start Jikan API (mongodb, typesense, redis, jikan-api workers)" + echo "stop Stop Jikan API" + echo "validate-prereqs Validate pre-reqs installed (docker, docker-compose)" + echo "execute-indexers Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days)" + echo "" +} + +validate_prereqs() { + docker_exists=$(command -v docker) + docker_compose_exists=$(command -v docker-compose) + podman_exists=$(command -v podman) + podman_compose_exists=$(command -v podman-compose) + + if [ -x "$docker_exists" ] && [ -x "$podman_exists" ]; then + echo -e "'docker' is not installed. \xE2\x9D\x8C" + exit 1 + else + echo -e "Docker is Installed. \xE2\x9C\x94" + fi + + if [ -x "$docker_exists" ]; then + DOCKER_CMD="docker" + docker -v >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo -e "'docker' is not executable without sudo. \xE2\x9D\x8C" + exit 1 + fi + elif [ -n "$podman_exists" ]; then + DOCKER_CMD="podman" + fi + + if [ -x "$docker_compose_exists" ] && [ -x "$docker_compose_exists" ]; then + echo -e "'docker-compose' is not installed. \xE2\x9D\x8C" + exit 1 + else + echo -e "Docker compose is Installed. \xE2\x9C\x94" + fi + + if [ -x "$docker_compose_exists" ]; then + DOCKER_COMPOSE_CMD="docker-compose" + elif [ -x "$podman_compose_exists" ]; then + DOCKER_COMPOSE_CMD="podman-compose" + else + echo "Error" + exit 1 + fi +} + +build_image() { + validate_prereqs + $DOCKER_CMD inspect jikanme/jikan-rest:"$_JIKAN_API_VERSION" &> /dev/null && $DOCKER_CMD rmi jikanme/jikan-rest:"$_JIKAN_API_VERSION" + $DOCKER_CMD build --rm --compress -t jikanme/jikan-rest:"$_JIKAN_API_VERSION" . + $DOCKER_CMD tag jikanme/jikan-rest:"$_JIKAN_API_VERSION" jikanme/jikan-rest:latest +} + +ensure_secrets() { + declare -a secrets=("db_password" "db_admin_password" "redis_password" "typesense_api_key") + + if [ ! -f "db_username.txt" ]; then + echo "db_username.txt not found, please provide a db_username [default is jikan]:" + read -r db_username + if [ -z "$db_username" ]; then + db_username="jikan" + fi + echo -n "$db_username" > "db_username.txt" + else + echo -e "db_username.txt found, using it's value. \xE2\x9C\x94" + fi + + if [ ! -f "db_admin_username.txt" ]; then + echo "db_admin_username.txt not found, please provide a db_admin_username [default is jikan_admin]:" + read -r db_admin_username + if [ -z "$db_admin_username" ]; then + db_admin_username="jikan_admin" + fi + echo -n "$db_admin_username" > "db_admin_username.txt" + else + echo -e "db_admin_username.txt found, using it's value. \xE2\x9C\x94" + fi + + for secret_name in "${secrets[@]}" + do + if [ ! -f "$secret_name.txt" ]; then + if [ "$secret_name" == "db_username" ]; then + generated_secret="jikan" + else + generated_secret=$(LC_ALL=c tr -dc 'A-Za-z0-9!'\''()*+,-;<=>_' "$secret_name.txt" + else + echo -e "$secret_name.txt found, using it's value. \xE2\x9C\x94" + fi + done +} + +start() { + # todo: create a marker file for initial startup, and on initial startup ask the user whether they want a local image or the remote one + validate_prereqs + ensure_secrets + exec $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" up -d +} + +case "$1" in + "help") + display_help + ;; + "validate-prereqs") + validate_prereqs + ;; + "build-image") + build_image + ;; + "start") + start + ;; + "stop") + validate_prereqs + $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" down + ;; + "execute-indexers") + echo "Indexing anime..." + $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:anime + echo "Indexing manga..." + $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:manga + echo "Indexing characters and people..." + $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:common + echo "Indexing genres..." + $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:genres + echo "Indexing producers..." + $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:producers + echo "Indexing done!" + ;; + *) + echo "No command specified, displaying help" + display_help + ;; +esac diff --git a/container_usage.md b/container_usage.md index ffc64fd7..bce6bd74 100644 --- a/container_usage.md +++ b/container_usage.md @@ -1,5 +1,79 @@ # 🐳 Running Jikan API in a container +The most easiest way to get started is to use our container setup cli script after checking out the repo with git (linux only): + +```bash +./container-setup.sh start +``` + +This will: + +- Prompt you for the required passwords and usernames +- Sets up a production ready setup with `redis`, `typesense` and `mongodb` (almost same as the public api at `api.jikan.moe`) +- Sets mongodb to use max 1gb of memory +- Configures jikan-api to add CORS headers to responses. + +> **Note**: The script supports both `docker` and `podman`. In case of `podman` please bare in mind that sometimes the container name resolution doesn't work on the container network. +> In those cases you might have to install `aardvark-dns` package. On `Arch Linux` podman uses `netavark` network by default (in 2023) so you will need to install the before mentioned package. + +The script has the following prerequisites and will notify you if these are not present: + +- git +- `docker` or `podman` +- `docker-compose` or `podman-compose` + +### Available commands in the cli script + +``` +============================================================ +Jikan API Container Setup CLI +============================================================ +Syntax: ./container-setup.sh [command] +---commands--- +help Print CLI help +build-image Build Image Locally +start Start Jikan API (mongodb, typesense, redis, jikan-api workers) +stop Stop Jikan API +validate-prereqs Validate pre-reqs installed (docker, docker-compose) +execute-indexers Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days) +``` + +### Running the indexer with the script + +When you first startup the app you will have an empty database. To fill it up you can execute the following command: + +```bash +./container-setup.sh execute-indexers +``` + +Please note that this command can take 4-5 days to run. You can run it in the background with the `&` marker: + +```bash +./container-setup.sh execute-indexers & +``` + +If interrupted then you will have to manually resume the indexing, otherwise the above command will just start again from the beginning. + +### Updating to a newer version + +You need to stop the app first: + +```bash +./container-setup.sh stop +``` + +Then remove the jikan-api image from your local storage and pull the new one. Set the `JIKAN_API_VERSION` environment variable to the latest image tag. This can be either `latest` or the version `v4.0.0-11`. + +```bash +JIKAN_API_VERSION=latest ./container-setup.sh start +``` + +## More customised setups + +Some of you might only want to run the `jikan-rest` app with only mongodb, without the more sophisticated search functionality. In those cases we don't have a `docker-compose` config for you. You need to start the `jikan-rest` container with atleast a `mongodb` instance. +The `jikan-rest` container will require a `.env` file mounted where you configure the credentials for `mongodb`. + + ```bash docker run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env jikanme/jikan-rest:latest ``` @@ -14,8 +88,7 @@ docker run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env jikanme/jikan-r > address of these services. > **Tip**: If you run the container on a non-default network, you can use the container names in the configuration to -> specify the address of services like MongoDB and TypeSense. However, this is not a concern if you -> use `docker-compose`. +> specify the address of services like MongoDB and TypeSense. There is also a `Dockerfile` in the repo which you can use to build the container image and startup the app in a container: @@ -25,13 +98,7 @@ docker build -t jikan-rest:nightly . docker run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env jikan-rest:nightly ``` -If you need a different CPU architecture, set the `TARGET_ARCH` build argument: - -```bash -docker build -t jikan-rest:nightly --build-arg TARGET_ARCH=arm64 . -``` - -`TARGET_ARCH` is `amd64` by default. +> Most of the time it's enough to just use the image from [Docker Hub](https://hub.docker.com/r/jikanme/jikan-rest). ### Docker compose usage @@ -39,13 +106,24 @@ docker build -t jikan-rest:nightly --build-arg TARGET_ARCH=arm64 . docker-compose up ``` -Docker compose will use the `.env` file from the folder where you execute it from to load configurations for the -services. If you don't have a `.env` file yet in the folder, copy the `.env.dist` file, and set the passwords. +This does the same thing as the `container-setup.sh` script mostly, but you will have to create the secret files yourself. The following secret files are required for credentials (put them next to the `docker-compose.yml` file): + +- db_admin_password.txt +- db_admin_username.txt +- db_password.txt +- db_username.txt +- redis_password.txt +- typesense_api_key.txt + +You can customise the Jikan API config through `./docker/config/.env.compose` file. (E.g. you don't want CORS headers) > **Please note**: The syntax rules of docker compose for `.env` applies > here: https://docs.docker.com/compose/env-file/#syntax-rules -#### Note for Podman +> **Additional configuration**: You can change the mongodb memory usage via `MONGO_CACHE_SIZE_GB` environment variable. +> It sets how many gigabytes of memory is available for wired tiger. Default is `1`. This is useful for systems with low memory capacity. + +### Note for Podman If you build the container image yourself with podman, the resulting image format will be OCI by default. To make the health checks work in that situation you need to run the container the following way: @@ -54,9 +132,9 @@ To make the health checks work in that situation you need to run the container t podman run -d --name=jikan-rest -p 8080:8080 -v ./.env:/app/.env --health-start-period=5s --health-cmd="curl --fail http://localhost:2114/health?plugin=http || exit 1" jikan-rest:nightly ``` -#### Configuration of the container +### Configuration of the container -You can change the settings of Jikan through setting environment variables via the `-e` command line argument option for +You can also change the settings of Jikan through setting environment variables via the `-e` command line argument option for the `docker run` command. These environment variables are the same as the options found in the `.env` file. We also provide a sample file called `.env.dist`. @@ -101,6 +179,3 @@ the [Configuration Wiki page](https://github.com/jikan-me/jikan-rest/wiki/Config - the php processes ingesting the http requests - [Supercronic](https://github.com/aptible/supercronic), which runs cron jobs. - Queue workers for populating the search index and other background jobs. -- The container does all the scheduled background jobs automatically out-of-the-box: - - Importing new documents into the search index. - - Updating entries from upstream. diff --git a/docker-compose.yml b/docker-compose.yml index f7815e22..e1c0843d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,76 +1,94 @@ -# For now this is just for local development. This is not production ready. version: '3.8' volumes: - mongo-data: {} - redis-data: {} - tmp-data: {} - typesense-data: {} + mongo-data: { } + redis-data: { } + typesense-data: { } + +networks: + jikan_network: { } + +secrets: + db_username: + file: db_username.txt + db_password: + file: db_password.txt + db_admin_username: + file: db_admin_username.txt + db_admin_password: + file: db_admin_password.txt + redis_password: + file: redis_password.txt + typesense_api_key: + file: typesense_api_key.txt services: - jikan_rest: &jikan_rest - build: - context: . - dockerfile: Dockerfile + jikan_rest: + image: "jikanme/jikan-rest:${_JIKAN_API_VERSION:-latest}" user: "${APP_UID:-10001}:${APP_GID:-10001}" + networks: + - jikan_network + secrets: + - db_username + - db_password + - typesense_api_key + - redis_password environment: PS1: '\[\033[1;32m\]\[\033[1;36m\][\u@\h] \[\033[1;34m\]\w\[\033[0;35m\] \[\033[1;36m\]# \[\033[0m\]' - HOME: /tmp - APP_DEBUG: 'true' - APP_ENV: local - REDIS_HOST: redis - REDIS_PASSWORD: "${REDIS_PASSWORD:-null}" - DB_CONNECTION: mongodb - DB_HOST: mongodb - DB_DATABASE: jikan - DB_PORT: 27017 - DB_ADMIN: jikan - DB_USERNAME: "${DB_USERNAME}" - DB_PASSWORD: "${DB_PASSWORD}" - volumes: - - /etc/passwd:/etc/passwd:ro - - /etc/group:/etc/group:ro - - tmp-data:/tmp:rw - - .:/app:rw - depends_on: - mongodb: {condition: service_healthy} - redis: {condition: service_healthy} - typesense: {condition: service_healthy} - - web: - <<: *jikan_rest + env_file: + - ./docker/config/.env.compose ports: - '8080:8080/tcp' + hostname: jikan-rest-api healthcheck: test: [ 'CMD-SHELL', 'wget --spider -q "http://127.0.0.1:2114/health?plugin=http"' ] interval: 2s timeout: 2s - - cron: - <<: *jikan_rest - command: supercronic /etc/supercronic/laravel # it runs artisan schedule:run + links: + - mongodb:mongodb + - redis:redis + - typesense:typesense + depends_on: + mongodb: { condition: service_healthy } + redis: { condition: service_healthy } + typesense: { condition: service_healthy } mongodb: - image: mongo:focal + image: docker.io/mongo:focal + hostname: mongodb volumes: - mongo-data:/data/db + - ./docker/mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro ports: - - '27017/tcp' - command: --wiredTigerCacheSizeGB 1 + - 27017/tcp + command: "--wiredTigerCacheSizeGB ${MONGO_CACHE_SIZE_GB:-1.0}" + networks: + - jikan_network + secrets: + - db_username + - db_password + - db_admin_username + - db_admin_password environment: - MONGO_INITDB_ROOT_USERNAME: "${DB_USERNAME:-root}" - MONGO_INITDB_ROOT_PASSWORD: "${DB_PASSWORD}" + MONGO_INITDB_ROOT_USERNAME_FILE: /run/secrets/db_admin_username + MONGO_INITDB_ROOT_PASSWORD_FILE: /run/secrets/db_admin_password + MONGO_INITDB_DATABASE: jikan_admin healthcheck: - test: echo 'db.runCommand("ping").ok' | mongo mongodb://localhost:27017 --quiet + test: echo 'db.runCommand("ping").ok' | mongosh mongodb://localhost:27017 --quiet interval: 30s timeout: 10s retries: 5 redis: - image: redis:7-alpine + image: docker.io/redis:6-alpine + hostname: redis + secrets: + - redis_password + networks: + - jikan_network command: - /bin/sh - -c - - redis-server --requirepass "$${REDIS_PASSWORD:?REDIS_PASSWORD variable is not set}" + - redis-server --requirepass "$$(cat /run/secrets/redis_password)" volumes: - redis-data:/data:rw ports: @@ -81,13 +99,24 @@ services: timeout: 1s typesense: - image: typesense/typesense:0.23.1 + image: docker.io/typesense/typesense:0.24.1 + hostname: typesense entrypoint: /bin/sh + secrets: + - typesense_api_key + networks: + - jikan_network command: - -c - - /opt/typesense-server --data-dir /data --api-key "$${TYPESENSE_API_KEY:?TYPESENSE_API_KEY variable is not set}" - restart: no + - TYPESENSE_API_KEY="$$(cat /run/secrets/typesense_api_key)" /opt/typesense-server --data-dir /data + deploy: + restart_policy: + condition: none volumes: - typesense-data:/data ports: - "8108/tcp" + healthcheck: + test: [ 'CMD-SHELL', '{ ! [ -f "curl_created" ] && apt -qq update -y && apt -qq install -y curl && touch curl_created && curl -s -f http://localhost:8108/health; } || { curl -s -f http://localhost:8108/health; }' ] + interval: 5s + timeout: 2s diff --git a/docker-entrypoint.php b/docker-entrypoint.php index 3eccdcd7..8fd3fe74 100644 --- a/docker-entrypoint.php +++ b/docker-entrypoint.php @@ -19,6 +19,10 @@ "DB_PASSWORD" => "" ]; +// get a copy of the current env vars. +// these are the ones that are set during the container creation +$current_env = $_ENV; + if (!file_exists(".env")) { copy(".env.dist", ".env"); $writer = new \MirazMac\DotEnv\Writer(__DIR__ . '/' . '.env'); @@ -29,9 +33,35 @@ $writer->write(); } +// We'd like to support Container secrets. So we'll check if any of the env vars has a __FILE suffix +// then we'll try to load the file and set the env var to the contents of the file. +// https://docs.docker.com/engine/swarm/secrets/ +// Additionally we need to write the secrets to the .env file so the workers in roadrunner can access them. +// (it might just pass down the global env vars, but haven't tested that yet) +$envWriter = new \MirazMac\DotEnv\Writer(__DIR__ . '/' . '.env'); +$itemsWritten = 0; +foreach (array_keys($current_env) as $env_key) { + if (!str_contains($env_key, "__FILE")) { + continue; + } + if (!file_exists($current_env[$env_key])) { + echo "Couldn't load secret: " . $_ENV[$env_key] . PHP_EOL; + continue; + } + $originalKey = str_replace("__FILE", "", $env_key); + $secretsFileContents = file_get_contents($current_env[$env_key]); + $envWriter->set($originalKey, str_replace(["\n", "\r"], "", $secretsFileContents)); + $itemsWritten++; +} + +if ($itemsWritten > 0) { + $envWriter->write(); + echo "Secrets loaded successfully.\n"; +} $dotenv = Dotenv::createImmutable(__DIR__); $dotenv->load(); + $current_env = $_ENV; if ($current_env["SCOUT_DRIVER"] === "typesense" && empty($current_env["TYPESENSE_API_KEY"])) { diff --git a/docker/config/.env.compose b/docker/config/.env.compose new file mode 100644 index 00000000..193d4530 --- /dev/null +++ b/docker/config/.env.compose @@ -0,0 +1,21 @@ +APP_DEBUG=false +LOG_LEVEL=info +APP_ENV=production +CACHING=true +CACHE_DRIVER=redis +REDIS_HOST=redis +REDIS_PASSWORD__FILE=/run/secrets/redis_password +DB_CONNECTION=mongodb +DB_HOST=mongodb +DB_DATABASE=jikan +DB_USERNAME__FILE=/run/secrets/db_username +DB_ADMIN__FILE=/run/secrets/db_username +DB_PASSWORD__FILE=/run/secrets/db_password +SCOUT_DRIVER=typesense +SCOUT_QUEUE=false +TYPESENSE_HOST=typesense +TYPESENSE_PORT=8108 +TYPESENSE_API_KEY__FILE=/run/secrets/typesense_api_key +CORS_MIDDLEWARE=true +MICROCACHING=true +MICROCACHING_EXPIRE=60 diff --git a/docker/mongo-init.js b/docker/mongo-init.js new file mode 100644 index 00000000..01274f0e --- /dev/null +++ b/docker/mongo-init.js @@ -0,0 +1,17 @@ +const userToCreate = fs.readFileSync('/run/secrets/db_username', 'utf8'); +const userPassword = fs.readFileSync('/run/secrets/db_password', 'utf8'); +db = db.getSiblingDB("admin"); + +db.createUser({ + user: userToCreate, + pwd: userPassword, + roles: [{ role: "readWrite", db: "jikan" }], +}); + +db = db.getSiblingDB("jikan"); + +db.createUser({ + user: userToCreate, + pwd: userPassword, + roles: [{ role: "readWrite", db: "jikan" }], +}); diff --git a/storage/app/.gitignore b/storage/app/.gitignore index ce9fcc1f..11df8699 100644 --- a/storage/app/.gitignore +++ b/storage/app/.gitignore @@ -2,3 +2,5 @@ failovers.json source_failover.lock jikan_model_classes.json +container_compose_runtime +container_runtime