diff --git a/librephotos-docker/.devcontainer.json b/librephotos-docker/.devcontainer.json new file mode 100644 index 0000000..372f017 --- /dev/null +++ b/librephotos-docker/.devcontainer.json @@ -0,0 +1,14 @@ +// See https://aka.ms/vscode-remote/devcontainer.json for format details. +{ + "dockerComposeFile": ["docker-compose.yml", "docker-compose.dev.yml"], + "service": "backend", + "workspaceFolder": "/code/", + "extensions": ["ms-python.python", + "ms-vscode.atom-keybindings", + "mrorz.language-gettext", + "ms-python.vscode-pylance", + "batisteo.vscode-django", + "keno.uikit-3-snippets", + "dbaeumer.vscode-eslint", + "christian-kohler.npm-intellisense"] +} diff --git a/librephotos-docker/.dockerignore b/librephotos-docker/.dockerignore new file mode 100644 index 0000000..3e4e90f --- /dev/null +++ b/librephotos-docker/.dockerignore @@ -0,0 +1,7 @@ +.gitignore +Dockerfile +Makefile +README.md +screenshots/ +tags +media/ diff --git a/librephotos-docker/.github/workflows/backend-base.yml b/librephotos-docker/.github/workflows/backend-base.yml new file mode 100644 index 0000000..eec1b89 --- /dev/null +++ b/librephotos-docker/.github/workflows/backend-base.yml @@ -0,0 +1,51 @@ +name: backend-base + +on: + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'backend/base/**' + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Running Tests" + + build: + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./backend/base + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-base:dev diff --git a/librephotos-docker/.github/workflows/backend-dependencies.yml b/librephotos-docker/.github/workflows/backend-dependencies.yml new file mode 100644 index 0000000..06a38c8 --- /dev/null +++ b/librephotos-docker/.github/workflows/backend-dependencies.yml @@ -0,0 +1,51 @@ +name: backend-dependencies + +on: + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'backend/dependencies/**' + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Running Tests" + + build: + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./backend/dependencies + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-dependencies:dev diff --git a/librephotos-docker/.github/workflows/backend.yml b/librephotos-docker/.github/workflows/backend.yml new file mode 100644 index 0000000..13cb7d1 --- /dev/null +++ b/librephotos-docker/.github/workflows/backend.yml @@ -0,0 +1,90 @@ +name: backend + +on: + release: + types: [created] + push: + branches: + - main + paths: + - 'backend/**' + repository_dispatch: + types: backend-commit-event + +env: + IMAGE_NAME: librephotos + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Running Tests" + + build: + name: Building dev build + # Ensure test job passes before pushing image. + if: github.event_name != 'release' + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./backend + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos:dev + + release-job: + name: Releasing + if: github.event_name == 'release' && github.event.action == 'created' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./backend + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos:${{ github.event.release.tag_name }},reallibrephotos/librephotos:latest diff --git a/librephotos-docker/.github/workflows/dlib-builder.yml b/librephotos-docker/.github/workflows/dlib-builder.yml new file mode 100644 index 0000000..4192373 --- /dev/null +++ b/librephotos-docker/.github/workflows/dlib-builder.yml @@ -0,0 +1,55 @@ +# This is a basic workflow to help you get started with Actions + +name: Build dlib dependency + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the main branch + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'dependencies/dlib/**' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./dependencies/dlib + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/dlib-builder:dev + upload: + # The type of runner that the job will run on + runs-on: reallibrephotos/dlib-builder:dev + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Upload a Build Artifact + uses: actions/upload-artifact@v2.3.1 + with: + name: dlib + path: /tmp/builds/dlib.zip diff --git a/librephotos-docker/.github/workflows/frontend.yml b/librephotos-docker/.github/workflows/frontend.yml new file mode 100644 index 0000000..ed5da9a --- /dev/null +++ b/librephotos-docker/.github/workflows/frontend.yml @@ -0,0 +1,99 @@ +name: frontend + +on: + release: + types: [created] + repository_dispatch: + types: frontend-commit-event + + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'frontend/**' + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: librephotos-frontend + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Tests" + + build: + if: github.event_name != 'release' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./frontend + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-frontend:dev + + release-job: + name: Releasing + if: github.event_name == 'release' && github.event.action == 'created' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./frontend + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-frontend:${{ github.event.release.tag_name }},reallibrephotos/librephotos-frontend:latest diff --git a/librephotos-docker/.github/workflows/proxy.yml b/librephotos-docker/.github/workflows/proxy.yml new file mode 100644 index 0000000..59ac48c --- /dev/null +++ b/librephotos-docker/.github/workflows/proxy.yml @@ -0,0 +1,88 @@ +name: proxy + +on: + release: + types: [created] + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'proxy/**' + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Tests" + + build: + if: github.event_name != 'release' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./proxy + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-proxy:dev + cache-from: type=gha + cache-to: type=gha,mode=max + + release-job: + name: Releasing + if: github.event_name == 'release' && github.event.action == 'created' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./proxy + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-proxy:${{ github.event.release.tag_name }},reallibrephotos/librephotos-proxy:latest diff --git a/librephotos-docker/.gitignore b/librephotos-docker/.gitignore new file mode 100644 index 0000000..1841d63 --- /dev/null +++ b/librephotos-docker/.gitignore @@ -0,0 +1,172 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# visual studio +.vs/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# LibrePhotos +densecap/data/models/densecap/densecap-pretrained-vgg16.t7 +*/*.pkl +*/*/*.pkl +thumbnails +media +samplephotos +tags +api/places365/model/ +Conv2d.patch +Linear.patch +Sequential.patch +BatchNorm2d.patch +AvgPool2d.patch +ReLU.patch +run_docker.sh +logs/ +playground +api/im2txt/data/ +api/im2txt/models/ +api/im2txt/png/ +*.ipynb +api/im2txt/*.tar.gz +api/places365/*.tar.gz +*.db +media* +protected_media +librephotos +vscode/server-extensions +vscode/server-insiders-extensions diff --git a/librephotos-docker/LICENSE b/librephotos-docker/LICENSE new file mode 100644 index 0000000..dc92c51 --- /dev/null +++ b/librephotos-docker/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 LibrePhotos + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/librephotos-docker/Makefile b/librephotos-docker/Makefile new file mode 100644 index 0000000..4faaaa4 --- /dev/null +++ b/librephotos-docker/Makefile @@ -0,0 +1,22 @@ +.PHONY: default build rename + +include librephotos.env +REPLACE_NAMES=sed 's/__backend_name__/$(BACKEND_CONT_NAME)/g; s/__frontend_name__/$(FRONTEND_CONT_NAME)/g; s/__proxy_name__/$(PROXY_CONT_NAME)/g; s/__redis_name__/$(REDIS_CONT_NAME)/g; s/__db_name__/$(DB_CONT_NAME)/g; s/__pgadmin_name__/$(PGADMIN_CONT_NAME)/g; s/__network_name__/$(NETWORK_NAME)/g' + +default: build + +build: build-backend build-frontend build-proxy + +build-backend: + docker build -t reallibrephotos/librephotos:latest backend + +build-frontend: + docker build -t reallibrephotos/librephotos-frontend:latest frontend + +build-proxy: + docker build -t reallibrephotos/librephotos-proxy:latest proxy + +rename: + $(REPLACE_NAMES) docker-compose.raw > docker-compose.yml + $(REPLACE_NAMES) docker-compose.dev.raw > docker-compose.dev.yml + $(REPLACE_NAMES) docker-compose.e2e.raw > docker-compose.e2e.yml diff --git a/librephotos-docker/README.md b/librephotos-docker/README.md new file mode 100644 index 0000000..f22ca1f --- /dev/null +++ b/librephotos-docker/README.md @@ -0,0 +1,13 @@ +# For common users: +Follow the instructions [here](https://docs.librephotos.com/1/standard_install/) + +# For developers: +Follow the instructions [here](https://docs.librephotos.com/1/dev_install/) + +# To use your own container names: +Follow the normal instructions as per your chosen build, but after updating the `.env` file to choose your container names, run +``` sh +make rename +``` +Then you can resume following the normal instructions. + diff --git a/librephotos-docker/backend/Dockerfile b/librephotos-docker/backend/Dockerfile new file mode 100644 index 0000000..edf742e --- /dev/null +++ b/librephotos-docker/backend/Dockerfile @@ -0,0 +1,14 @@ +FROM reallibrephotos/librephotos-dependencies:dev +# actual project +ARG DEBUG +WORKDIR /code +RUN git clone --depth 1 https://github.com/LibrePhotos/librephotos . +RUN pip install --no-cache-dir -r requirements.txt +RUN if [ "$DEBUG" = 1 ] ; then \ + pip install setuptools==57.5.0; \ + pip install -r requirements.dev.txt; \ + fi +EXPOSE 8001 + +COPY entrypoint.sh /entrypoint.sh +CMD ["/entrypoint.sh"] diff --git a/librephotos-docker/backend/base/Dockerfile b/librephotos-docker/backend/base/Dockerfile new file mode 100644 index 0000000..26afeab --- /dev/null +++ b/librephotos-docker/backend/base/Dockerfile @@ -0,0 +1,93 @@ +FROM ubuntu:jammy +ARG TARGETPLATFORM +ENV DEBIAN_FRONTEND=noninteractive + +# Install python +RUN apt-get update \ + && apt-get install -y python3-pip python3-dev \ + && cd /usr/local/bin \ + && ln -s /usr/bin/python3 python \ + && pip3 install --upgrade pip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# system packages installation +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + bzip2 \ + curl \ + ffmpeg \ + git \ + libboost-all-dev \ + libcfitsio-dev \ + libexif-dev \ + libexpat-dev \ + libexpat1-dev \ + libgif-dev \ + libgl1-mesa-glx \ + libglib2.0-dev \ + libgsf-1-dev \ + libheif-dev \ + libimage-exiftool-perl \ + libimagequant-dev \ + libjpeg-dev \ + liblapack-dev \ + liblcms2-dev \ + libmagic1 \ + libopenblas-dev \ + libopenexr-dev \ + liborc-dev \ + libpng-dev \ + libpq-dev \ + libraw-dev \ + librsvg2-dev \ + libsm6 \ + libtiff5-dev \ + libtool \ + libtool-bin \ + libwebp-dev \ + libxrender-dev \ + pkg-config \ + rustc \ + libtinfo5 \ + swig \ + unzip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then pip3 install --no-cache-dir torch torchvision -f https://torch.kmtea.eu/whl/stable.html; else pip3 install --no-cache-dir torch torchvision; fi + +RUN pip3 install --no-cache-dir cmake==3.21.2 + +#Build and install libraw +WORKDIR /tmp/builds +RUN git clone https://github.com/LibRaw/LibRaw && \ + cd LibRaw && \ + git reset --hard 2a9a4de21ea7f5d15314da8ee5f27feebf239655 && \ + autoreconf --install && \ + ./configure && \ + make && \ + make install && \ + rm -rf /tmp/builds/* + +#Build and install imagemagick +WORKDIR /tmp/builds +ARG IMAGEMAGICK_VERSION=7.1.0-48 +RUN curl -SL https://imagemagick.org/archive/releases/ImageMagick-${IMAGEMAGICK_VERSION}.tar.xz | tar -xJf- && \ + cd ImageMagick-${IMAGEMAGICK_VERSION} && \ + ./configure --with-modules && \ + make install && \ + ldconfig /usr/local/lib && \ + rm -rf /tmp/builds/* + +# Build and install libvips +WORKDIR /tmp/builds +ARG VIPSVERSION=8.13.0 +RUN curl -SL https://github.com/libvips/libvips/releases/download/v${VIPSVERSION}/vips-${VIPSVERSION}.tar.gz | tar -xz \ + && cd vips-${VIPSVERSION} \ + && ./configure \ + && make V=0 \ + && make install \ + && ldconfig \ + && rm -rf /tmp/builds/* diff --git a/librephotos-docker/backend/dependencies/Dockerfile b/librephotos-docker/backend/dependencies/Dockerfile new file mode 100644 index 0000000..8e4ab59 --- /dev/null +++ b/librephotos-docker/backend/dependencies/Dockerfile @@ -0,0 +1,21 @@ +FROM reallibrephotos/librephotos-base:dev + +# Build and install dlib +# Compile it WITHOUT AVX and SSE4 instructions to ensure compatibility +WORKDIR /tmp/builds +RUN git clone --depth 1 --branch 'v19.24' https://github.com/davisking/dlib.git && \ + mkdir dlib/build && \ + cd dlib/build && \ + cmake .. -DDLIB_USE_CUDA=0 -DUSE_AVX_INSTRUCTIONS=0 -DLIB_NO_GUI_SUPPORT=0 && \ + cmake --build . && \ + cd /tmp/builds/dlib && \ + python3 setup.py install --no USE_AVX_INSTRUCTIONS --no DLIB_USE_CUDA --no USE_SSE4_INSTRUCTIONS && \ + rm -rf /tmp/builds/* + +# Download pretrained models +WORKDIR /data_models +RUN mkdir -p /root/.cache/torch/hub/checkpoints/ && \ + curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/places365.tar.gz | tar -zxC /data_models/ && \ + curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/im2txt.tar.gz | tar -zxC /data_models/ && \ + curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/clip-embeddings.tar.gz | tar -zxC /data_models/ && \ + curl -SL https://download.pytorch.org/models/resnet152-b121ed2d.pth -o /root/.cache/torch/hub/checkpoints/resnet152-b121ed2d.pth diff --git a/librephotos-docker/backend/entrypoint.sh b/librephotos-docker/backend/entrypoint.sh new file mode 100644 index 0000000..d6a4bd4 --- /dev/null +++ b/librephotos-docker/backend/entrypoint.sh @@ -0,0 +1,27 @@ +#! /bin/bash +export PYTHONUNBUFFERED=TRUE +export PYTHONFAULTHANDLER=1 +mkdir -p /logs +python image_similarity/main.py 2>&1 | tee /logs/gunicorn_image_similarity.log & +python manage.py showmigrations | tee /logs/show_migrate.log +python manage.py migrate | tee /logs/command_migrate.log +python manage.py showmigrations | tee /logs/show_migrate.log +python manage.py clear_cache + +if [ -n "$ADMIN_USERNAME" ] +then + python manage.py createadmin -u $ADMIN_USERNAME $ADMIN_EMAIL 2>&1 | tee /logs/command_createadmin.log +fi + +echo "Running backend server..." + +python manage.py rqworker default 2>&1 | tee /logs/rqworker.log & + +if [ "$DEBUG" = 1 ] +then + echo "development backend starting" + gunicorn --worker-class=gevent --reload --bind 0.0.0.0:8001 --log-level=info ownphotos.wsgi 2>&1 | tee /logs/gunicorn_django.log +else + echo "production backend starting" + gunicorn --worker-class=gevent --bind 0.0.0.0:8001 --log-level=info ownphotos.wsgi 2>&1 | tee /logs/gunicorn_django.log +fi diff --git a/librephotos-docker/dependencies/dlib/Dockerfile b/librephotos-docker/dependencies/dlib/Dockerfile new file mode 100644 index 0000000..1ed8c41 --- /dev/null +++ b/librephotos-docker/dependencies/dlib/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:hirsute +ARG TARGETPLATFORM +ENV DEBIAN_FRONTEND=noninteractive + +# Install python +RUN apt-get update \ + && apt-get install -y python3-pip python3-dev \ + && cd /usr/local/bin \ + && ln -s /usr/bin/python3 python \ + && pip3 install --upgrade pip + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + zip \ + git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install cmake==3.21.2 + +# Build and install dlib +WORKDIR /tmp/builds +RUN git clone --depth 1 --branch 'v19.24' https://github.com/davisking/dlib.git && \ + mkdir dlib/build && \ + cd dlib/build && \ + cmake .. -DDLIB_USE_CUDA=0 -DUSE_AVX_INSTRUCTIONS=0 -DLIB_NO_GUI_SUPPORT=0 && \ + cmake --build . && \ + zip -r dlib.zip dlib \ diff --git a/librephotos-docker/docker-compose.dev.raw b/librephotos-docker/docker-compose.dev.raw new file mode 100644 index 0000000..cb416fd --- /dev/null +++ b/librephotos-docker/docker-compose.dev.raw @@ -0,0 +1,73 @@ +# Run options: +# 1. There are no options - This add additional tools to aid in the development of Libre Photos +# run cmd: docker-compose up -f docker-compose.yml -f docker-compose.dev.yml -d +# 2. Current added tools: +# pgadmin User admin@admin pass admin port 3001 + +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: '3.8' +services: + + __proxy_name__: + tty: true + build: + context: ./proxy + dockerfile: Dockerfile + container_name: __proxy_name__ + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + + __frontend_name__: + tty: true + environment: + - DEBUG=1 + - WDS_SOCKET_PORT=0 # needed for webpack-dev-server + build: + context: ./frontend + dockerfile: Dockerfile.dev + container_name: __frontend_name__ + volumes: + - ${codedir}/librephotos-frontend:/usr/src/app + + __backend_name__: + tty: true + stdin_open: true + environment: + - DEBUG=1 + build: + context: ./backend + dockerfile: Dockerfile + args: + DEBUG: 1 + container_name: __backend_name__ + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + - ${data}/logs:/logs + - ${data}/cache:/root/.cache + - ${codedir}/librephotos:/code + - ./vscode/server-extensions:/root/.vscode-server/extensions + - ./vscode/server-insiders-extensions:/root/.vscode-server-insiders/extensions + - ./vscode/settings.json:/code/.vscode/settings.json + - ./backend/entrypoint.sh:/entrypoint.sh + + __pgadmin_name__: + image: dpage/pgadmin4 + container_name: __pgadmin_name__ + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@admin.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} + volumes: + - ${pgAdminLocation}/pgadmin:/root/.pgadmin + ports: + - "3001:80" + restart: unless-stopped diff --git a/librephotos-docker/docker-compose.dev.yml b/librephotos-docker/docker-compose.dev.yml new file mode 100644 index 0000000..41e3bbb --- /dev/null +++ b/librephotos-docker/docker-compose.dev.yml @@ -0,0 +1,73 @@ +# Run options: +# 1. There are no options - This add additional tools to aid in the development of Libre Photos +# run cmd: docker-compose up -f docker-compose.yml -f docker-compose.dev.yml -d +# 2. Current added tools: +# pgadmin User admin@admin pass admin port 3001 + +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: '3.8' +services: + + proxy: + tty: true + build: + context: ./proxy + dockerfile: Dockerfile + container_name: proxy + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + + frontend: + tty: true + environment: + - DEBUG=1 + - WDS_SOCKET_PORT=0 # needed for webpack-dev-server + build: + context: ./frontend + dockerfile: Dockerfile.dev + container_name: frontend + volumes: + - ${codedir}/librephotos-frontend:/usr/src/app + + backend: + tty: true + stdin_open: true + environment: + - DEBUG=1 + build: + context: ./backend + dockerfile: Dockerfile + args: + DEBUG: 1 + container_name: backend + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + - ${data}/logs:/logs + - ${data}/cache:/root/.cache + - ${codedir}/librephotos:/code + - ./vscode/server-extensions:/root/.vscode-server/extensions + - ./vscode/server-insiders-extensions:/root/.vscode-server-insiders/extensions + - ./vscode/settings.json:/code/.vscode/settings.json + - ./backend/entrypoint.sh:/entrypoint.sh + + pgadmin: + image: dpage/pgadmin4 + container_name: pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@admin.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} + volumes: + - ${pgAdminLocation}/pgadmin:/root/.pgadmin + ports: + - "3001:80" + restart: unless-stopped diff --git a/librephotos-docker/docker-compose.e2e.raw b/librephotos-docker/docker-compose.e2e.raw new file mode 100644 index 0000000..82da04e --- /dev/null +++ b/librephotos-docker/docker-compose.e2e.raw @@ -0,0 +1,137 @@ +networks: + e2e: + +volumes: + e2e_db: + e2e_cache: + e2e_logs: + e2e_protected_media: + e2e_scan_directory: + +services: + e2e-__proxy_name__: + image: reallibrephotos/librephotos-proxy:${tag} + container_name: e2e-__proxy_name__ + restart: unless-stopped + environment: + BACKEND_NAME: e2e-__backend_name__ + FRONTEND_NAME: e2e-__frontend_name__ + volumes: + - e2e_scan_directory:/data + - e2e_protected_media:/protected_media + ports: + - 8080:80 + healthcheck: + test: curl -sI localhost | grep HTTP | grep 200 + interval: 5s + timeout: 5s + retries: 12 + depends_on: + e2e-__backend_name__: + condition: service_healthy + e2e-__frontend_name__: + condition: service_healthy + networks: + - e2e + + e2e-__db_name__: + image: postgres:13 + container_name: e2e-__db_name__ + restart: unless-stopped + environment: + - POSTGRES_USER=${dbUser} + - POSTGRES_PASSWORD=${dbPass} + - POSTGRES_DB=${dbName} + volumes: + - e2e_db:/var/lib/postgresql/data + healthcheck: + test: psql -U ${dbUser} -d ${dbName} -c "SELECT 1;" + interval: 5s + timeout: 5s + retries: 12 + networks: + - e2e + + e2e-__frontend_name__: + container_name: e2e-__frontend_name__ + environment: + - DEBUG=1 + - WDS_SOCKET_PORT=0 # needed for webpack-dev-server + tty: true + build: + context: ./frontend + dockerfile: Dockerfile.dev + restart: unless-stopped + volumes: + - ${codedir}/librephotos-frontend:/usr/src/app + healthcheck: + test: curl -sI localhost:3000 | grep HTTP | grep 200 + interval: 5s + timeout: 5s + retries: 24 + depends_on: + e2e-__backend_name__: + condition: service_healthy + networks: + - e2e + + e2e-__backend_name__: + container_name: e2e-__backend_name__ + tty: true + stdin_open: true + build: + context: ./backend + dockerfile: Dockerfile + args: + DEBUG: 1 + restart: unless-stopped + volumes: + - ${codedir}/librephotos:/code + - e2e_scan_directory:/data + - e2e_protected_media:/protected_media + - e2e_logs:/logs + - e2e_cache:/root/.cache + environment: + - SECRET_KEY=${shhhhKey:-} + - BACKEND_HOST=e2e-__backend_name__ + - ADMIN_EMAIL=${adminEmail:-admin@localhost} + - ADMIN_USERNAME=${userName:-admin} + - ADMIN_PASSWORD=${userPass:-admin} + - DB_BACKEND=postgresql + - DB_NAME=${dbName} + - DB_USER=${dbUser} + - DB_PASS=${dbPass} + - DB_HOST=e2e-__db_name__ + - DB_PORT=5432 + - REDIS_HOST=e2e-__redis_name__ + - REDIS_PORT=6379 + - MAPBOX_API_KEY=${mapApiKey:-} + - WEB_CONCURRENCY=${gunniWorkers:-1} + - SKIP_PATTERNS=${skipPatterns:-} + - ALLOW_UPLOAD=${allowUpload:-true} + - DEBUG=1 + - HEAVYWEIGHT_PROCESS=${HEAVYWEIGHT_PROCESS:-1} + healthcheck: + test: curl -sI localhost:8001 | grep HTTP | grep 401 + interval: 5s + timeout: 5s + retries: 24 + depends_on: + e2e-__db_name__: + condition: service_healthy + e2e-__redis_name__: + condition: service_healthy + networks: + - e2e + + e2e-__redis_name__: + image: redis:6 + container_name: e2e-__redis_name__ + restart: unless-stopped + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 5s + timeout: 5s + retries: 12 + networks: + - e2e diff --git a/librephotos-docker/docker-compose.e2e.yml b/librephotos-docker/docker-compose.e2e.yml new file mode 100644 index 0000000..f54230d --- /dev/null +++ b/librephotos-docker/docker-compose.e2e.yml @@ -0,0 +1,137 @@ +networks: + e2e: + +volumes: + e2e_db: + e2e_cache: + e2e_logs: + e2e_protected_media: + e2e_scan_directory: + +services: + e2e-proxy: + image: reallibrephotos/librephotos-proxy:${tag} + container_name: e2e-proxy + restart: unless-stopped + environment: + BACKEND_NAME: e2e-backend + FRONTEND_NAME: e2e-frontend + volumes: + - e2e_scan_directory:/data + - e2e_protected_media:/protected_media + ports: + - 8080:80 + healthcheck: + test: curl -sI localhost | grep HTTP | grep 200 + interval: 5s + timeout: 5s + retries: 12 + depends_on: + e2e-backend: + condition: service_healthy + e2e-frontend: + condition: service_healthy + networks: + - e2e + + e2e-db: + image: postgres:13 + container_name: e2e-db + restart: unless-stopped + environment: + - POSTGRES_USER=${dbUser} + - POSTGRES_PASSWORD=${dbPass} + - POSTGRES_DB=${dbName} + volumes: + - e2e_db:/var/lib/postgresql/data + healthcheck: + test: psql -U ${dbUser} -d ${dbName} -c "SELECT 1;" + interval: 5s + timeout: 5s + retries: 12 + networks: + - e2e + + e2e-frontend: + container_name: e2e-frontend + environment: + - DEBUG=1 + - WDS_SOCKET_PORT=0 # needed for webpack-dev-server + tty: true + build: + context: ./frontend + dockerfile: Dockerfile.dev + restart: unless-stopped + volumes: + - ${codedir}/librephotos-frontend:/usr/src/app + healthcheck: + test: curl -sI localhost:3000 | grep HTTP | grep 200 + interval: 5s + timeout: 5s + retries: 24 + depends_on: + e2e-backend: + condition: service_healthy + networks: + - e2e + + e2e-backend: + container_name: e2e-backend + tty: true + stdin_open: true + build: + context: ./backend + dockerfile: Dockerfile + args: + DEBUG: 1 + restart: unless-stopped + volumes: + - ${codedir}/librephotos:/code + - e2e_scan_directory:/data + - e2e_protected_media:/protected_media + - e2e_logs:/logs + - e2e_cache:/root/.cache + environment: + - SECRET_KEY=${shhhhKey:-} + - BACKEND_HOST=e2e-backend + - ADMIN_EMAIL=${adminEmail:-admin@localhost} + - ADMIN_USERNAME=${userName:-admin} + - ADMIN_PASSWORD=${userPass:-admin} + - DB_BACKEND=postgresql + - DB_NAME=${dbName} + - DB_USER=${dbUser} + - DB_PASS=${dbPass} + - DB_HOST=e2e-db + - DB_PORT=5432 + - REDIS_HOST=e2e-redis + - REDIS_PORT=6379 + - MAPBOX_API_KEY=${mapApiKey:-} + - WEB_CONCURRENCY=${gunniWorkers:-1} + - SKIP_PATTERNS=${skipPatterns:-} + - ALLOW_UPLOAD=${allowUpload:-true} + - DEBUG=1 + - HEAVYWEIGHT_PROCESS=${HEAVYWEIGHT_PROCESS:-1} + healthcheck: + test: curl -sI localhost:8001 | grep HTTP | grep 401 + interval: 5s + timeout: 5s + retries: 24 + depends_on: + e2e-db: + condition: service_healthy + e2e-redis: + condition: service_healthy + networks: + - e2e + + e2e-redis: + image: redis:6 + container_name: e2e-redis + restart: unless-stopped + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 5s + timeout: 5s + retries: 12 + networks: + - e2e diff --git a/librephotos-docker/docker-compose.raw b/librephotos-docker/docker-compose.raw new file mode 100644 index 0000000..27ad718 --- /dev/null +++ b/librephotos-docker/docker-compose.raw @@ -0,0 +1,90 @@ +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: "3.8" +services: + __proxy_name__: + image: reallibrephotos/librephotos-proxy:${tag} + container_name: __proxy_name__ + restart: unless-stopped + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + ports: + - ${httpPort}:80 + depends_on: + - __backend_name__ + - __frontend_name__ + + __db_name__: + image: postgres:13 + container_name: __db_name__ + restart: unless-stopped + environment: + - POSTGRES_USER=${dbUser} + - POSTGRES_PASSWORD=${dbPass} + - POSTGRES_DB=${dbName} + volumes: + - ${data}/db:/var/lib/postgresql/data + command: postgres -c fsync=off -c synchronous_commit=off -c full_page_writes=off -c random_page_cost=1.0 + healthcheck: + test: psql -U ${dbUser} -d ${dbName} -c "SELECT 1;" + interval: 5s + timeout: 5s + retries: 5 + + __frontend_name__: + image: reallibrephotos/librephotos-frontend:${tag} + container_name: __frontend_name__ + restart: unless-stopped + + __backend_name__: + image: reallibrephotos/librephotos:${tag} + container_name: __backend_name__ + restart: unless-stopped + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + - ${data}/logs:/logs + - ${data}/cache:/root/.cache + environment: + - SECRET_KEY=${shhhhKey:-} + - BACKEND_HOST=__backend_name__ + - ADMIN_EMAIL=${adminEmail:-} + - ADMIN_USERNAME=${userName:-} + - ADMIN_PASSWORD=${userPass:-} + - DB_BACKEND=postgresql + - DB_NAME=${dbName} + - DB_USER=${dbUser} + - DB_PASS=${dbPass} + - DB_HOST=${dbHost} + - DB_PORT=5432 + - REDIS_HOST=__redis_name__ + - REDIS_PORT=6379 + - MAPBOX_API_KEY=${mapApiKey:-} + - WEB_CONCURRENCY=${gunniWorkers:-1} + - SKIP_PATTERNS=${skipPatterns:-} + - ALLOW_UPLOAD=${allowUpload:-false} + - DEBUG=0 + - HEAVYWEIGHT_PROCESS=${HEAVYWEIGHT_PROCESS:-} + depends_on: + __db_name__: + condition: service_healthy + __redis_name__: + condition: service_healthy + + __redis_name__: + image: redis:6 + container_name: __redis_name__ + restart: unless-stopped + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 5s + timeout: 5s + retries: 12 diff --git a/librephotos-docker/docker-compose.yml b/librephotos-docker/docker-compose.yml new file mode 100644 index 0000000..9d910a4 --- /dev/null +++ b/librephotos-docker/docker-compose.yml @@ -0,0 +1,90 @@ +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: "3.8" +services: + proxy: + image: reallibrephotos/librephotos-proxy:${tag} + container_name: proxy + restart: unless-stopped + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + ports: + - ${httpPort}:80 + depends_on: + - backend + - frontend + + db: + image: postgres:13 + container_name: db + restart: unless-stopped + environment: + - POSTGRES_USER=${dbUser} + - POSTGRES_PASSWORD=${dbPass} + - POSTGRES_DB=${dbName} + volumes: + - ${data}/db:/var/lib/postgresql/data + command: postgres -c fsync=off -c synchronous_commit=off -c full_page_writes=off -c random_page_cost=1.0 + healthcheck: + test: psql -U ${dbUser} -d ${dbName} -c "SELECT 1;" + interval: 5s + timeout: 5s + retries: 5 + + frontend: + image: reallibrephotos/librephotos-frontend:${tag} + container_name: frontend + restart: unless-stopped + + backend: + image: reallibrephotos/librephotos:${tag} + container_name: backend + restart: unless-stopped + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + - ${data}/logs:/logs + - ${data}/cache:/root/.cache + environment: + - SECRET_KEY=${shhhhKey:-} + - BACKEND_HOST=backend + - ADMIN_EMAIL=${adminEmail:-} + - ADMIN_USERNAME=${userName:-} + - ADMIN_PASSWORD=${userPass:-} + - DB_BACKEND=postgresql + - DB_NAME=${dbName} + - DB_USER=${dbUser} + - DB_PASS=${dbPass} + - DB_HOST=${dbHost} + - DB_PORT=5432 + - REDIS_HOST=redis + - REDIS_PORT=6379 + - MAPBOX_API_KEY=${mapApiKey:-} + - WEB_CONCURRENCY=${gunniWorkers:-1} + - SKIP_PATTERNS=${skipPatterns:-} + - ALLOW_UPLOAD=${allowUpload:-false} + - DEBUG=0 + - HEAVYWEIGHT_PROCESS=${HEAVYWEIGHT_PROCESS:-} + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + + redis: + image: redis:6 + container_name: redis + restart: unless-stopped + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 5s + timeout: 5s + retries: 12 diff --git a/librephotos-docker/frontend/Dockerfile b/librephotos-docker/frontend/Dockerfile new file mode 100644 index 0000000..3a797ed --- /dev/null +++ b/librephotos-docker/frontend/Dockerfile @@ -0,0 +1,20 @@ +FROM node:13-slim as builder + +RUN apt-get update && apt-get install -y git + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app +ENV CLI_WIDTH 80 +RUN git clone https://github.com/LibrePhotos/librephotos-frontend /usr/src/app +RUN npm install --legacy-peer-deps +RUN npm run postinstall +RUN npm run build + +FROM halverneus/static-file-server +ENV PORT 3000 +EXPOSE 3000 + +COPY --from=builder /usr/src/app/build /web + +ENTRYPOINT ["/serve"] +CMD [] \ No newline at end of file diff --git a/librephotos-docker/frontend/Dockerfile.dev b/librephotos-docker/frontend/Dockerfile.dev new file mode 100644 index 0000000..200d07e --- /dev/null +++ b/librephotos-docker/frontend/Dockerfile.dev @@ -0,0 +1,8 @@ +FROM node:13-slim + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app +RUN apt-get update && apt-get install git curl -y +EXPOSE 3000 +COPY entrypoint.sh /entrypoint.sh +CMD ["/entrypoint.sh"] diff --git a/librephotos-docker/frontend/entrypoint.sh b/librephotos-docker/frontend/entrypoint.sh new file mode 100644 index 0000000..c7e9f90 --- /dev/null +++ b/librephotos-docker/frontend/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +echo "installing frontend" +npm install --legacy-peer-deps +npm run postinstall +npm run start + +# DANGEROUSLY_DISABLE_HOST_CHECK=true HOST=0.0.0.0 npm start diff --git a/librephotos-docker/k8s/README.md b/librephotos-docker/k8s/README.md new file mode 100644 index 0000000..936bec4 --- /dev/null +++ b/librephotos-docker/k8s/README.md @@ -0,0 +1,27 @@ +# Kubernetes Installation + +1. Clone this repo and change to this directory. +1. Check the values in `kustomization.yaml`, in the `images` section, to make sure they're pointing to the latest + release. +1. Consider changing the sizes of the volumes in `pvcs.yaml`. +1. Edit the hostnames in `ingress.yaml`. Consider installing [cert-manager](https://cert-manager.io/) and uncommenting + the relevant portions of `ingress.yaml`. +1. Edit the values in `config/backend.env` to suit your configuration. +1. Install these manifests to your cluster with `kubectl apply -k .`. +1. Create a secret for PostgreSQL authentication. + ``` + kubectl create secret generic database -n librephotos DB_PASS=$(openssl rand -hex 16) DB_USER=librephotos + ``` +1. Create a secret for the backend's key, admin password, and optional MapBox API key. + ``` + kubectl create secret generic backend -n librephotos SECRET_KEY=$key ADMIN_PASSWORD=$password MAPBOX_API_KEY=$apikey + ``` + Substitute values for `$key`, `$password`, and `$apikey`. Make sure you remember the `$password` so you can log in. + +If you want, you can watch the Pods get ready with `kubectl get pod -n librephotos -w`. Once they're all running, +point your browser at the hostname from `ingress.yaml`, and log in as `admin`. + +# Upgrading + +Change the values in `kustomization.yaml`, in the `images` section, to point to the latest versions. Then just rerun +the `kubectl apply -k .` command. diff --git a/librephotos-docker/k8s/backend.yaml b/librephotos-docker/k8s/backend.yaml new file mode 100644 index 0000000..11c0dae --- /dev/null +++ b/librephotos-docker/k8s/backend.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend +spec: + selector: + matchLabels: + app: backend + strategy: + rollingUpdate: + maxSurge: 0 + template: + metadata: + labels: + app: backend + spec: + containers: + - name: backend + image: backend-placeholder + envFrom: + - configMapRef: + name: backend + - secretRef: + name: backend + - secretRef: + name: database + securityContext: + readOnlyRootFilesystem: true + ports: + - name: backend + containerPort: 8001 + volumeMounts: + - name: photos + mountPath: /data + - name: protected-media + mountPath: /protected_media + - name: logs + mountPath: /logs + - name: cache + mountPath: /root/.cache + - name: backend-tmp + mountPath: /tmp + - name: proxy + image: proxy-placeholder + securityContext: + readOnlyRootFilesystem: true + ports: + - name: proxy + containerPort: 80 + volumeMounts: + - name: photos + mountPath: /data + - name: protected-media + mountPath: /protected_media + - name: nginx-cache + mountPath: /var/cache/nginx + - name: proxy-var-run + mountPath: /var/run + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + volumes: + - name: photos + persistentVolumeClaim: + claimName: photos + - name: protected-media + persistentVolumeClaim: + claimName: protected + - name: logs + emptyDir: {} + - name: cache + emptyDir: {} + - name: backend-tmp + emptyDir: {} + - name: nginx-cache + emptyDir: {} + - name: proxy-var-run + emptyDir: {} + +--- + +apiVersion: v1 +kind: Service +metadata: + name: backend +spec: + selector: + app: backend + ports: + - name: http + port: 8001 + targetPort: backend + +--- + +apiVersion: v1 +kind: Service +metadata: + name: proxy +spec: + selector: + app: backend + ports: + - name: http + port: 80 + targetPort: proxy diff --git a/librephotos-docker/k8s/config/backend.env b/librephotos-docker/k8s/config/backend.env new file mode 100644 index 0000000..cb31b81 --- /dev/null +++ b/librephotos-docker/k8s/config/backend.env @@ -0,0 +1,10 @@ +BACKEND_HOST=backend +ADMIN_EMAIL=somebody@example.com +ADMIN_USERNAME=admin +DB_BACKEND=postgresql +DB_NAME=librephotos +DB_HOST=postgres +DB_PORT=5432 +REDIS_HOST=redis +REDIS_PORT=6379 +WEB_CONCURRENCY=2 diff --git a/librephotos-docker/k8s/db.yaml b/librephotos-docker/k8s/db.yaml new file mode 100644 index 0000000..2a6e606 --- /dev/null +++ b/librephotos-docker/k8s/db.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + strategy: + rollingUpdate: + maxSurge: 0 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgresql-placeholder + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: DB_PASS + name: database + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + key: DB_USER + name: database + - name: POSTGRES_DB + value: librephotos + livenessProbe: + exec: + command: + - pg_isready + - -h + - localhost + - -U + - librephotos + ports: + - containerPort: 5432 + name: psql + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: data + securityContext: + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + volumes: + - name: data + persistentVolumeClaim: + claimName: postgres + +--- + +apiVersion: v1 +kind: Service +metadata: + name: postgres +spec: + ports: + - port: 5432 + name: psql + selector: + app: postgres diff --git a/librephotos-docker/k8s/frontend.yaml b/librephotos-docker/k8s/frontend.yaml new file mode 100644 index 0000000..c6ee189 --- /dev/null +++ b/librephotos-docker/k8s/frontend.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend +spec: + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + spec: + containers: + - name: frontend + image: frontend-placeholder + ports: + - name: http + containerPort: 3000 + securityContext: + readOnlyRootFilesystem: true + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: frontend +spec: + ports: + - port: 3000 + name: http + selector: + app: frontend diff --git a/librephotos-docker/k8s/ingress.yaml b/librephotos-docker/k8s/ingress.yaml new file mode 100644 index 0000000..cec7aba --- /dev/null +++ b/librephotos-docker/k8s/ingress.yaml @@ -0,0 +1,44 @@ +# This ingress will make your photo site available to web browsers. For this to work, you'll need an ingress controller +# already installed in your cluster: https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/ +# You should use TLS/SSL to protect the site. Either use cert-manager (https://cert-manager.io/) or something else. +# If TLS/SSL is available, uncomment the annotation below, and the tls section. + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: photos.example.com + # annotations: + # ingress.kubernetes.io/ssl-redirect: "true" +spec: + ingressClassName: nginx + rules: + - host: photos.example.com + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: proxy + port: + name: http + # tls: + # - hosts: + # - photos.example.com + # secretName: photos.example.com + +--- + +# If you're using cert-manager, uncomment this to request a certificate that will be used by the ingress above. + +# apiVersion: cert-manager.io/v1 +# kind: Certificate +# metadata: +# name: photos.example.com +# spec: +# secretName: photos.example.com +# dnsNames: +# - photos.example.com +# issuerRef: +# kind: ClusterIssuer +# name: letsencrypt diff --git a/librephotos-docker/k8s/kustomization.yaml b/librephotos-docker/k8s/kustomization.yaml new file mode 100644 index 0000000..00a706c --- /dev/null +++ b/librephotos-docker/k8s/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: librephotos + +resources: +- backend.yaml +- db.yaml +- frontend.yaml +- ingress.yaml +- ns.yaml +- pvcs.yaml +- redis.yaml + +configMapGenerator: +- name: backend + envs: + - config/backend.env + +images: +- name: backend-placeholder + newName: reallibrephotos/librephotos + newTag: 2022w12 +- name: frontend-placeholder + newName: reallibrephotos/librephotos-frontend + newTag: 2022w12 +- name: proxy-placeholder + newName: reallibrephotos/librephotos-proxy + newTag: 2022w10 +- name: postgresql-placeholder + newName: postgres + newTag: "13" +- name: redis-placeholder + newName: redis + newTag: "6" diff --git a/librephotos-docker/k8s/ns.yaml b/librephotos-docker/k8s/ns.yaml new file mode 100644 index 0000000..a187019 --- /dev/null +++ b/librephotos-docker/k8s/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: librephotos diff --git a/librephotos-docker/k8s/pvcs.yaml b/librephotos-docker/k8s/pvcs.yaml new file mode 100644 index 0000000..a281a94 --- /dev/null +++ b/librephotos-docker/k8s/pvcs.yaml @@ -0,0 +1,42 @@ +# This volume holds your photos. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: photos +spec: + resources: + requests: + storage: 30Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + +--- + +# This volume contains thumbnails. Setting it to 10-15% the size of "photos" is probably reasonable. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: protected +spec: + resources: + requests: + storage: 3Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + +--- + +# The postgres volume holds all the metadata. If it's 1% the size of "photos," you're probably fine. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres +spec: + resources: + requests: + storage: 1Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce diff --git a/librephotos-docker/k8s/redis.yaml b/librephotos-docker/k8s/redis.yaml new file mode 100644 index 0000000..dfcd4f5 --- /dev/null +++ b/librephotos-docker/k8s/redis.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis +spec: + selector: + matchLabels: + app: redis + strategy: + rollingUpdate: + maxSurge: 0 + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis-placeholder + ports: + - name: redis + containerPort: 6379 + securityContext: + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + ports: + - port: 6379 + name: redis + selector: + app: redis diff --git a/librephotos-docker/libredata/.gitignore b/librephotos-docker/libredata/.gitignore new file mode 100644 index 0000000..1e04ba0 --- /dev/null +++ b/librephotos-docker/libredata/.gitignore @@ -0,0 +1,8 @@ +# .gitignore sample +################### + +# Ignore all files in this dir... +* + +# ... except for this one. +!.gitignore diff --git a/librephotos-docker/librephotos.env b/librephotos-docker/librephotos.env new file mode 100644 index 0000000..95c28c8 --- /dev/null +++ b/librephotos-docker/librephotos.env @@ -0,0 +1,62 @@ +# This file contains all the things you need to change to set up your Libre Photos. +# There are a few items that must be set for it to work such as the location of your photos. +# After the mandatory entries there are some optional ones that you may set. + +# Start of mandatory changes. + +# Location of your photos. +scanDirectory=./librephotos/pictures + +# Internal data of LibrePhotos +data=./librephotos/data + +# ------------------------------------------------------------------------------------------------ + +# Wow, we are at the optional now. Pretty easy so far. You do not have to change any of the below. + +#What port should Libre Photos be accessed at (Default 3000) +httpPort=3000 + +# What branch should we install the latest weekly build or the development branch (dev) +tag=latest + +# Number of workers, which take care of the request to the api. This setting can dramatically affect the ram usage. +# A positive integer generally in the 2-4 x $(NUM_CORES) range. +# You’ll want to vary this a bit to find the best for your particular workload. +# Each worker needs 800MB of RAM. Change at your own will. Default is 2. +gunniWorkers=2 + +# You can set the database name. Did you know Libre Photos was forked from OwnPhotos? +dbName=librephotos + +# Here you can change the user name for the database. +dbUser=docker + +# The password used by the database. +dbPass=AaAa1234 + +# Default minimum rating to interpret as favorited. This default value is used when creating a new user. +# Users can change this in their settings (Dashboards > Library). +DEFAULT_FAVORITE_MIN_RATING=4 + +# Database host. Only change this if you want to use your own existing Postgres server. If using your own server, you can remove the 'db' container from docker-compose.yml. If you're changing the name of the DB's container name (DB_CONT_NAME further down), you need to set this variable to match that name too. +dbHost=db + +# Set the names of the docker containers to your own entries. Or don't, I'm not your dad. +# Changing these will require you to `make rename` to rename the services, and start the system with your chosen `docker-compose up -d` invocation again. +# Note that changing the DB_CONT_NAME will also need you to set the `dbHost` variable to the same value. +DB_CONT_NAME=db +BACKEND_CONT_NAME=backend +FRONTEND_CONT_NAME=frontend +PROXY_CONT_NAME=proxy +REDIS_CONT_NAME=redis +PGADMIN_CONT_NAME=pgadmin +# --------------------------------------------------------------------------------------------- + +# If you are not a developer ignore the following parameters: you will never need them. + +# Where shall we store the backend and frontend code files. +codedir=./librephotos/code + +# Location for pgAdmin +pgAdminLocation=./librephotos/pgadmin diff --git a/librephotos-docker/proxy/Dockerfile b/librephotos-docker/proxy/Dockerfile new file mode 100644 index 0000000..f850d38 --- /dev/null +++ b/librephotos-docker/proxy/Dockerfile @@ -0,0 +1,6 @@ +FROM nginx +ENV BACKEND_NAME=${BACKEND_NAME:-backend} +ENV FRONTEND_NAME=${FRONTEND_NAME:-frontend} +ENV VAR_PREFIX='$' +COPY nginx.tpl /etc/nginx/nginx.tpl +ENTRYPOINT envsubst < /etc/nginx/nginx.tpl > /etc/nginx/nginx.conf; nginx -g 'daemon off;' diff --git a/librephotos-docker/proxy/nginx.tpl b/librephotos-docker/proxy/nginx.tpl new file mode 100644 index 0000000..1d9775b --- /dev/null +++ b/librephotos-docker/proxy/nginx.tpl @@ -0,0 +1,62 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log debug; + +events { + worker_connections 1024; +} + +http { + server { + listen 80; + + location / { + # React routes are entirely on the App side in the web browser + # Always proxy to root with the same page request when nginx 404s + error_page 404 /; + proxy_intercept_errors on; + proxy_set_header Host ${VAR_PREFIX}host; + proxy_pass http://${FRONTEND_NAME}:3000/; + } + location ~ ^/(api|media)/ { + proxy_set_header X-Forwarded-Proto ${VAR_PREFIX}scheme; + proxy_set_header X-Real-IP ${VAR_PREFIX}remote_addr; + proxy_set_header Host ${BACKEND_NAME}; + include uwsgi_params; + proxy_pass http://${BACKEND_NAME}:8001; + } + # needed for webpack-dev-server + location /ws { + proxy_pass http://${FRONTEND_NAME}:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade ${VAR_PREFIX}http_upgrade; + proxy_set_header Connection "upgrade"; + } + # Django media + location /protected_media { + internal; + alias /protected_media/; + } + + location /static/drf-yasg { + proxy_pass http://${BACKEND_NAME}:8001; + } + + location /data { + internal; + alias /data/; + } + + # Original Photos + location /original { + internal; + alias /data/; + } + # Nextcloud Original Photos + location /nextcloud_original { + internal; + alias /data/nextcloud_media/; + } + } +} diff --git a/librephotos-docker/vscode/settings.json b/librephotos-docker/vscode/settings.json new file mode 100644 index 0000000..31f641c --- /dev/null +++ b/librephotos-docker/vscode/settings.json @@ -0,0 +1,23 @@ +{ + "files.exclude": { + "**/*.py[co]": true, + "**/*.so": true, + "**/__pycache__": true + }, + "python.pythonPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--exclude: .+/migrations/", + "--max-line-length=119" + ], + "python.linting.pylintArgs": [ + "--load-plugins=pylint_django", + "-d", + "E0239", + "-d", + "C0111" + ], + "python.linting.pylintEnabled": true, + "python.sortImports.path": "isort" +} diff --git a/netbox/config/.gitignore b/netbox/config/.gitignore new file mode 100644 index 0000000..1e04ba0 --- /dev/null +++ b/netbox/config/.gitignore @@ -0,0 +1,8 @@ +# .gitignore sample +################### + +# Ignore all files in this dir... +* + +# ... except for this one. +!.gitignore diff --git a/netbox/docker-compose.yml b/netbox/docker-compose.yml new file mode 100644 index 0000000..0b9b134 --- /dev/null +++ b/netbox/docker-compose.yml @@ -0,0 +1,35 @@ +--- +version: "2.1" +services: + netbox: + image: lscr.io/linuxserver/netbox:latest + container_name: netbox + environment: + - PUID=1000 + - PGID=1000 + - TZ= + - SUPERUSER_EMAIL= + - SUPERUSER_PASSWORD= + - ALLOWED_HOST= + - DB_NAME= + - DB_USER= + - DB_PASSWORD= + - DB_HOST= + - DB_PORT= + - REDIS_HOST= + - REDIS_PORT= + - REDIS_PASSWORD= + - REDIS_DB_TASK= + - REDIS_DB_CACHE= + - BASE_PATH= #optional + - REMOTE_AUTH_ENABLED= #optional + - REMOTE_AUTH_BACKEND= #optional + - REMOTE_AUTH_HEADER= #optional + - REMOTE_AUTH_AUTO_CREATE_USER= #optional + - REMOTE_AUTH_DEFAULT_GROUPS= #optional + - REMOTE_AUTH_DEFAULT_PERMISSIONS= #optional + volumes: + - :/config + ports: + - 8000:8000 + restart: unless-stopped \ No newline at end of file diff --git a/netbox/readme.md b/netbox/readme.md new file mode 100644 index 0000000..8c5f4e5 --- /dev/null +++ b/netbox/readme.md @@ -0,0 +1 @@ +https://github.com/netbox-community/netbox-docker/wiki/Getting-Started