diff --git a/.vscode/extensions.json b/.vscode/extensions.json index cdd9de7..674a595 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,11 +1,17 @@ { "recommendations": [ - "quodatum.ext1", "quodatum.vscode-basex", "ms-azuretools.vscode-docker", "dbaeumer.vscode-eslint", "eamodio.gitlens", "kelvin.vscode-sshfs", - "alefragnani.project-manager" + "alefragnani.project-manager", + "redhat.vscode-xml", + "deltaxml.xpath-notebook", + "maattdd.gitless", + "buenon.scratchpads", + "bierner.markdown-mermaid", + "mfederczuk.w3c-ebnf", + "quodatum.xquery-notebook" ] } \ No newline at end of file diff --git a/caddy/Caddyfile b/caddy/Caddyfile index 18da98c..9841e3f 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -7,7 +7,7 @@ } # Physical -(omv) { +(omv) { reverse_proxy 192.168.1.6:{args.0} } (n2) { @@ -87,10 +87,12 @@ quodatum.duckdns.org { import omv 8444 } - @warden host warden.quodatum.duckdns.org + @warden host warden.quodatum.duckdns.org:443 handle @warden { - import auth_quodatum - import omv 8882 + reverse_proxy /notifications/hub 0.0.0.0:3012 + reverse_proxy 0.0.0.0:8882 { + header_up X-Real-IP {remote_host} + } } @nextcloud host nextcloud.quodatum.duckdns.org @@ -179,3 +181,8 @@ xu4.local { root * links file_server browse } +warden.local:443 { + tls "bunce.andy@gmail.com" + reverse_proxy 0.0.0.0:8882 + reverse_proxy /notifications/hub 0.0.0.0:3012 +} \ No newline at end of file diff --git a/caddy/Dockerfile b/caddy/Dockerfile index 7732c18..ca5fc14 100644 --- a/caddy/Dockerfile +++ b/caddy/Dockerfile @@ -2,11 +2,11 @@ # @created 2022-03 # author="Andy Bunce" -ARG VERSION=caddy:2.6.2 +ARG VERSION=caddy:2.6.4 FROM $VERSION-builder AS builder RUN xcaddy build \ - --with github.com/caddy-dns/duckdns + --with github.com/caddy-dns/duckdns \ --with github.com/greenpau/caddy-security \ --with github.com/greenpau/caddy-trace \ --with github.com/greenpau/caddy-git diff --git a/caddy/docker-compose.yml b/caddy/docker-compose.yml index 473b56e..dde2e53 100644 --- a/caddy/docker-compose.yml +++ b/caddy/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: caddy: - image: caddy-saffron:1.0.4 + image: caddy-saffron:1.0.5 build: . container_name: caddy restart: unless-stopped diff --git a/code-bmy/Dockerfile b/code-bmy/Dockerfile index 4fbb597..322acbf 100644 --- a/code-bmy/Dockerfile +++ b/code-bmy/Dockerfile @@ -6,7 +6,7 @@ # hardcoded for amd64 # Main image -FROM ghcr.io/linuxserver/code-server:4.5.1 +FROM ghcr.io/linuxserver/code-server:latest ARG TARGETARCH ARG BUILDARCH @@ -36,14 +36,14 @@ RUN curl https://files.basex.org/releases/9.5.1/BaseX951.zip | jar xv && \ ENV PATH=$PATH:/basex/bin # install docker cli only -RUN curl https://download.docker.com/linux/static/stable/x86_64/docker-20.10.17.tgz | tar zxv -C /tmp/ && \ +RUN curl https://download.docker.com/linux/static/stable/x86_64/docker-23.0.1.tgz | tar zxv -C /tmp/ && \ cp /tmp/docker/docker /usr/local/bin && rm -r /tmp/docker; # install maven -RUN curl https://dlcdn.apache.org/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.tar.gz | \ +RUN curl https://dlcdn.apache.org/maven/maven-3/3.9.0/binaries/apache-maven-3.9.0-bin.tar.gz | \ tar zxv -C /usr/local/bin && \ cd /usr/local/bin && \ - ln -s apache-maven-3.8.6 maven + ln -s apache-maven-3.9.0 maven ENV PATH=$PATH:/usr/local/bin/maven/bin # install node+npm LTS see https://www.how2shout.com/linux/how-to-install-node-js-npm-on-debian-11-bullseye/ diff --git a/code-server4/Dockerfile b/code-server4/Dockerfile index d4ec9f5..9632fda 100644 --- a/code-server4/Dockerfile +++ b/code-server4/Dockerfile @@ -1,23 +1,23 @@ # -# add openjdk 11 and basex on code-server docker image +# add openjdk 17 and basex on code-server docker image # @author Andy Bunce FROM alpine:latest AS builder RUN apk --no-cache add zip -ADD https://files.basex.org/releases/9.5/BaseX95.zip /srv +ADD https://files.basex.org/releases/10.4/BaseX104.zip /srv RUN cd /srv && unzip *.zip && rm *.zip # custom options COPY .basex /srv/basex/ # Main image -FROM ghcr.io/linuxserver/code-server:version-v3.9.1 +FROM ghcr.io/linuxserver/code-server:latest LABEL author="Andy Bunce" LABEL company="Quodatum Ltd" LABEL maintainer="andy@quodatum.com" ENV JAVA_HOME="/usr/lib/jvm/default-jvm/" -RUN apk add --no-cache bash openjdk11-jre-headless +RUN apk add --no-cache bash openjdk17-jre-headless # Has to be set explictly to find binaries ENV PATH=$PATH:${JAVA_HOME}/bin diff --git a/code-serverx/Dockerfile b/code-serverx/Dockerfile index 499e717..2c8c1af 100644 --- a/code-serverx/Dockerfile +++ b/code-serverx/Dockerfile @@ -1,24 +1,27 @@ # A code-server image with added -# openjdk@11 -# BaseX installed@9.7.3 +# openjdk@17 +# BaseX installed@10.6 +# docker build --tag=vscode4.13-basex10.6 . # @author Andy Bunce # Bugs: # hardcoded for amd64 # Main image -FROM ghcr.io/linuxserver/code-server:4.5.1 +FROM ghcr.io/linuxserver/code-server:4.13.0 ARG TARGETARCH ARG BUILDARCH +ARG PUID=1000 +ARG PGID=1000 RUN echo "I'm building vscode-basex for ${BUILDARCH} on ${TARGETARCH} User: ${PUID}" LABEL author="Andy Bunce" LABEL company="Quodatum Ltd" LABEL maintainer="quodatum@gmail.com" -# Install OpenJDK-11 +# Install OpenJDK-17 RUN apt-get update && \ - apt-get install -y openjdk-11-jdk-headless; + apt-get install -y openjdk-17-jdk-headless; # Fix certificate issues RUN apt-get update && \ @@ -27,23 +30,33 @@ RUN apt-get update && \ update-ca-certificates -f; # Setup JAVA_HOME -- useful for docker commandline -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64/ +ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64/ RUN export JAVA_HOME # install BaseX to /basex/ -RUN curl https://files.basex.org/releases/9.7.3/BaseX973.zip | jar xv && \ +RUN curl https://files.basex.org/releases/10.6/BaseX106.zip | jar xv && \ chown -R $PUID:$PGID /basex && chmod a+x /basex/bin/* ; ENV PATH=$PATH:/basex/bin # install docker cli only -RUN curl https://download.docker.com/linux/static/stable/x86_64/docker-20.10.17.tgz | tar zxv -C /tmp/ && \ - cp /tmp/docker/docker /usr/local/bin && rm -r /tmp/docker; +# https://docs.docker.com/engine/install/ubuntu/ +RUN install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +RUN chmod a+r /etc/apt/keyrings/docker.gpg +RUN echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +RUN apt-get update && apt-get install -y docker-ce-cli + +#RUN curl https://download.docker.com/linux/static/stable/x86_64/docker-23.0.1.tgz | tar zxv -C /tmp/ && \ +# cp /tmp/docker/docker /usr/local/bin && rm -r /tmp/docker; # install maven -RUN curl https://dlcdn.apache.org/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.tar.gz | \ +RUN curl https://dlcdn.apache.org/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz | \ tar zxv -C /usr/local/bin && \ cd /usr/local/bin && \ - ln -s apache-maven-3.8.6 maven + ln -s apache-maven-3.9.2 maven ENV PATH=$PATH:/usr/local/bin/maven/bin # install node+npm LTS see https://www.how2shout.com/linux/how-to-install-node-js-npm-on-debian-11-bullseye/ diff --git a/code-serverx/config/.vscode/buenon.scratchpads-0.0.7.vsix b/code-serverx/config/.vscode/buenon.scratchpads-0.0.7.vsix deleted file mode 100644 index 3359a1c..0000000 Binary files a/code-serverx/config/.vscode/buenon.scratchpads-0.0.7.vsix and /dev/null differ diff --git a/code-serverx/config/.vscode/deltaxml.xslt-xpath-1.1.0.vsix b/code-serverx/config/.vscode/deltaxml.xslt-xpath-1.1.0.vsix deleted file mode 100644 index ca4d179..0000000 Binary files a/code-serverx/config/.vscode/deltaxml.xslt-xpath-1.1.0.vsix and /dev/null differ diff --git a/code-serverx/config/.vscode/extensions.json b/code-serverx/config/.vscode/extensions.json index 6084485..674a595 100644 --- a/code-serverx/config/.vscode/extensions.json +++ b/code-serverx/config/.vscode/extensions.json @@ -1,11 +1,17 @@ { "recommendations": [ + "quodatum.vscode-basex", "ms-azuretools.vscode-docker", "dbaeumer.vscode-eslint", "eamodio.gitlens", "kelvin.vscode-sshfs", "alefragnani.project-manager", - "johnpapa.vscode-peacock", - "nonoroazoro.syncing" + "redhat.vscode-xml", + "deltaxml.xpath-notebook", + "maattdd.gitless", + "buenon.scratchpads", + "bierner.markdown-mermaid", + "mfederczuk.w3c-ebnf", + "quodatum.xquery-notebook" ] } \ No newline at end of file diff --git a/code-serverx/config/.vscode/raffazizzi.sxml-0.2.2.vsix b/code-serverx/config/.vscode/raffazizzi.sxml-0.2.2.vsix deleted file mode 100644 index adee8d6..0000000 Binary files a/code-serverx/config/.vscode/raffazizzi.sxml-0.2.2.vsix and /dev/null differ diff --git a/code-serverx/config/.vscode/vscode-basex-0.0.47.vsix b/code-serverx/config/.vscode/vscode-basex-0.0.47.vsix deleted file mode 100644 index 90f56ff..0000000 Binary files a/code-serverx/config/.vscode/vscode-basex-0.0.47.vsix and /dev/null differ diff --git a/code-serverx/config/.vscode/w3c-ebnf-0.2.0.vsix b/code-serverx/config/.vscode/w3c-ebnf-0.2.0.vsix deleted file mode 100644 index 130e6aa..0000000 Binary files a/code-serverx/config/.vscode/w3c-ebnf-0.2.0.vsix and /dev/null differ diff --git a/home-gallery/data/.gitignore b/home-gallery/data/.gitignore new file mode 100644 index 0000000..1e04ba0 --- /dev/null +++ b/home-gallery/data/.gitignore @@ -0,0 +1,8 @@ +# .gitignore sample +################### + +# Ignore all files in this dir... +* + +# ... except for this one. +!.gitignore diff --git a/home-gallery/docker-compose.yml b/home-gallery/docker-compose.yml new file mode 100644 index 0000000..e57ec1f --- /dev/null +++ b/home-gallery/docker-compose.yml @@ -0,0 +1,32 @@ +version: "3.9" +services: + api: + # custom build via + #build: packages/api-server + image: xemle/home-gallery-api-server + environment: + # TensorflowJS backends + # - cpu: slowest and best support + # - wasm: good perfromance for arm64 and amd64 platforms + # - node: best performance on amd64 platform + #- BACKEND=cpu + - BACKEND=wasm + #- BACKEND=node + gallery: + # custom build via + #build: . + image: xemle/home-gallery + environment: + - GALLERY_API_SERVER=http://api:3000 + #- GALLERY_API_SERVER_CONCURRENT=1 # On low powered devices + #- GALLERY_API_SERVER_TIMEOUT=60 # On low powered devices + - GALLERY_OPEN_BROWSER=false + volumes: + - ./data:/data + # Mount your media directories below /data + - /srv/dev-disk-by-uuid-45e2e732-9e00-4a82-a7cc-9c743e033671/datastore/PicasaStarter/pictures/Pictures:/data/Pictures + ports: + - "4000:3000" + user: "${CURRENT_USER}" + entrypoint: ['node', '/app/gallery.js'] + command: ['run', 'server'] \ No newline at end of file diff --git a/home-gallery/readme.md b/home-gallery/readme.md new file mode 100644 index 0000000..fbcec4a --- /dev/null +++ b/home-gallery/readme.md @@ -0,0 +1,12 @@ +https://docs.home-gallery.org/index.html + +``` +mkdir -p data + +docker-compose run gallery run init --source /data/Pictures +docker-compose up -d + +docker-compose run gallery run import --initial +# or +docker-compose run gallery run import --update +``` \ No newline at end of file diff --git a/jellyfin/docker-compose.yml b/jellyfin/docker-compose.yml index a1ecf76..83670cd 100644 --- a/jellyfin/docker-compose.yml +++ b/jellyfin/docker-compose.yml @@ -4,6 +4,7 @@ services: image: jellyfin/jellyfin container_name: jellyfin user: 1000:100 + # https://github.com/jellyfin-archive/jellyfin-docs/blob/master/general/administration/hardware-acceleration.md#hardware-acceleration-on-docker-linux group_add: - 107 network_mode: "host" diff --git a/librephotos-docker/.devcontainer.json b/librephotos-docker/.devcontainer.json new file mode 100644 index 0000000..372f017 --- /dev/null +++ b/librephotos-docker/.devcontainer.json @@ -0,0 +1,14 @@ +// See https://aka.ms/vscode-remote/devcontainer.json for format details. +{ + "dockerComposeFile": ["docker-compose.yml", "docker-compose.dev.yml"], + "service": "backend", + "workspaceFolder": "/code/", + "extensions": ["ms-python.python", + "ms-vscode.atom-keybindings", + "mrorz.language-gettext", + "ms-python.vscode-pylance", + "batisteo.vscode-django", + "keno.uikit-3-snippets", + "dbaeumer.vscode-eslint", + "christian-kohler.npm-intellisense"] +} diff --git a/librephotos-docker/.dockerignore b/librephotos-docker/.dockerignore new file mode 100644 index 0000000..3e4e90f --- /dev/null +++ b/librephotos-docker/.dockerignore @@ -0,0 +1,7 @@ +.gitignore +Dockerfile +Makefile +README.md +screenshots/ +tags +media/ diff --git a/librephotos-docker/.env b/librephotos-docker/.env new file mode 100644 index 0000000..bf082ca --- /dev/null +++ b/librephotos-docker/.env @@ -0,0 +1,62 @@ +# This file contains all the things you need to change to set up your Libre Photos. +# There are a few items that must be set for it to work such as the location of your photos. +# After the mandatory entries there are some optional ones that you may set. + +# Start of mandatory changes. + +# Location of your photos. +scanDirectory=./librephotos + +# Internal data of LibrePhotos +data=./libredata + +# ------------------------------------------------------------------------------------------------ + +# Wow, we are at the optional now. Pretty easy so far. You do not have to change any of the below. + +#What port should Libre Photos be accessed at (Default 3000) +httpPort=3000 + +# What branch should we install the latest weekly build or the development branch (dev) +tag=latest + +# Number of workers, which take care of the request to the api. This setting can dramatically affect the ram usage. +# A positive integer generally in the 2-4 x $(NUM_CORES) range. +# You’ll want to vary this a bit to find the best for your particular workload. +# Each worker needs 800MB of RAM. Change at your own will. Default is 2. +gunniWorkers=2 + +# You can set the database name. Did you know Libre Photos was forked from OwnPhotos? +dbName=librephotos + +# Here you can change the user name for the database. +dbUser=docker + +# The password used by the database. +dbPass=AaAa1234 + +# Default minimum rating to interpret as favorited. This default value is used when creating a new user. +# Users can change this in their settings (Dashboards > Library). +DEFAULT_FAVORITE_MIN_RATING=4 + +# Database host. Only change this if you want to use your own existing Postgres server. If using your own server, you can remove the 'db' container from docker-compose.yml. If you're changing the name of the DB's container name (DB_CONT_NAME further down), you need to set this variable to match that name too. +dbHost=db + +# Set the names of the docker containers to your own entries. Or don't, I'm not your dad. +# Changing these will require you to `make rename` to rename the services, and start the system with your chosen `docker-compose up -d` invocation again. +# Note that changing the DB_CONT_NAME will also need you to set the `dbHost` variable to the same value. +DB_CONT_NAME=db +BACKEND_CONT_NAME=backend +FRONTEND_CONT_NAME=frontend +PROXY_CONT_NAME=proxy +REDIS_CONT_NAME=redis +PGADMIN_CONT_NAME=pgadmin +# --------------------------------------------------------------------------------------------- + +# If you are not a developer ignore the following parameters: you will never need them. + +# Where shall we store the backend and frontend code files. +codedir=./librephotos/code + +# Location for pgAdmin +pgAdminLocation=./librephotos/pgadmin diff --git a/librephotos-docker/.github/workflows/backend-base.yml b/librephotos-docker/.github/workflows/backend-base.yml new file mode 100644 index 0000000..eec1b89 --- /dev/null +++ b/librephotos-docker/.github/workflows/backend-base.yml @@ -0,0 +1,51 @@ +name: backend-base + +on: + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'backend/base/**' + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Running Tests" + + build: + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./backend/base + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-base:dev diff --git a/librephotos-docker/.github/workflows/backend-dependencies.yml b/librephotos-docker/.github/workflows/backend-dependencies.yml new file mode 100644 index 0000000..06a38c8 --- /dev/null +++ b/librephotos-docker/.github/workflows/backend-dependencies.yml @@ -0,0 +1,51 @@ +name: backend-dependencies + +on: + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'backend/dependencies/**' + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Running Tests" + + build: + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./backend/dependencies + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-dependencies:dev diff --git a/librephotos-docker/.github/workflows/backend.yml b/librephotos-docker/.github/workflows/backend.yml new file mode 100644 index 0000000..13cb7d1 --- /dev/null +++ b/librephotos-docker/.github/workflows/backend.yml @@ -0,0 +1,90 @@ +name: backend + +on: + release: + types: [created] + push: + branches: + - main + paths: + - 'backend/**' + repository_dispatch: + types: backend-commit-event + +env: + IMAGE_NAME: librephotos + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Running Tests" + + build: + name: Building dev build + # Ensure test job passes before pushing image. + if: github.event_name != 'release' + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./backend + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos:dev + + release-job: + name: Releasing + if: github.event_name == 'release' && github.event.action == 'created' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./backend + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos:${{ github.event.release.tag_name }},reallibrephotos/librephotos:latest diff --git a/librephotos-docker/.github/workflows/dlib-builder.yml b/librephotos-docker/.github/workflows/dlib-builder.yml new file mode 100644 index 0000000..4192373 --- /dev/null +++ b/librephotos-docker/.github/workflows/dlib-builder.yml @@ -0,0 +1,55 @@ +# This is a basic workflow to help you get started with Actions + +name: Build dlib dependency + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the main branch + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'dependencies/dlib/**' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./dependencies/dlib + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/dlib-builder:dev + upload: + # The type of runner that the job will run on + runs-on: reallibrephotos/dlib-builder:dev + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Upload a Build Artifact + uses: actions/upload-artifact@v2.3.1 + with: + name: dlib + path: /tmp/builds/dlib.zip diff --git a/librephotos-docker/.github/workflows/frontend.yml b/librephotos-docker/.github/workflows/frontend.yml new file mode 100644 index 0000000..ed5da9a --- /dev/null +++ b/librephotos-docker/.github/workflows/frontend.yml @@ -0,0 +1,99 @@ +name: frontend + +on: + release: + types: [created] + repository_dispatch: + types: frontend-commit-event + + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'frontend/**' + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: librephotos-frontend + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Tests" + + build: + if: github.event_name != 'release' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./frontend + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-frontend:dev + + release-job: + name: Releasing + if: github.event_name == 'release' && github.event.action == 'created' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./frontend + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-frontend:${{ github.event.release.tag_name }},reallibrephotos/librephotos-frontend:latest diff --git a/librephotos-docker/.github/workflows/proxy.yml b/librephotos-docker/.github/workflows/proxy.yml new file mode 100644 index 0000000..59ac48c --- /dev/null +++ b/librephotos-docker/.github/workflows/proxy.yml @@ -0,0 +1,88 @@ +name: proxy + +on: + release: + types: [created] + push: + # Publish `dev` as Docker `latest` image. + branches: + - main + paths: + - 'proxy/**' + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: echo "To-Do Add Tests" + + build: + if: github.event_name != 'release' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./proxy + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-proxy:dev + cache-from: type=gha + cache-to: type=gha,mode=max + + release-job: + name: Releasing + if: github.event_name == 'release' && github.event.action == 'created' + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./proxy + platforms: linux/amd64,linux/arm64 + push: true + tags: reallibrephotos/librephotos-proxy:${{ github.event.release.tag_name }},reallibrephotos/librephotos-proxy:latest diff --git a/librephotos-docker/.gitignore b/librephotos-docker/.gitignore new file mode 100644 index 0000000..ad56f6d --- /dev/null +++ b/librephotos-docker/.gitignore @@ -0,0 +1,172 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +#.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# visual studio +.vs/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# LibrePhotos +densecap/data/models/densecap/densecap-pretrained-vgg16.t7 +*/*.pkl +*/*/*.pkl +thumbnails +media +samplephotos +tags +api/places365/model/ +Conv2d.patch +Linear.patch +Sequential.patch +BatchNorm2d.patch +AvgPool2d.patch +ReLU.patch +run_docker.sh +logs/ +playground +api/im2txt/data/ +api/im2txt/models/ +api/im2txt/png/ +*.ipynb +api/im2txt/*.tar.gz +api/places365/*.tar.gz +*.db +media* +protected_media +librephotos +vscode/server-extensions +vscode/server-insiders-extensions diff --git a/librephotos-docker/LICENSE b/librephotos-docker/LICENSE new file mode 100644 index 0000000..dc92c51 --- /dev/null +++ b/librephotos-docker/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 LibrePhotos + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/librephotos-docker/Makefile b/librephotos-docker/Makefile new file mode 100644 index 0000000..4faaaa4 --- /dev/null +++ b/librephotos-docker/Makefile @@ -0,0 +1,22 @@ +.PHONY: default build rename + +include librephotos.env +REPLACE_NAMES=sed 's/__backend_name__/$(BACKEND_CONT_NAME)/g; s/__frontend_name__/$(FRONTEND_CONT_NAME)/g; s/__proxy_name__/$(PROXY_CONT_NAME)/g; s/__redis_name__/$(REDIS_CONT_NAME)/g; s/__db_name__/$(DB_CONT_NAME)/g; s/__pgadmin_name__/$(PGADMIN_CONT_NAME)/g; s/__network_name__/$(NETWORK_NAME)/g' + +default: build + +build: build-backend build-frontend build-proxy + +build-backend: + docker build -t reallibrephotos/librephotos:latest backend + +build-frontend: + docker build -t reallibrephotos/librephotos-frontend:latest frontend + +build-proxy: + docker build -t reallibrephotos/librephotos-proxy:latest proxy + +rename: + $(REPLACE_NAMES) docker-compose.raw > docker-compose.yml + $(REPLACE_NAMES) docker-compose.dev.raw > docker-compose.dev.yml + $(REPLACE_NAMES) docker-compose.e2e.raw > docker-compose.e2e.yml diff --git a/librephotos-docker/README.md b/librephotos-docker/README.md new file mode 100644 index 0000000..f22ca1f --- /dev/null +++ b/librephotos-docker/README.md @@ -0,0 +1,13 @@ +# For common users: +Follow the instructions [here](https://docs.librephotos.com/1/standard_install/) + +# For developers: +Follow the instructions [here](https://docs.librephotos.com/1/dev_install/) + +# To use your own container names: +Follow the normal instructions as per your chosen build, but after updating the `.env` file to choose your container names, run +``` sh +make rename +``` +Then you can resume following the normal instructions. + diff --git a/librephotos-docker/backend/Dockerfile b/librephotos-docker/backend/Dockerfile new file mode 100644 index 0000000..edf742e --- /dev/null +++ b/librephotos-docker/backend/Dockerfile @@ -0,0 +1,14 @@ +FROM reallibrephotos/librephotos-dependencies:dev +# actual project +ARG DEBUG +WORKDIR /code +RUN git clone --depth 1 https://github.com/LibrePhotos/librephotos . +RUN pip install --no-cache-dir -r requirements.txt +RUN if [ "$DEBUG" = 1 ] ; then \ + pip install setuptools==57.5.0; \ + pip install -r requirements.dev.txt; \ + fi +EXPOSE 8001 + +COPY entrypoint.sh /entrypoint.sh +CMD ["/entrypoint.sh"] diff --git a/librephotos-docker/backend/base/Dockerfile b/librephotos-docker/backend/base/Dockerfile new file mode 100644 index 0000000..26afeab --- /dev/null +++ b/librephotos-docker/backend/base/Dockerfile @@ -0,0 +1,93 @@ +FROM ubuntu:jammy +ARG TARGETPLATFORM +ENV DEBIAN_FRONTEND=noninteractive + +# Install python +RUN apt-get update \ + && apt-get install -y python3-pip python3-dev \ + && cd /usr/local/bin \ + && ln -s /usr/bin/python3 python \ + && pip3 install --upgrade pip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# system packages installation +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + bzip2 \ + curl \ + ffmpeg \ + git \ + libboost-all-dev \ + libcfitsio-dev \ + libexif-dev \ + libexpat-dev \ + libexpat1-dev \ + libgif-dev \ + libgl1-mesa-glx \ + libglib2.0-dev \ + libgsf-1-dev \ + libheif-dev \ + libimage-exiftool-perl \ + libimagequant-dev \ + libjpeg-dev \ + liblapack-dev \ + liblcms2-dev \ + libmagic1 \ + libopenblas-dev \ + libopenexr-dev \ + liborc-dev \ + libpng-dev \ + libpq-dev \ + libraw-dev \ + librsvg2-dev \ + libsm6 \ + libtiff5-dev \ + libtool \ + libtool-bin \ + libwebp-dev \ + libxrender-dev \ + pkg-config \ + rustc \ + libtinfo5 \ + swig \ + unzip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then pip3 install --no-cache-dir torch torchvision -f https://torch.kmtea.eu/whl/stable.html; else pip3 install --no-cache-dir torch torchvision; fi + +RUN pip3 install --no-cache-dir cmake==3.21.2 + +#Build and install libraw +WORKDIR /tmp/builds +RUN git clone https://github.com/LibRaw/LibRaw && \ + cd LibRaw && \ + git reset --hard 2a9a4de21ea7f5d15314da8ee5f27feebf239655 && \ + autoreconf --install && \ + ./configure && \ + make && \ + make install && \ + rm -rf /tmp/builds/* + +#Build and install imagemagick +WORKDIR /tmp/builds +ARG IMAGEMAGICK_VERSION=7.1.0-48 +RUN curl -SL https://imagemagick.org/archive/releases/ImageMagick-${IMAGEMAGICK_VERSION}.tar.xz | tar -xJf- && \ + cd ImageMagick-${IMAGEMAGICK_VERSION} && \ + ./configure --with-modules && \ + make install && \ + ldconfig /usr/local/lib && \ + rm -rf /tmp/builds/* + +# Build and install libvips +WORKDIR /tmp/builds +ARG VIPSVERSION=8.13.0 +RUN curl -SL https://github.com/libvips/libvips/releases/download/v${VIPSVERSION}/vips-${VIPSVERSION}.tar.gz | tar -xz \ + && cd vips-${VIPSVERSION} \ + && ./configure \ + && make V=0 \ + && make install \ + && ldconfig \ + && rm -rf /tmp/builds/* diff --git a/librephotos-docker/backend/dependencies/Dockerfile b/librephotos-docker/backend/dependencies/Dockerfile new file mode 100644 index 0000000..8e4ab59 --- /dev/null +++ b/librephotos-docker/backend/dependencies/Dockerfile @@ -0,0 +1,21 @@ +FROM reallibrephotos/librephotos-base:dev + +# Build and install dlib +# Compile it WITHOUT AVX and SSE4 instructions to ensure compatibility +WORKDIR /tmp/builds +RUN git clone --depth 1 --branch 'v19.24' https://github.com/davisking/dlib.git && \ + mkdir dlib/build && \ + cd dlib/build && \ + cmake .. -DDLIB_USE_CUDA=0 -DUSE_AVX_INSTRUCTIONS=0 -DLIB_NO_GUI_SUPPORT=0 && \ + cmake --build . && \ + cd /tmp/builds/dlib && \ + python3 setup.py install --no USE_AVX_INSTRUCTIONS --no DLIB_USE_CUDA --no USE_SSE4_INSTRUCTIONS && \ + rm -rf /tmp/builds/* + +# Download pretrained models +WORKDIR /data_models +RUN mkdir -p /root/.cache/torch/hub/checkpoints/ && \ + curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/places365.tar.gz | tar -zxC /data_models/ && \ + curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/im2txt.tar.gz | tar -zxC /data_models/ && \ + curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/clip-embeddings.tar.gz | tar -zxC /data_models/ && \ + curl -SL https://download.pytorch.org/models/resnet152-b121ed2d.pth -o /root/.cache/torch/hub/checkpoints/resnet152-b121ed2d.pth diff --git a/librephotos-docker/backend/entrypoint.sh b/librephotos-docker/backend/entrypoint.sh new file mode 100644 index 0000000..d6a4bd4 --- /dev/null +++ b/librephotos-docker/backend/entrypoint.sh @@ -0,0 +1,27 @@ +#! /bin/bash +export PYTHONUNBUFFERED=TRUE +export PYTHONFAULTHANDLER=1 +mkdir -p /logs +python image_similarity/main.py 2>&1 | tee /logs/gunicorn_image_similarity.log & +python manage.py showmigrations | tee /logs/show_migrate.log +python manage.py migrate | tee /logs/command_migrate.log +python manage.py showmigrations | tee /logs/show_migrate.log +python manage.py clear_cache + +if [ -n "$ADMIN_USERNAME" ] +then + python manage.py createadmin -u $ADMIN_USERNAME $ADMIN_EMAIL 2>&1 | tee /logs/command_createadmin.log +fi + +echo "Running backend server..." + +python manage.py rqworker default 2>&1 | tee /logs/rqworker.log & + +if [ "$DEBUG" = 1 ] +then + echo "development backend starting" + gunicorn --worker-class=gevent --reload --bind 0.0.0.0:8001 --log-level=info ownphotos.wsgi 2>&1 | tee /logs/gunicorn_django.log +else + echo "production backend starting" + gunicorn --worker-class=gevent --bind 0.0.0.0:8001 --log-level=info ownphotos.wsgi 2>&1 | tee /logs/gunicorn_django.log +fi diff --git a/librephotos-docker/dependencies/dlib/Dockerfile b/librephotos-docker/dependencies/dlib/Dockerfile new file mode 100644 index 0000000..1ed8c41 --- /dev/null +++ b/librephotos-docker/dependencies/dlib/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:hirsute +ARG TARGETPLATFORM +ENV DEBIAN_FRONTEND=noninteractive + +# Install python +RUN apt-get update \ + && apt-get install -y python3-pip python3-dev \ + && cd /usr/local/bin \ + && ln -s /usr/bin/python3 python \ + && pip3 install --upgrade pip + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + zip \ + git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install cmake==3.21.2 + +# Build and install dlib +WORKDIR /tmp/builds +RUN git clone --depth 1 --branch 'v19.24' https://github.com/davisking/dlib.git && \ + mkdir dlib/build && \ + cd dlib/build && \ + cmake .. -DDLIB_USE_CUDA=0 -DUSE_AVX_INSTRUCTIONS=0 -DLIB_NO_GUI_SUPPORT=0 && \ + cmake --build . && \ + zip -r dlib.zip dlib \ diff --git a/librephotos-docker/docker-compose.dev.raw b/librephotos-docker/docker-compose.dev.raw new file mode 100644 index 0000000..cb416fd --- /dev/null +++ b/librephotos-docker/docker-compose.dev.raw @@ -0,0 +1,73 @@ +# Run options: +# 1. There are no options - This add additional tools to aid in the development of Libre Photos +# run cmd: docker-compose up -f docker-compose.yml -f docker-compose.dev.yml -d +# 2. Current added tools: +# pgadmin User admin@admin pass admin port 3001 + +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: '3.8' +services: + + __proxy_name__: + tty: true + build: + context: ./proxy + dockerfile: Dockerfile + container_name: __proxy_name__ + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + + __frontend_name__: + tty: true + environment: + - DEBUG=1 + - WDS_SOCKET_PORT=0 # needed for webpack-dev-server + build: + context: ./frontend + dockerfile: Dockerfile.dev + container_name: __frontend_name__ + volumes: + - ${codedir}/librephotos-frontend:/usr/src/app + + __backend_name__: + tty: true + stdin_open: true + environment: + - DEBUG=1 + build: + context: ./backend + dockerfile: Dockerfile + args: + DEBUG: 1 + container_name: __backend_name__ + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + - ${data}/logs:/logs + - ${data}/cache:/root/.cache + - ${codedir}/librephotos:/code + - ./vscode/server-extensions:/root/.vscode-server/extensions + - ./vscode/server-insiders-extensions:/root/.vscode-server-insiders/extensions + - ./vscode/settings.json:/code/.vscode/settings.json + - ./backend/entrypoint.sh:/entrypoint.sh + + __pgadmin_name__: + image: dpage/pgadmin4 + container_name: __pgadmin_name__ + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@admin.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} + volumes: + - ${pgAdminLocation}/pgadmin:/root/.pgadmin + ports: + - "3001:80" + restart: unless-stopped diff --git a/librephotos-docker/docker-compose.dev.yml b/librephotos-docker/docker-compose.dev.yml new file mode 100644 index 0000000..41e3bbb --- /dev/null +++ b/librephotos-docker/docker-compose.dev.yml @@ -0,0 +1,73 @@ +# Run options: +# 1. There are no options - This add additional tools to aid in the development of Libre Photos +# run cmd: docker-compose up -f docker-compose.yml -f docker-compose.dev.yml -d +# 2. Current added tools: +# pgadmin User admin@admin pass admin port 3001 + +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: '3.8' +services: + + proxy: + tty: true + build: + context: ./proxy + dockerfile: Dockerfile + container_name: proxy + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + + frontend: + tty: true + environment: + - DEBUG=1 + - WDS_SOCKET_PORT=0 # needed for webpack-dev-server + build: + context: ./frontend + dockerfile: Dockerfile.dev + container_name: frontend + volumes: + - ${codedir}/librephotos-frontend:/usr/src/app + + backend: + tty: true + stdin_open: true + environment: + - DEBUG=1 + build: + context: ./backend + dockerfile: Dockerfile + args: + DEBUG: 1 + container_name: backend + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + - ${data}/logs:/logs + - ${data}/cache:/root/.cache + - ${codedir}/librephotos:/code + - ./vscode/server-extensions:/root/.vscode-server/extensions + - ./vscode/server-insiders-extensions:/root/.vscode-server-insiders/extensions + - ./vscode/settings.json:/code/.vscode/settings.json + - ./backend/entrypoint.sh:/entrypoint.sh + + pgadmin: + image: dpage/pgadmin4 + container_name: pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@admin.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} + volumes: + - ${pgAdminLocation}/pgadmin:/root/.pgadmin + ports: + - "3001:80" + restart: unless-stopped diff --git a/librephotos-docker/docker-compose.e2e.raw b/librephotos-docker/docker-compose.e2e.raw new file mode 100644 index 0000000..82da04e --- /dev/null +++ b/librephotos-docker/docker-compose.e2e.raw @@ -0,0 +1,137 @@ +networks: + e2e: + +volumes: + e2e_db: + e2e_cache: + e2e_logs: + e2e_protected_media: + e2e_scan_directory: + +services: + e2e-__proxy_name__: + image: reallibrephotos/librephotos-proxy:${tag} + container_name: e2e-__proxy_name__ + restart: unless-stopped + environment: + BACKEND_NAME: e2e-__backend_name__ + FRONTEND_NAME: e2e-__frontend_name__ + volumes: + - e2e_scan_directory:/data + - e2e_protected_media:/protected_media + ports: + - 8080:80 + healthcheck: + test: curl -sI localhost | grep HTTP | grep 200 + interval: 5s + timeout: 5s + retries: 12 + depends_on: + e2e-__backend_name__: + condition: service_healthy + e2e-__frontend_name__: + condition: service_healthy + networks: + - e2e + + e2e-__db_name__: + image: postgres:13 + container_name: e2e-__db_name__ + restart: unless-stopped + environment: + - POSTGRES_USER=${dbUser} + - POSTGRES_PASSWORD=${dbPass} + - POSTGRES_DB=${dbName} + volumes: + - e2e_db:/var/lib/postgresql/data + healthcheck: + test: psql -U ${dbUser} -d ${dbName} -c "SELECT 1;" + interval: 5s + timeout: 5s + retries: 12 + networks: + - e2e + + e2e-__frontend_name__: + container_name: e2e-__frontend_name__ + environment: + - DEBUG=1 + - WDS_SOCKET_PORT=0 # needed for webpack-dev-server + tty: true + build: + context: ./frontend + dockerfile: Dockerfile.dev + restart: unless-stopped + volumes: + - ${codedir}/librephotos-frontend:/usr/src/app + healthcheck: + test: curl -sI localhost:3000 | grep HTTP | grep 200 + interval: 5s + timeout: 5s + retries: 24 + depends_on: + e2e-__backend_name__: + condition: service_healthy + networks: + - e2e + + e2e-__backend_name__: + container_name: e2e-__backend_name__ + tty: true + stdin_open: true + build: + context: ./backend + dockerfile: Dockerfile + args: + DEBUG: 1 + restart: unless-stopped + volumes: + - ${codedir}/librephotos:/code + - e2e_scan_directory:/data + - e2e_protected_media:/protected_media + - e2e_logs:/logs + - e2e_cache:/root/.cache + environment: + - SECRET_KEY=${shhhhKey:-} + - BACKEND_HOST=e2e-__backend_name__ + - ADMIN_EMAIL=${adminEmail:-admin@localhost} + - ADMIN_USERNAME=${userName:-admin} + - ADMIN_PASSWORD=${userPass:-admin} + - DB_BACKEND=postgresql + - DB_NAME=${dbName} + - DB_USER=${dbUser} + - DB_PASS=${dbPass} + - DB_HOST=e2e-__db_name__ + - DB_PORT=5432 + - REDIS_HOST=e2e-__redis_name__ + - REDIS_PORT=6379 + - MAPBOX_API_KEY=${mapApiKey:-} + - WEB_CONCURRENCY=${gunniWorkers:-1} + - SKIP_PATTERNS=${skipPatterns:-} + - ALLOW_UPLOAD=${allowUpload:-true} + - DEBUG=1 + - HEAVYWEIGHT_PROCESS=${HEAVYWEIGHT_PROCESS:-1} + healthcheck: + test: curl -sI localhost:8001 | grep HTTP | grep 401 + interval: 5s + timeout: 5s + retries: 24 + depends_on: + e2e-__db_name__: + condition: service_healthy + e2e-__redis_name__: + condition: service_healthy + networks: + - e2e + + e2e-__redis_name__: + image: redis:6 + container_name: e2e-__redis_name__ + restart: unless-stopped + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 5s + timeout: 5s + retries: 12 + networks: + - e2e diff --git a/librephotos-docker/docker-compose.e2e.yml b/librephotos-docker/docker-compose.e2e.yml new file mode 100644 index 0000000..f54230d --- /dev/null +++ b/librephotos-docker/docker-compose.e2e.yml @@ -0,0 +1,137 @@ +networks: + e2e: + +volumes: + e2e_db: + e2e_cache: + e2e_logs: + e2e_protected_media: + e2e_scan_directory: + +services: + e2e-proxy: + image: reallibrephotos/librephotos-proxy:${tag} + container_name: e2e-proxy + restart: unless-stopped + environment: + BACKEND_NAME: e2e-backend + FRONTEND_NAME: e2e-frontend + volumes: + - e2e_scan_directory:/data + - e2e_protected_media:/protected_media + ports: + - 8080:80 + healthcheck: + test: curl -sI localhost | grep HTTP | grep 200 + interval: 5s + timeout: 5s + retries: 12 + depends_on: + e2e-backend: + condition: service_healthy + e2e-frontend: + condition: service_healthy + networks: + - e2e + + e2e-db: + image: postgres:13 + container_name: e2e-db + restart: unless-stopped + environment: + - POSTGRES_USER=${dbUser} + - POSTGRES_PASSWORD=${dbPass} + - POSTGRES_DB=${dbName} + volumes: + - e2e_db:/var/lib/postgresql/data + healthcheck: + test: psql -U ${dbUser} -d ${dbName} -c "SELECT 1;" + interval: 5s + timeout: 5s + retries: 12 + networks: + - e2e + + e2e-frontend: + container_name: e2e-frontend + environment: + - DEBUG=1 + - WDS_SOCKET_PORT=0 # needed for webpack-dev-server + tty: true + build: + context: ./frontend + dockerfile: Dockerfile.dev + restart: unless-stopped + volumes: + - ${codedir}/librephotos-frontend:/usr/src/app + healthcheck: + test: curl -sI localhost:3000 | grep HTTP | grep 200 + interval: 5s + timeout: 5s + retries: 24 + depends_on: + e2e-backend: + condition: service_healthy + networks: + - e2e + + e2e-backend: + container_name: e2e-backend + tty: true + stdin_open: true + build: + context: ./backend + dockerfile: Dockerfile + args: + DEBUG: 1 + restart: unless-stopped + volumes: + - ${codedir}/librephotos:/code + - e2e_scan_directory:/data + - e2e_protected_media:/protected_media + - e2e_logs:/logs + - e2e_cache:/root/.cache + environment: + - SECRET_KEY=${shhhhKey:-} + - BACKEND_HOST=e2e-backend + - ADMIN_EMAIL=${adminEmail:-admin@localhost} + - ADMIN_USERNAME=${userName:-admin} + - ADMIN_PASSWORD=${userPass:-admin} + - DB_BACKEND=postgresql + - DB_NAME=${dbName} + - DB_USER=${dbUser} + - DB_PASS=${dbPass} + - DB_HOST=e2e-db + - DB_PORT=5432 + - REDIS_HOST=e2e-redis + - REDIS_PORT=6379 + - MAPBOX_API_KEY=${mapApiKey:-} + - WEB_CONCURRENCY=${gunniWorkers:-1} + - SKIP_PATTERNS=${skipPatterns:-} + - ALLOW_UPLOAD=${allowUpload:-true} + - DEBUG=1 + - HEAVYWEIGHT_PROCESS=${HEAVYWEIGHT_PROCESS:-1} + healthcheck: + test: curl -sI localhost:8001 | grep HTTP | grep 401 + interval: 5s + timeout: 5s + retries: 24 + depends_on: + e2e-db: + condition: service_healthy + e2e-redis: + condition: service_healthy + networks: + - e2e + + e2e-redis: + image: redis:6 + container_name: e2e-redis + restart: unless-stopped + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 5s + timeout: 5s + retries: 12 + networks: + - e2e diff --git a/librephotos-docker/docker-compose.raw b/librephotos-docker/docker-compose.raw new file mode 100644 index 0000000..27ad718 --- /dev/null +++ b/librephotos-docker/docker-compose.raw @@ -0,0 +1,90 @@ +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: "3.8" +services: + __proxy_name__: + image: reallibrephotos/librephotos-proxy:${tag} + container_name: __proxy_name__ + restart: unless-stopped + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + ports: + - ${httpPort}:80 + depends_on: + - __backend_name__ + - __frontend_name__ + + __db_name__: + image: postgres:13 + container_name: __db_name__ + restart: unless-stopped + environment: + - POSTGRES_USER=${dbUser} + - POSTGRES_PASSWORD=${dbPass} + - POSTGRES_DB=${dbName} + volumes: + - ${data}/db:/var/lib/postgresql/data + command: postgres -c fsync=off -c synchronous_commit=off -c full_page_writes=off -c random_page_cost=1.0 + healthcheck: + test: psql -U ${dbUser} -d ${dbName} -c "SELECT 1;" + interval: 5s + timeout: 5s + retries: 5 + + __frontend_name__: + image: reallibrephotos/librephotos-frontend:${tag} + container_name: __frontend_name__ + restart: unless-stopped + + __backend_name__: + image: reallibrephotos/librephotos:${tag} + container_name: __backend_name__ + restart: unless-stopped + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + - ${data}/logs:/logs + - ${data}/cache:/root/.cache + environment: + - SECRET_KEY=${shhhhKey:-} + - BACKEND_HOST=__backend_name__ + - ADMIN_EMAIL=${adminEmail:-} + - ADMIN_USERNAME=${userName:-} + - ADMIN_PASSWORD=${userPass:-} + - DB_BACKEND=postgresql + - DB_NAME=${dbName} + - DB_USER=${dbUser} + - DB_PASS=${dbPass} + - DB_HOST=${dbHost} + - DB_PORT=5432 + - REDIS_HOST=__redis_name__ + - REDIS_PORT=6379 + - MAPBOX_API_KEY=${mapApiKey:-} + - WEB_CONCURRENCY=${gunniWorkers:-1} + - SKIP_PATTERNS=${skipPatterns:-} + - ALLOW_UPLOAD=${allowUpload:-false} + - DEBUG=0 + - HEAVYWEIGHT_PROCESS=${HEAVYWEIGHT_PROCESS:-} + depends_on: + __db_name__: + condition: service_healthy + __redis_name__: + condition: service_healthy + + __redis_name__: + image: redis:6 + container_name: __redis_name__ + restart: unless-stopped + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 5s + timeout: 5s + retries: 12 diff --git a/librephotos-docker/docker-compose.yml b/librephotos-docker/docker-compose.yml new file mode 100644 index 0000000..9d910a4 --- /dev/null +++ b/librephotos-docker/docker-compose.yml @@ -0,0 +1,90 @@ +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: "3.8" +services: + proxy: + image: reallibrephotos/librephotos-proxy:${tag} + container_name: proxy + restart: unless-stopped + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + ports: + - ${httpPort}:80 + depends_on: + - backend + - frontend + + db: + image: postgres:13 + container_name: db + restart: unless-stopped + environment: + - POSTGRES_USER=${dbUser} + - POSTGRES_PASSWORD=${dbPass} + - POSTGRES_DB=${dbName} + volumes: + - ${data}/db:/var/lib/postgresql/data + command: postgres -c fsync=off -c synchronous_commit=off -c full_page_writes=off -c random_page_cost=1.0 + healthcheck: + test: psql -U ${dbUser} -d ${dbName} -c "SELECT 1;" + interval: 5s + timeout: 5s + retries: 5 + + frontend: + image: reallibrephotos/librephotos-frontend:${tag} + container_name: frontend + restart: unless-stopped + + backend: + image: reallibrephotos/librephotos:${tag} + container_name: backend + restart: unless-stopped + volumes: + - ${scanDirectory}:/data + - ${data}/protected_media:/protected_media + - ${data}/logs:/logs + - ${data}/cache:/root/.cache + environment: + - SECRET_KEY=${shhhhKey:-} + - BACKEND_HOST=backend + - ADMIN_EMAIL=${adminEmail:-} + - ADMIN_USERNAME=${userName:-} + - ADMIN_PASSWORD=${userPass:-} + - DB_BACKEND=postgresql + - DB_NAME=${dbName} + - DB_USER=${dbUser} + - DB_PASS=${dbPass} + - DB_HOST=${dbHost} + - DB_PORT=5432 + - REDIS_HOST=redis + - REDIS_PORT=6379 + - MAPBOX_API_KEY=${mapApiKey:-} + - WEB_CONCURRENCY=${gunniWorkers:-1} + - SKIP_PATTERNS=${skipPatterns:-} + - ALLOW_UPLOAD=${allowUpload:-false} + - DEBUG=0 + - HEAVYWEIGHT_PROCESS=${HEAVYWEIGHT_PROCESS:-} + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + + redis: + image: redis:6 + container_name: redis + restart: unless-stopped + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 5s + timeout: 5s + retries: 12 diff --git a/librephotos-docker/frontend/Dockerfile b/librephotos-docker/frontend/Dockerfile new file mode 100644 index 0000000..3a797ed --- /dev/null +++ b/librephotos-docker/frontend/Dockerfile @@ -0,0 +1,20 @@ +FROM node:13-slim as builder + +RUN apt-get update && apt-get install -y git + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app +ENV CLI_WIDTH 80 +RUN git clone https://github.com/LibrePhotos/librephotos-frontend /usr/src/app +RUN npm install --legacy-peer-deps +RUN npm run postinstall +RUN npm run build + +FROM halverneus/static-file-server +ENV PORT 3000 +EXPOSE 3000 + +COPY --from=builder /usr/src/app/build /web + +ENTRYPOINT ["/serve"] +CMD [] \ No newline at end of file diff --git a/librephotos-docker/frontend/Dockerfile.dev b/librephotos-docker/frontend/Dockerfile.dev new file mode 100644 index 0000000..200d07e --- /dev/null +++ b/librephotos-docker/frontend/Dockerfile.dev @@ -0,0 +1,8 @@ +FROM node:13-slim + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app +RUN apt-get update && apt-get install git curl -y +EXPOSE 3000 +COPY entrypoint.sh /entrypoint.sh +CMD ["/entrypoint.sh"] diff --git a/librephotos-docker/frontend/entrypoint.sh b/librephotos-docker/frontend/entrypoint.sh new file mode 100644 index 0000000..c7e9f90 --- /dev/null +++ b/librephotos-docker/frontend/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +echo "installing frontend" +npm install --legacy-peer-deps +npm run postinstall +npm run start + +# DANGEROUSLY_DISABLE_HOST_CHECK=true HOST=0.0.0.0 npm start diff --git a/librephotos-docker/k8s/README.md b/librephotos-docker/k8s/README.md new file mode 100644 index 0000000..936bec4 --- /dev/null +++ b/librephotos-docker/k8s/README.md @@ -0,0 +1,27 @@ +# Kubernetes Installation + +1. Clone this repo and change to this directory. +1. Check the values in `kustomization.yaml`, in the `images` section, to make sure they're pointing to the latest + release. +1. Consider changing the sizes of the volumes in `pvcs.yaml`. +1. Edit the hostnames in `ingress.yaml`. Consider installing [cert-manager](https://cert-manager.io/) and uncommenting + the relevant portions of `ingress.yaml`. +1. Edit the values in `config/backend.env` to suit your configuration. +1. Install these manifests to your cluster with `kubectl apply -k .`. +1. Create a secret for PostgreSQL authentication. + ``` + kubectl create secret generic database -n librephotos DB_PASS=$(openssl rand -hex 16) DB_USER=librephotos + ``` +1. Create a secret for the backend's key, admin password, and optional MapBox API key. + ``` + kubectl create secret generic backend -n librephotos SECRET_KEY=$key ADMIN_PASSWORD=$password MAPBOX_API_KEY=$apikey + ``` + Substitute values for `$key`, `$password`, and `$apikey`. Make sure you remember the `$password` so you can log in. + +If you want, you can watch the Pods get ready with `kubectl get pod -n librephotos -w`. Once they're all running, +point your browser at the hostname from `ingress.yaml`, and log in as `admin`. + +# Upgrading + +Change the values in `kustomization.yaml`, in the `images` section, to point to the latest versions. Then just rerun +the `kubectl apply -k .` command. diff --git a/librephotos-docker/k8s/backend.yaml b/librephotos-docker/k8s/backend.yaml new file mode 100644 index 0000000..11c0dae --- /dev/null +++ b/librephotos-docker/k8s/backend.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend +spec: + selector: + matchLabels: + app: backend + strategy: + rollingUpdate: + maxSurge: 0 + template: + metadata: + labels: + app: backend + spec: + containers: + - name: backend + image: backend-placeholder + envFrom: + - configMapRef: + name: backend + - secretRef: + name: backend + - secretRef: + name: database + securityContext: + readOnlyRootFilesystem: true + ports: + - name: backend + containerPort: 8001 + volumeMounts: + - name: photos + mountPath: /data + - name: protected-media + mountPath: /protected_media + - name: logs + mountPath: /logs + - name: cache + mountPath: /root/.cache + - name: backend-tmp + mountPath: /tmp + - name: proxy + image: proxy-placeholder + securityContext: + readOnlyRootFilesystem: true + ports: + - name: proxy + containerPort: 80 + volumeMounts: + - name: photos + mountPath: /data + - name: protected-media + mountPath: /protected_media + - name: nginx-cache + mountPath: /var/cache/nginx + - name: proxy-var-run + mountPath: /var/run + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + volumes: + - name: photos + persistentVolumeClaim: + claimName: photos + - name: protected-media + persistentVolumeClaim: + claimName: protected + - name: logs + emptyDir: {} + - name: cache + emptyDir: {} + - name: backend-tmp + emptyDir: {} + - name: nginx-cache + emptyDir: {} + - name: proxy-var-run + emptyDir: {} + +--- + +apiVersion: v1 +kind: Service +metadata: + name: backend +spec: + selector: + app: backend + ports: + - name: http + port: 8001 + targetPort: backend + +--- + +apiVersion: v1 +kind: Service +metadata: + name: proxy +spec: + selector: + app: backend + ports: + - name: http + port: 80 + targetPort: proxy diff --git a/librephotos-docker/k8s/config/backend.env b/librephotos-docker/k8s/config/backend.env new file mode 100644 index 0000000..cb31b81 --- /dev/null +++ b/librephotos-docker/k8s/config/backend.env @@ -0,0 +1,10 @@ +BACKEND_HOST=backend +ADMIN_EMAIL=somebody@example.com +ADMIN_USERNAME=admin +DB_BACKEND=postgresql +DB_NAME=librephotos +DB_HOST=postgres +DB_PORT=5432 +REDIS_HOST=redis +REDIS_PORT=6379 +WEB_CONCURRENCY=2 diff --git a/librephotos-docker/k8s/db.yaml b/librephotos-docker/k8s/db.yaml new file mode 100644 index 0000000..2a6e606 --- /dev/null +++ b/librephotos-docker/k8s/db.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + strategy: + rollingUpdate: + maxSurge: 0 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgresql-placeholder + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: DB_PASS + name: database + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + key: DB_USER + name: database + - name: POSTGRES_DB + value: librephotos + livenessProbe: + exec: + command: + - pg_isready + - -h + - localhost + - -U + - librephotos + ports: + - containerPort: 5432 + name: psql + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: data + securityContext: + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + volumes: + - name: data + persistentVolumeClaim: + claimName: postgres + +--- + +apiVersion: v1 +kind: Service +metadata: + name: postgres +spec: + ports: + - port: 5432 + name: psql + selector: + app: postgres diff --git a/librephotos-docker/k8s/frontend.yaml b/librephotos-docker/k8s/frontend.yaml new file mode 100644 index 0000000..c6ee189 --- /dev/null +++ b/librephotos-docker/k8s/frontend.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend +spec: + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + spec: + containers: + - name: frontend + image: frontend-placeholder + ports: + - name: http + containerPort: 3000 + securityContext: + readOnlyRootFilesystem: true + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: frontend +spec: + ports: + - port: 3000 + name: http + selector: + app: frontend diff --git a/librephotos-docker/k8s/ingress.yaml b/librephotos-docker/k8s/ingress.yaml new file mode 100644 index 0000000..cec7aba --- /dev/null +++ b/librephotos-docker/k8s/ingress.yaml @@ -0,0 +1,44 @@ +# This ingress will make your photo site available to web browsers. For this to work, you'll need an ingress controller +# already installed in your cluster: https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/ +# You should use TLS/SSL to protect the site. Either use cert-manager (https://cert-manager.io/) or something else. +# If TLS/SSL is available, uncomment the annotation below, and the tls section. + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: photos.example.com + # annotations: + # ingress.kubernetes.io/ssl-redirect: "true" +spec: + ingressClassName: nginx + rules: + - host: photos.example.com + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: proxy + port: + name: http + # tls: + # - hosts: + # - photos.example.com + # secretName: photos.example.com + +--- + +# If you're using cert-manager, uncomment this to request a certificate that will be used by the ingress above. + +# apiVersion: cert-manager.io/v1 +# kind: Certificate +# metadata: +# name: photos.example.com +# spec: +# secretName: photos.example.com +# dnsNames: +# - photos.example.com +# issuerRef: +# kind: ClusterIssuer +# name: letsencrypt diff --git a/librephotos-docker/k8s/kustomization.yaml b/librephotos-docker/k8s/kustomization.yaml new file mode 100644 index 0000000..00a706c --- /dev/null +++ b/librephotos-docker/k8s/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: librephotos + +resources: +- backend.yaml +- db.yaml +- frontend.yaml +- ingress.yaml +- ns.yaml +- pvcs.yaml +- redis.yaml + +configMapGenerator: +- name: backend + envs: + - config/backend.env + +images: +- name: backend-placeholder + newName: reallibrephotos/librephotos + newTag: 2022w12 +- name: frontend-placeholder + newName: reallibrephotos/librephotos-frontend + newTag: 2022w12 +- name: proxy-placeholder + newName: reallibrephotos/librephotos-proxy + newTag: 2022w10 +- name: postgresql-placeholder + newName: postgres + newTag: "13" +- name: redis-placeholder + newName: redis + newTag: "6" diff --git a/librephotos-docker/k8s/ns.yaml b/librephotos-docker/k8s/ns.yaml new file mode 100644 index 0000000..a187019 --- /dev/null +++ b/librephotos-docker/k8s/ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: librephotos diff --git a/librephotos-docker/k8s/pvcs.yaml b/librephotos-docker/k8s/pvcs.yaml new file mode 100644 index 0000000..a281a94 --- /dev/null +++ b/librephotos-docker/k8s/pvcs.yaml @@ -0,0 +1,42 @@ +# This volume holds your photos. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: photos +spec: + resources: + requests: + storage: 30Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + +--- + +# This volume contains thumbnails. Setting it to 10-15% the size of "photos" is probably reasonable. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: protected +spec: + resources: + requests: + storage: 3Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + +--- + +# The postgres volume holds all the metadata. If it's 1% the size of "photos," you're probably fine. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres +spec: + resources: + requests: + storage: 1Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce diff --git a/librephotos-docker/k8s/redis.yaml b/librephotos-docker/k8s/redis.yaml new file mode 100644 index 0000000..dfcd4f5 --- /dev/null +++ b/librephotos-docker/k8s/redis.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis +spec: + selector: + matchLabels: + app: redis + strategy: + rollingUpdate: + maxSurge: 0 + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis-placeholder + ports: + - name: redis + containerPort: 6379 + securityContext: + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + ports: + - port: 6379 + name: redis + selector: + app: redis diff --git a/librephotos-docker/libredata/.gitignore b/librephotos-docker/libredata/.gitignore new file mode 100644 index 0000000..1e04ba0 --- /dev/null +++ b/librephotos-docker/libredata/.gitignore @@ -0,0 +1,8 @@ +# .gitignore sample +################### + +# Ignore all files in this dir... +* + +# ... except for this one. +!.gitignore diff --git a/librephotos-docker/librephotos.env b/librephotos-docker/librephotos.env new file mode 100644 index 0000000..95c28c8 --- /dev/null +++ b/librephotos-docker/librephotos.env @@ -0,0 +1,62 @@ +# This file contains all the things you need to change to set up your Libre Photos. +# There are a few items that must be set for it to work such as the location of your photos. +# After the mandatory entries there are some optional ones that you may set. + +# Start of mandatory changes. + +# Location of your photos. +scanDirectory=./librephotos/pictures + +# Internal data of LibrePhotos +data=./librephotos/data + +# ------------------------------------------------------------------------------------------------ + +# Wow, we are at the optional now. Pretty easy so far. You do not have to change any of the below. + +#What port should Libre Photos be accessed at (Default 3000) +httpPort=3000 + +# What branch should we install the latest weekly build or the development branch (dev) +tag=latest + +# Number of workers, which take care of the request to the api. This setting can dramatically affect the ram usage. +# A positive integer generally in the 2-4 x $(NUM_CORES) range. +# You’ll want to vary this a bit to find the best for your particular workload. +# Each worker needs 800MB of RAM. Change at your own will. Default is 2. +gunniWorkers=2 + +# You can set the database name. Did you know Libre Photos was forked from OwnPhotos? +dbName=librephotos + +# Here you can change the user name for the database. +dbUser=docker + +# The password used by the database. +dbPass=AaAa1234 + +# Default minimum rating to interpret as favorited. This default value is used when creating a new user. +# Users can change this in their settings (Dashboards > Library). +DEFAULT_FAVORITE_MIN_RATING=4 + +# Database host. Only change this if you want to use your own existing Postgres server. If using your own server, you can remove the 'db' container from docker-compose.yml. If you're changing the name of the DB's container name (DB_CONT_NAME further down), you need to set this variable to match that name too. +dbHost=db + +# Set the names of the docker containers to your own entries. Or don't, I'm not your dad. +# Changing these will require you to `make rename` to rename the services, and start the system with your chosen `docker-compose up -d` invocation again. +# Note that changing the DB_CONT_NAME will also need you to set the `dbHost` variable to the same value. +DB_CONT_NAME=db +BACKEND_CONT_NAME=backend +FRONTEND_CONT_NAME=frontend +PROXY_CONT_NAME=proxy +REDIS_CONT_NAME=redis +PGADMIN_CONT_NAME=pgadmin +# --------------------------------------------------------------------------------------------- + +# If you are not a developer ignore the following parameters: you will never need them. + +# Where shall we store the backend and frontend code files. +codedir=./librephotos/code + +# Location for pgAdmin +pgAdminLocation=./librephotos/pgadmin diff --git a/librephotos-docker/proxy/Dockerfile b/librephotos-docker/proxy/Dockerfile new file mode 100644 index 0000000..f850d38 --- /dev/null +++ b/librephotos-docker/proxy/Dockerfile @@ -0,0 +1,6 @@ +FROM nginx +ENV BACKEND_NAME=${BACKEND_NAME:-backend} +ENV FRONTEND_NAME=${FRONTEND_NAME:-frontend} +ENV VAR_PREFIX='$' +COPY nginx.tpl /etc/nginx/nginx.tpl +ENTRYPOINT envsubst < /etc/nginx/nginx.tpl > /etc/nginx/nginx.conf; nginx -g 'daemon off;' diff --git a/librephotos-docker/proxy/nginx.tpl b/librephotos-docker/proxy/nginx.tpl new file mode 100644 index 0000000..1d9775b --- /dev/null +++ b/librephotos-docker/proxy/nginx.tpl @@ -0,0 +1,62 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log debug; + +events { + worker_connections 1024; +} + +http { + server { + listen 80; + + location / { + # React routes are entirely on the App side in the web browser + # Always proxy to root with the same page request when nginx 404s + error_page 404 /; + proxy_intercept_errors on; + proxy_set_header Host ${VAR_PREFIX}host; + proxy_pass http://${FRONTEND_NAME}:3000/; + } + location ~ ^/(api|media)/ { + proxy_set_header X-Forwarded-Proto ${VAR_PREFIX}scheme; + proxy_set_header X-Real-IP ${VAR_PREFIX}remote_addr; + proxy_set_header Host ${BACKEND_NAME}; + include uwsgi_params; + proxy_pass http://${BACKEND_NAME}:8001; + } + # needed for webpack-dev-server + location /ws { + proxy_pass http://${FRONTEND_NAME}:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade ${VAR_PREFIX}http_upgrade; + proxy_set_header Connection "upgrade"; + } + # Django media + location /protected_media { + internal; + alias /protected_media/; + } + + location /static/drf-yasg { + proxy_pass http://${BACKEND_NAME}:8001; + } + + location /data { + internal; + alias /data/; + } + + # Original Photos + location /original { + internal; + alias /data/; + } + # Nextcloud Original Photos + location /nextcloud_original { + internal; + alias /data/nextcloud_media/; + } + } +} diff --git a/librephotos-docker/vscode/settings.json b/librephotos-docker/vscode/settings.json new file mode 100644 index 0000000..31f641c --- /dev/null +++ b/librephotos-docker/vscode/settings.json @@ -0,0 +1,23 @@ +{ + "files.exclude": { + "**/*.py[co]": true, + "**/*.so": true, + "**/__pycache__": true + }, + "python.pythonPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--exclude: .+/migrations/", + "--max-line-length=119" + ], + "python.linting.pylintArgs": [ + "--load-plugins=pylint_django", + "-d", + "E0239", + "-d", + "C0111" + ], + "python.linting.pylintEnabled": true, + "python.sortImports.path": "isort" +} diff --git a/miniflux/docker-compose.yml b/miniflux/docker-compose.yml new file mode 100644 index 0000000..7d85f5f --- /dev/null +++ b/miniflux/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3' +services: + miniflux: + image: miniflux/miniflux:latest + ports: + - "8050:8080" + depends_on: + - db + environment: + - DATABASE_URL=postgres://miniflux:secret@db/miniflux?sslmode=disable + - RUN_MIGRATIONS=1 + - CREATE_ADMIN=1 + - ADMIN_USERNAME=admin + - ADMIN_PASSWORD=test123 + db: + image: postgres:latest + environment: + - POSTGRES_USER=miniflux + - POSTGRES_PASSWORD=secret + volumes: + - miniflux-db:/var/lib/postgresql/data +volumes: + miniflux-db: \ No newline at end of file diff --git a/miniflux/readme.md b/miniflux/readme.md new file mode 100644 index 0000000..660da15 --- /dev/null +++ b/miniflux/readme.md @@ -0,0 +1,2 @@ +* https://lukesingham.com/rss-feed-reader/ +* https://miniflux.app/ \ No newline at end of file diff --git a/minio/docker-compose.yml b/minio/docker-compose.yml new file mode 100644 index 0000000..d4a3056 --- /dev/null +++ b/minio/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3' + +services: + minio: + image: minio/minio + ports: + - "7000:9000" + - "7001:9001" + volumes: + - minio_storage:/data + environment: + MINIO_ROOT_USER: admin + MINIO_ROOT_PASSWORD: Strong#Pass#2022 + command: server --console-address ":9001" /data + +volumes: + minio_storage: {} \ No newline at end of file diff --git a/netbox/config/.gitignore b/netbox/config/.gitignore new file mode 100644 index 0000000..1e04ba0 --- /dev/null +++ b/netbox/config/.gitignore @@ -0,0 +1,8 @@ +# .gitignore sample +################### + +# Ignore all files in this dir... +* + +# ... except for this one. +!.gitignore diff --git a/netbox/docker-compose.yml b/netbox/docker-compose.yml new file mode 100644 index 0000000..0b9b134 --- /dev/null +++ b/netbox/docker-compose.yml @@ -0,0 +1,35 @@ +--- +version: "2.1" +services: + netbox: + image: lscr.io/linuxserver/netbox:latest + container_name: netbox + environment: + - PUID=1000 + - PGID=1000 + - TZ= + - SUPERUSER_EMAIL= + - SUPERUSER_PASSWORD= + - ALLOWED_HOST= + - DB_NAME= + - DB_USER= + - DB_PASSWORD= + - DB_HOST= + - DB_PORT= + - REDIS_HOST= + - REDIS_PORT= + - REDIS_PASSWORD= + - REDIS_DB_TASK= + - REDIS_DB_CACHE= + - BASE_PATH= #optional + - REMOTE_AUTH_ENABLED= #optional + - REMOTE_AUTH_BACKEND= #optional + - REMOTE_AUTH_HEADER= #optional + - REMOTE_AUTH_AUTO_CREATE_USER= #optional + - REMOTE_AUTH_DEFAULT_GROUPS= #optional + - REMOTE_AUTH_DEFAULT_PERMISSIONS= #optional + volumes: + - :/config + ports: + - 8000:8000 + restart: unless-stopped \ No newline at end of file diff --git a/netbox/readme.md b/netbox/readme.md new file mode 100644 index 0000000..8c5f4e5 --- /dev/null +++ b/netbox/readme.md @@ -0,0 +1 @@ +https://github.com/netbox-community/netbox-docker/wiki/Getting-Started diff --git a/vaultwarden/docker-compose.yml b/vaultwarden/docker-compose.yml new file mode 100644 index 0000000..9fc4ba9 --- /dev/null +++ b/vaultwarden/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3' + +services: + vaultwarden: + container_name: vaultwarden + image: vaultwarden/server:latest + restart: unless-stopped + environment: + - WEBSOCKET_ENABLED=true + volumes: + - ./data/:/data/ + ports: + - 127.0.0.1:8882:80 + - 127.0.0.1:3012:3012 \ No newline at end of file diff --git a/vaultwarden/readme.md b/vaultwarden/readme.md new file mode 100644 index 0000000..785a3a6 --- /dev/null +++ b/vaultwarden/readme.md @@ -0,0 +1,5 @@ +https://www.linode.com/docs/guides/how-to-self-host-the-vaultwarden-password-manager/ + +https://webdock.io/en/docs/how-guides/app-installation-and-setup/how-install-vaultwarden-your-webdock-server +# was +https://www.techaddressed.com/tutorials/vaultwarden-docker-compose/ \ No newline at end of file diff --git a/warden/docker-compose.yml b/warden/docker-compose.yml new file mode 100644 index 0000000..ad8d308 --- /dev/null +++ b/warden/docker-compose.yml @@ -0,0 +1,64 @@ +# This Docker Compose configuration can be used to set up a vaultwarden +# setup on the *local* network. +# +# Many existing configurations are meant to be used on a public network +# with a Let's Encrypt certificate, its own domain etc. +# This configuration is meant to remain in the local network and not get +# exposed to the public facing internet to provide an additional layer +# of security. This way, passwords will only sync when in the local network. +# +# This configuration assumes that it will be available from https://nasty.local:86 +# but the hostname and port can be changes freely. +# Simply copy this file to the homeserver, customize the admin token and hostname +# any run "docker-compose up -d" to start the containers. +# +# By default, self-sign up will be deisabled so no new users can simply register an +# account. To create an account, you will first need to open the admin console +# at https://nasty.local:86/admin and invite the new user via their email address. +# Unless manually added, no SMTP server is connected and thus no mails will be sent. +# You can simply now register using the email address you invited using the frontend. +# +# A self-signed HTTPS certificate will be created for hosting so you might need +# to add the self-created HTTPS autority certificate found at +# ./vw-data/caddy-data/caddy/pki/authorities/local/root.crt to use the server +# +# All data can be backed up by simply copying the "./vw-data" folder that contains +# all data for vaultwarden and the Caddy reverse proxy +version: '3' + +services: + vaultwarden: + image: vaultwarden/server:latest + container_name: vaultwarden2 + restart: unless-stopped + environment: + - WEBSOCKET_ENABLED=true + - ADMIN_TOKEN=myadmintoken + - SIGNUPS_ALLOWED=true + - DOMAIN=https://warden.local + # - SIGNUPS_ALLOWED=true + # - SIGNUPS_VERIFY=true + volumes: + - ./vw-data/vw:/data + networks: + - vaultwarden2 + + caddy: + image: caddy:2 + container_name: caddy2 + command: caddy reverse-proxy --from warden.local:86 --to vaultwarden:80 + restart: unless-stopped + ports: + - 86:86 + - 87:443 + volumes: + - ./vw-data/caddy-config:/config + - ./vw-data/caddy-data:/data + environment: + - DOMAIN=warden.local + networks: + - vaultwarden2 + +networks: + vaultwarden2: + driver: bridge