Update runner image

Ubuntu 20.04 was deprecated

.github/workflows/build.yml

@@ -16,7 +16,7 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v2
 

.gitignore

@@ -21,6 +21,9 @@ vendor
 # GraphQL generated output
 internal/api/generated_*.go
 
+# Generated locale files
+ui/login/locales/*
+
 ####
 # Visual Studio
 ####

.golangci.yml

@@ -48,8 +48,6 @@ linters-settings:
     ignore-generated-header: true
     severity: error
     confidence: 0.8
-    error-code: 1
-    warning-code: 1
     rules:
       - name: blank-imports
         disabled: true

Makefile

@@ -281,6 +281,10 @@ generate-ui:
 generate-backend: touch-ui
 	go generate ./cmd/stash
 
+.PHONY: generate-login-locale
+generate-login-locale:
+	go generate ./ui
+
 .PHONY: generate-dataloaders
 generate-dataloaders:
 	go generate ./internal/api/loaders
@@ -351,7 +355,10 @@ ifdef STASH_SOURCEMAPS
 endif
 
 .PHONY: ui
-ui: ui-env
+ui: ui-only generate-login-locale
+
+.PHONY: ui-only
+ui-only: ui-env
 	cd ui/v2.5 && yarn build
 
 .PHONY: zip-ui

README.md

@@ -81,9 +81,9 @@ For more help you can:
 # Customization
 
 ## Themes and CSS Customization
-There is a [directory of community-created themes](https://docs.stashapp.cc/user-interface-ui/themes) on Stash-Docs, along with instructions on how to install them.
+There is a [directory of community-created themes](https://docs.stashapp.cc/themes/list) on Stash-Docs.
 
-You can also change the Stash interface to fit your desired style with various snippets from [Custom CSS snippets](https://docs.stashapp.cc/user-interface-ui/custom-css-snippets).
+You can also change the Stash interface to fit your desired style with various snippets from [Custom CSS snippets](https://docs.stashapp.cc/themes/custom-css-snippets).
 
 # For Developers
 

cmd/stash/main.go

@@ -152,6 +152,9 @@ func recoverPanic() {
 func exitError(err error) {
 	exitCode = 1
 	logger.Error(err)
+	// #5784 - log to stdout as well as the logger
+	// this does mean that it will log twice if the logger is set to stdout
+	fmt.Println(err)
 	if desktop.IsDesktop() {
 		desktop.FatalError(err)
 	}

docker/build/x86_64/Dockerfile

@@ -1,7 +1,7 @@
 # This dockerfile should be built with `make docker-build` from the stash root.
 
 # Build Frontend
-FROM node:alpine as frontend
+FROM node:20-alpine AS frontend
 RUN apk add --no-cache make git
 ## cache node_modules separately
 COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/
@@ -13,19 +13,22 @@ RUN make pre-ui
 RUN make generate-ui
 ARG GITHASH
 ARG STASH_VERSION
-RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui
+RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui-only
 
 # Build Backend
-FROM golang:1.22-alpine as backend
+FROM golang:1.22.8-alpine AS backend
 RUN apk add --no-cache make alpine-sdk
 WORKDIR /stash
 COPY ./go* ./*.go Makefile gqlgen.yml .gqlgenc.yml /stash/
+COPY ./graphql /stash/graphql/
 COPY ./scripts /stash/scripts/
 COPY ./pkg /stash/pkg/
-COPY ./cmd /stash/cmd
-COPY ./internal /stash/internal
+COPY ./cmd /stash/cmd/
+COPY ./internal /stash/internal/
+# needed for generate-login-locale
+COPY ./ui /stash/ui/
+RUN make generate-backend generate-login-locale
 COPY --from=frontend /stash /stash/
-RUN make generate-backend
 ARG GITHASH
 ARG STASH_VERSION
 RUN make flags-release flags-pie stash

docker/build/x86_64/Dockerfile-CUDA

@@ -1,7 +1,8 @@
 # This dockerfile should be built with `make docker-cuda-build` from the stash root.
+ARG CUDA_VERSION=12.8.0
 
 # Build Frontend
-FROM node:alpine as frontend
+FROM node:20-alpine AS frontend
 RUN apk add --no-cache make git
 ## cache node_modules separately
 COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/
@@ -13,37 +14,47 @@ RUN make pre-ui
 RUN make generate-ui
 ARG GITHASH
 ARG STASH_VERSION
-RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui
+RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui-only
 
 # Build Backend
-FROM golang:1.22-bullseye as backend
+FROM golang:1.22.8-bullseye AS backend
 RUN apt update && apt install -y build-essential golang
 WORKDIR /stash
 COPY ./go* ./*.go Makefile gqlgen.yml .gqlgenc.yml /stash/
+COPY ./graphql /stash/graphql/
 COPY ./scripts /stash/scripts/
 COPY ./pkg /stash/pkg/
 COPY ./cmd /stash/cmd
 COPY ./internal /stash/internal
+# needed for generate-login-locale
+COPY ./ui /stash/ui/
+RUN make generate-backend generate-login-locale
 COPY --from=frontend /stash /stash/
-RUN make generate-backend
 ARG GITHASH
 ARG STASH_VERSION
 RUN make flags-release flags-pie stash
 
 # Final Runnable Image
-FROM nvidia/cuda:12.0.1-base-ubuntu22.04
-RUN apt update && apt upgrade -y && apt install -y ca-certificates libvips-tools ffmpeg wget intel-media-va-driver-non-free vainfo
-RUN rm -rf /var/lib/apt/lists/*
-COPY --from=backend /stash/stash /usr/bin/
+FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu24.04
+RUN apt update && apt upgrade -y && apt install -y \
+    # stash dependencies
+    ca-certificates libvips-tools ffmpeg \
+    # intel dependencies
+    intel-media-va-driver-non-free vainfo \
+    # python tools
+    python3 python3-pip && \
+  # cleanup
+  apt autoremove -y && apt clean && \
+  rm -rf /var/lib/apt/lists/*
+COPY --from=backend --chmod=555 /stash/stash /usr/bin/
 
 # NVENC Patch
 RUN mkdir -p /usr/local/bin /patched-lib
-RUN wget https://raw.githubusercontent.com/keylase/nvidia-patch/master/patch.sh -O /usr/local/bin/patch.sh
-RUN wget https://raw.githubusercontent.com/keylase/nvidia-patch/master/docker-entrypoint.sh -O /usr/local/bin/docker-entrypoint.sh
-RUN chmod +x /usr/local/bin/patch.sh /usr/local/bin/docker-entrypoint.sh /usr/bin/stash
+ADD --chmod=555 https://raw.githubusercontent.com/keylase/nvidia-patch/master/patch.sh /usr/local/bin/patch.sh
+ADD --chmod=555 https://raw.githubusercontent.com/keylase/nvidia-patch/master/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
-ENV LANG C.UTF-8
-ENV NVIDIA_VISIBLE_DEVICES all
+ENV LANG=C.UTF-8
+ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=video,utility
 ENV STASH_CONFIG_FILE=/root/.stash/config.yml
 EXPOSE 9999

docker/build/x86_64/README.md

@@ -4,7 +4,7 @@ This dockerfile is used to build a stash docker container using the current sour
 
 # Building the docker container
 
-From the top-level directory (should contain `main.go` file):
+From the top-level directory (should contain `tools.go` file):
 
 ```
 make docker-build

docker/production/README.md

@@ -1,12 +1,14 @@
 # Docker Installation (for most 64-bit GNU/Linux systems)
-StashApp is supported on most systems that support Docker and docker-compose. Your OS likely ships with or makes available the necessary packages.
+StashApp is supported on most systems that support Docker. Your OS likely ships with or makes available the necessary packages.
 
 ## Dependencies
-Only `docker` and `docker-compose` are required. For the most part your understanding of the technologies can be superficial. So long as you can follow commands and are open to reading a bit, you should be fine.
+Only `docker` is required. For the most part your understanding of the technologies can be superficial. So long as you can follow commands and are open to reading a bit, you should be fine.
 
-Installation instructions are available below, and if your distrobution's repository ships a current version of docker, you may use that.
+Installation instructions are available below, and if your distributions's repository ships a current version of docker, you may use that.
 https://docs.docker.com/engine/install/
 
+On some distributions, `docker compose` is shipped seperately, usually as `docker-cli-compose`. docker-compose is not recommended.
+
 ### Get the docker-compose.yml file
 
 Now you can either navigate to the [docker-compose.yml](https://raw.githubusercontent.com/stashapp/stash/develop/docker/production/docker-compose.yml) in the repository, or if you have curl, you can make your Linux console do it for you:
@@ -19,7 +21,7 @@ curl -o docker-compose.yml https://raw.githubusercontent.com/stashapp/stash/deve
 Once you have that file where you want it, modify the settings as you please, and then run:
 
 ```
-docker-compose up -d
+docker compose up -d
 ```
 
 Installing StashApp this way will by default bind stash to port 9999. This is available in your web browser locally at http://localhost:9999 or on your network as http://YOUR-LOCAL-IP:9999
@@ -29,9 +31,9 @@ Good luck and have fun!
 ### Docker
 Docker is effectively a cross-platform software package repository. It allows you to ship an entire environment in what's referred to as a container. Containers are intended to hold everything that is needed to run an application from one place to another, making it easy for everyone along the way to reproduce the environment.
 
-The StashApp docker container ships with everything you need to automatically build and run stash, including ffmpeg.
+The StashApp docker container ships with everything you need to automatically run stash, including ffmpeg.
 
-### docker-compose
-Docker Compose lets you specify how and where to run your containers, and to manage their environment. The docker-compose.yml file in this folder gets you a fully working instance of StashApp exactly as you would need it to have a reasonable instance for testing / developing on. If you are deploying a live instance for production, a reverse proxy (such as NGINX or Traefik) is recommended, but not required.
+### docker compose
+Docker Compose lets you specify how and where to run your containers, and to manage their environment. The docker-compose.yml file in this folder gets you a fully working instance of StashApp exactly as you would need it to have a reasonable instance for testing / developing on. If you are deploying a live instance for production, a [reverse proxy](https://docs.stashapp.cc/guides/reverse-proxy/) (such as NGINX or Traefik) is recommended, but not required.
 
 The latest version is always recommended.

docker/production/docker-compose.yml

@@ -1,6 +1,5 @@
 # APPNICENAME=Stash
 # APPDESCRIPTION=An organizer for your porn, written in Go
-version: '3.4'
 services:
   stash:
     image: stashapp/stash:latest
@@ -27,10 +26,12 @@ services:
       - /etc/localtime:/etc/localtime:ro
       ## Adjust below paths (the left part) to your liking.
       ## E.g. you can change ./config:/root/.stash to ./stash:/root/.stash
-      
+      ## The left part is the path on your host, the right part is the path in the stash container.
+
       ## Keep configs, scrapers, and plugins here.
       - ./config:/root/.stash
       ## Point this at your collection.
+      ## The left side is where your collection is on your host, the right side is where it will be in stash.
       - ./data:/data
       ## This is where your stash's metadata lives
       - ./metadata:/metadata

go.mod

@@ -19,9 +19,9 @@ require (
 	github.com/go-chi/cors v1.2.1
 	github.com/go-chi/httplog v0.3.1
 	github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4
-	github.com/gofrs/uuid/v5 v5.1.0
-	github.com/golang-jwt/jwt/v4 v4.5.1
+	github.com/golang-jwt/jwt/v4 v4.5.2
 	github.com/golang-migrate/migrate/v4 v4.16.2
+	github.com/google/uuid v1.6.0
 	github.com/gorilla/securecookie v1.1.1
 	github.com/gorilla/sessions v1.2.1
 	github.com/gorilla/websocket v1.5.0
@@ -51,12 +51,13 @@ require (
 	github.com/vektra/mockery/v2 v2.10.0
 	github.com/xWTF/chardet v0.0.0-20230208095535-c780f2ac244e
 	github.com/zencoder/go-dash/v3 v3.0.2
-	golang.org/x/crypto v0.28.0
+	golang.org/x/crypto v0.31.0
 	golang.org/x/image v0.18.0
-	golang.org/x/net v0.30.0
-	golang.org/x/sys v0.26.0
-	golang.org/x/term v0.25.0
-	golang.org/x/text v0.19.0
+	golang.org/x/net v0.33.0
+	golang.org/x/sys v0.28.0
+	golang.org/x/term v0.27.0
+	golang.org/x/text v0.21.0
+	golang.org/x/time v0.10.0
 	gopkg.in/guregu/null.v4 v4.0.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -78,7 +79,6 @@ require (
 	github.com/gobwas/ws v1.3.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -113,7 +113,7 @@ require (
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/mod v0.21.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -244,12 +244,10 @@ github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/K
 github.com/gobwas/ws v1.3.0 h1:sbeU3Y4Qzlb+MOzIe6mQGf7QR4Hkv6ZD0qhGkBFL2O0=
 github.com/gobwas/ws v1.3.0/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/uuid/v5 v5.1.0 h1:S5rqVKIigghZTCBKPCw0Y+bXkn26K3TB5mvQq2Ix8dk=
-github.com/gofrs/uuid/v5 v5.1.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
-github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
+github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA=
 github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
@@ -715,8 +713,8 @@ golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
-golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -809,8 +807,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -840,8 +838,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -932,13 +930,13 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
-golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
-golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
+golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -951,11 +949,13 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
+golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

gqlgen.yml

@@ -17,7 +17,7 @@ autobind:
   - github.com/stashapp/stash/pkg/scraper
   - github.com/stashapp/stash/internal/identify
   - github.com/stashapp/stash/internal/dlna
-  - github.com/stashapp/stash/pkg/scraper/stashbox
+  - github.com/stashapp/stash/pkg/stashbox
 
 models:
   # Scalars
@@ -35,12 +35,6 @@ models:
     model: github.com/stashapp/stash/internal/api.BoolMap
   PluginConfigMap:
     model: github.com/stashapp/stash/internal/api.PluginConfigMap
-  # define to force resolvers
-  Image:
-    model: github.com/stashapp/stash/pkg/models.Image
-    fields:
-      title:
-        resolver: true
   VideoFile:
     fields:
       # override float fields - #1572

graphql/schema/schema.graphql

@@ -45,6 +45,7 @@ type Query {
   findSceneMarkers(
     scene_marker_filter: SceneMarkerFilterType
     filter: FindFilterType
+    ids: [ID!]
   ): FindSceneMarkersResultType!
 
   findImage(id: ID, checksum: String): Image
@@ -173,6 +174,12 @@ type Query {
     input: ScrapeSingleGroupInput!
   ): [ScrapedGroup!]!
 
+  "Scrape for a single image"
+  scrapeSingleImage(
+    source: ScraperSourceInput!
+    input: ScrapeSingleImageInput!
+  ): [ScrapedImage!]!
+
   "Scrapes content based on a URL"
   scrapeURL(url: String!, ty: ScrapeContentType!): ScrapedContent
 
@@ -182,6 +189,8 @@ type Query {
   scrapeSceneURL(url: String!): ScrapedScene
   "Scrapes a complete gallery record based on a URL"
   scrapeGalleryURL(url: String!): ScrapedGallery
+  "Scrapes a complete image record based on a URL"
+  scrapeImageURL(url: String!): ScrapedImage
   "Scrapes a complete movie record based on a URL"
   scrapeMovieURL(url: String!): ScrapedMovie
     @deprecated(reason: "Use scrapeGroupURL instead")

graphql/schema/types/filters.graphql

@@ -542,6 +542,9 @@ input TagFilterType {
   "Filter by tag name"
   name: StringCriterionInput
 
+  "Filter by tag sort_name"
+  sort_name: StringCriterionInput
+
   "Filter by tag aliases"
   aliases: StringCriterionInput
 

graphql/schema/types/scraper.graphql

@@ -10,6 +10,7 @@ enum ScrapeType {
 "Type of the content a scraper generates"
 enum ScrapeContentType {
   GALLERY
+  IMAGE
   MOVIE
   GROUP
   PERFORMER
@@ -22,6 +23,7 @@ union ScrapedContent =
   | ScrapedTag
   | ScrapedScene
   | ScrapedGallery
+  | ScrapedImage
   | ScrapedMovie
   | ScrapedGroup
   | ScrapedPerformer
@@ -41,6 +43,8 @@ type Scraper {
   scene: ScraperSpec
   "Details for gallery scraper"
   gallery: ScraperSpec
+  "Details for image scraper"
+  image: ScraperSpec
   "Details for movie scraper"
   movie: ScraperSpec @deprecated(reason: "use group")
   "Details for group scraper"
@@ -128,6 +132,26 @@ input ScrapedGalleryInput {
   # no studio, tags or performers
 }
 
+type ScrapedImage {
+  title: String
+  code: String
+  details: String
+  photographer: String
+  urls: [String!]
+  date: String
+  studio: ScrapedStudio
+  tags: [ScrapedTag!]
+  performers: [ScrapedPerformer!]
+}
+
+input ScrapedImageInput {
+  title: String
+  code: String
+  details: String
+  urls: [String!]
+  date: String
+}
+
 input ScraperSourceInput {
   "Index of the configured stash-box instance to use. Should be unset if scraper_id is set"
   stash_box_index: Int @deprecated(reason: "use stash_box_endpoint")
@@ -190,6 +214,15 @@ input ScrapeSingleGalleryInput {
   gallery_input: ScrapedGalleryInput
 }
 
+input ScrapeSingleImageInput {
+  "Instructs to query by string"
+  query: String
+  "Instructs to query by image id"
+  image_id: ID
+  "Instructs to query by image fragment"
+  image_input: ScrapedImageInput
+}
+
 input ScrapeSingleMovieInput {
   "Instructs to query by string"
   query: String

graphql/schema/types/stash-box.graphql

@@ -2,12 +2,15 @@ type StashBox {
   endpoint: String!
   api_key: String!
   name: String!
+  max_requests_per_minute: Int!
 }
 
 input StashBoxInput {
   endpoint: String!
   api_key: String!
   name: String!
+  # defaults to 240
+  max_requests_per_minute: Int
 }
 
 type StashID {

graphql/schema/types/tag.graphql

@@ -1,6 +1,8 @@
 type Tag {
   id: ID!
   name: String!
+  "Value that does not appear in the UI but overrides name for sorting"
+  sort_name: String
   description: String
   aliases: [String!]!
   ignore_auto_tag: Boolean!
@@ -25,6 +27,8 @@ type Tag {
 
 input TagCreateInput {
   name: String!
+  "Value that does not appear in the UI but overrides name for sorting"
+  sort_name: String
   description: String
   aliases: [String!]
   ignore_auto_tag: Boolean
@@ -39,6 +43,8 @@ input TagCreateInput {
 input TagUpdateInput {
   id: ID!
   name: String
+  "Value that does not appear in the UI but overrides name for sorting"
+  sort_name: String
   description: String
   aliases: [String!]
   ignore_auto_tag: Boolean

graphql/stash-box/query.graphql

@@ -49,6 +49,8 @@ fragment PerformerFragment on Performer {
   aliases
   gender
   merged_ids
+  deleted
+  merged_into_id
   urls {
     ...URLFragment
   }
@@ -56,6 +58,7 @@ fragment PerformerFragment on Performer {
     ...ImageFragment
   }
   birth_date
+  death_date
   ethnicity
   country
   eye_color

internal/api/json.go

@@ -3,9 +3,11 @@ package api
 import (
 	"encoding/json"
 	"strings"
+
+	"github.com/stashapp/stash/pkg/models"
 )
 
-// JSONNumberToNumber converts a JSON number to either a float64 or int64.
+// jsonNumberToNumber converts a JSON number to either a float64 or int64.
 func jsonNumberToNumber(n json.Number) interface{} {
 	if strings.Contains(string(n), ".") {
 		f, _ := n.Float64()
@@ -15,6 +17,15 @@ func jsonNumberToNumber(n json.Number) interface{} {
 	return ret
 }
 
+// anyJSONNumberToNumber converts a JSON number using jsonNumberToNumber, otherwise it returns the existing value
+func anyJSONNumberToNumber(v any) any {
+	if n, ok := v.(json.Number); ok {
+		return jsonNumberToNumber(n)
+	}
+
+	return v
+}
+
 // ConvertMapJSONNumbers converts all JSON numbers in a map to either float64 or int64.
 func convertMapJSONNumbers(m map[string]interface{}) (ret map[string]interface{}) {
 	if m == nil {
@@ -34,3 +45,21 @@ func convertMapJSONNumbers(m map[string]interface{}) (ret map[string]interface{}
 
 	return ret
 }
+
+func convertCustomFieldCriterionValues(c models.CustomFieldCriterionInput) models.CustomFieldCriterionInput {
+	nv := make([]any, len(c.Value))
+	for i, v := range c.Value {
+		nv[i] = anyJSONNumberToNumber(v)
+	}
+	c.Value = nv
+	return c
+}
+
+func convertCustomFieldCriterionInputJSONNumbers(c []models.CustomFieldCriterionInput) []models.CustomFieldCriterionInput {
+	ret := make([]models.CustomFieldCriterionInput, len(c))
+	for i, v := range c {
+		ret[i] = convertCustomFieldCriterionValues(v)
+	}
+
+	return ret
+}

internal/api/resolver.go

@@ -13,7 +13,6 @@ import (
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/plugin/hook"
 	"github.com/stashapp/stash/pkg/scraper"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
 )
 
 var (
@@ -138,10 +137,6 @@ func (r *Resolver) withReadTxn(ctx context.Context, fn func(ctx context.Context)
 	return r.repository.WithReadTxn(ctx, fn)
 }
 
-func (r *Resolver) stashboxRepository() stashbox.Repository {
-	return stashbox.NewRepository(r.repository)
-}
-
 func (r *queryResolver) MarkerWall(ctx context.Context, q *string) (ret []*models.SceneMarker, err error) {
 	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
 		ret, err = r.repository.SceneMarker.Wall(ctx, q)

internal/api/resolver_model_image.go

@@ -18,11 +18,6 @@ func (r *imageResolver) getFiles(ctx context.Context, obj *models.Image) ([]mode
 	return files, firstError(errs)
 }
 
-func (r *imageResolver) Title(ctx context.Context, obj *models.Image) (*string, error) {
-	ret := obj.GetTitle()
-	return &ret, nil
-}
-
 func (r *imageResolver) VisualFiles(ctx context.Context, obj *models.Image) ([]VisualFile, error) {
 	files, err := r.getFiles(ctx, obj)
 	if err != nil {

internal/api/resolver_mutation_image.go

@@ -28,7 +28,7 @@ func (r *mutationResolver) getImage(ctx context.Context, id int) (ret *models.Im
 	return ret, nil
 }
 
-func (r *mutationResolver) ImageUpdate(ctx context.Context, input ImageUpdateInput) (ret *models.Image, err error) {
+func (r *mutationResolver) ImageUpdate(ctx context.Context, input models.ImageUpdateInput) (ret *models.Image, err error) {
 	translator := changesetTranslator{
 		inputMap: getUpdateInputMap(ctx),
 	}
@@ -46,7 +46,7 @@ func (r *mutationResolver) ImageUpdate(ctx context.Context, input ImageUpdateInp
 	return r.getImage(ctx, ret.ID)
 }
 
-func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*ImageUpdateInput) (ret []*models.Image, err error) {
+func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*models.ImageUpdateInput) (ret []*models.Image, err error) {
 	inputMaps := getUpdateInputMaps(ctx)
 
 	// Start the transaction and save the image
@@ -89,7 +89,7 @@ func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*ImageUpdat
 	return newRet, nil
 }
 
-func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInput, translator changesetTranslator) (*models.Image, error) {
+func (r *mutationResolver) imageUpdate(ctx context.Context, input models.ImageUpdateInput, translator changesetTranslator) (*models.Image, error) {
 	imageID, err := strconv.Atoi(input.ID)
 	if err != nil {
 		return nil, fmt.Errorf("converting id: %w", err)

internal/api/resolver_mutation_scene.go

@@ -694,6 +694,13 @@ func validateSceneMarkerEndSeconds(seconds, endSeconds float64) error {
 	return nil
 }
 
+func float64OrZero(f *float64) float64 {
+	if f == nil {
+		return 0
+	}
+	return *f
+}
+
 func (r *mutationResolver) SceneMarkerUpdate(ctx context.Context, input SceneMarkerUpdateInput) (*models.SceneMarker, error) {
 	markerID, err := strconv.Atoi(input.ID)
 	if err != nil {
@@ -784,7 +791,7 @@ func (r *mutationResolver) SceneMarkerUpdate(ctx context.Context, input SceneMar
 		}
 
 		// remove the marker preview if the scene changed or if the timestamp was changed
-		if existingMarker.SceneID != newMarker.SceneID || existingMarker.Seconds != newMarker.Seconds || existingMarker.EndSeconds != newMarker.EndSeconds {
+		if existingMarker.SceneID != newMarker.SceneID || existingMarker.Seconds != newMarker.Seconds || float64OrZero(existingMarker.EndSeconds) != float64OrZero(newMarker.EndSeconds) {
 			seconds := int(existingMarker.Seconds)
 			if err := fileDeleter.MarkMarkerFiles(existingScene, seconds); err != nil {
 				return err

internal/api/resolver_mutation_stash_box.go

@@ -7,6 +7,10 @@ import (
 
 	"github.com/stashapp/stash/internal/manager"
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/scene"
+	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
+	"github.com/stashapp/stash/pkg/stashbox"
 )
 
 func (r *mutationResolver) SubmitStashBoxFingerprints(ctx context.Context, input StashBoxFingerprintSubmissionInput) (bool, error) {
@@ -15,8 +19,23 @@ func (r *mutationResolver) SubmitStashBoxFingerprints(ctx context.Context, input
 		return false, err
 	}
 
+	ids, err := stringslice.StringSliceToIntSlice(input.SceneIds)
+	if err != nil {
+		return false, err
+	}
+
 	client := r.newStashBoxClient(*b)
-	return client.SubmitStashBoxFingerprints(ctx, input.SceneIds)
+
+	var scenes []*models.Scene
+
+	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+		scenes, err = r.sceneService.FindByIDs(ctx, ids, scene.LoadStashIDs, scene.LoadFiles)
+		return err
+	}); err != nil {
+		return false, err
+	}
+
+	return client.SubmitFingerprints(ctx, scenes)
 }
 
 func (r *mutationResolver) StashBoxBatchPerformerTag(ctx context.Context, input manager.StashBoxBatchTagInput) (string, error) {
@@ -69,17 +88,76 @@ func (r *mutationResolver) SubmitStashBoxSceneDraft(ctx context.Context, input S
 			logger.Errorf("Error getting scene cover: %v", err)
 		}
 
-		if err := scene.LoadURLs(ctx, r.repository.Scene); err != nil {
-			return fmt.Errorf("loading scene URLs: %w", err)
+		draft, err := r.makeSceneDraft(ctx, scene, cover)
+		if err != nil {
+			return err
 		}
 
-		res, err = client.SubmitSceneDraft(ctx, scene, cover)
+		res, err = client.SubmitSceneDraft(ctx, *draft)
 		return err
 	})
 
 	return res, err
 }
 
+func (r *mutationResolver) makeSceneDraft(ctx context.Context, s *models.Scene, cover []byte) (*stashbox.SceneDraft, error) {
+	if err := s.LoadURLs(ctx, r.repository.Scene); err != nil {
+		return nil, fmt.Errorf("loading scene URLs: %w", err)
+	}
+
+	if err := s.LoadStashIDs(ctx, r.repository.Scene); err != nil {
+		return nil, err
+	}
+
+	draft := &stashbox.SceneDraft{
+		Scene: s,
+	}
+
+	pqb := r.repository.Performer
+	sqb := r.repository.Studio
+
+	if s.StudioID != nil {
+		var err error
+		draft.Studio, err = sqb.Find(ctx, *s.StudioID)
+		if err != nil {
+			return nil, err
+		}
+		if draft.Studio == nil {
+			return nil, fmt.Errorf("studio with id %d not found", *s.StudioID)
+		}
+
+		if err := draft.Studio.LoadStashIDs(ctx, r.repository.Studio); err != nil {
+			return nil, err
+		}
+	}
+
+	// submit all file fingerprints
+	if err := s.LoadFiles(ctx, r.repository.Scene); err != nil {
+		return nil, err
+	}
+
+	scenePerformers, err := pqb.FindBySceneID(ctx, s.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, p := range scenePerformers {
+		if err := p.LoadStashIDs(ctx, pqb); err != nil {
+			return nil, err
+		}
+	}
+	draft.Performers = scenePerformers
+
+	draft.Tags, err = r.repository.Tag.FindBySceneID(ctx, s.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	draft.Cover = cover
+
+	return draft, nil
+}
+
 func (r *mutationResolver) SubmitStashBoxPerformerDraft(ctx context.Context, input StashBoxDraftSubmissionInput) (*string, error) {
 	b, err := resolveStashBox(input.StashBoxIndex, input.StashBoxEndpoint)
 	if err != nil {
@@ -105,7 +183,22 @@ func (r *mutationResolver) SubmitStashBoxPerformerDraft(ctx context.Context, inp
 			return fmt.Errorf("performer with id %d not found", id)
 		}
 
-		res, err = client.SubmitPerformerDraft(ctx, performer)
+		pqb := r.repository.Performer
+		if err := performer.LoadAliases(ctx, pqb); err != nil {
+			return err
+		}
+
+		if err := performer.LoadURLs(ctx, pqb); err != nil {
+			return err
+		}
+
+		if err := performer.LoadStashIDs(ctx, pqb); err != nil {
+			return err
+		}
+
+		img, _ := pqb.GetImage(ctx, performer.ID)
+
+		res, err = client.SubmitPerformerDraft(ctx, performer, img)
 		return err
 	})
 

internal/api/resolver_mutation_tag.go

@@ -33,6 +33,7 @@ func (r *mutationResolver) TagCreate(ctx context.Context, input TagCreateInput)
 	newTag := models.NewTag()
 
 	newTag.Name = input.Name
+	newTag.SortName = translator.string(input.SortName)
 	newTag.Aliases = models.NewRelatedStrings(input.Aliases)
 	newTag.Favorite = translator.bool(input.Favorite)
 	newTag.Description = translator.string(input.Description)
@@ -102,6 +103,7 @@ func (r *mutationResolver) TagUpdate(ctx context.Context, input TagUpdateInput)
 	updatedTag := models.NewTagPartial()
 
 	updatedTag.Name = translator.optionalString(input.Name, "name")
+	updatedTag.SortName = translator.optionalString(input.SortName, "sort_name")
 	updatedTag.Favorite = translator.optionalBool(input.Favorite, "favorite")
 	updatedTag.IgnoreAutoTag = translator.optionalBool(input.IgnoreAutoTag, "ignore_auto_tag")
 	updatedTag.Description = translator.optionalString(input.Description, "description")

internal/api/resolver_query_configuration.go

@@ -9,7 +9,6 @@ import (
 	"github.com/stashapp/stash/internal/manager/config"
 	"github.com/stashapp/stash/pkg/fsutil"
 	"github.com/stashapp/stash/pkg/models"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
 	"golang.org/x/text/collate"
 )
 
@@ -241,7 +240,7 @@ func makeConfigUIResult() map[string]interface{} {
 
 func (r *queryResolver) ValidateStashBoxCredentials(ctx context.Context, input config.StashBoxInput) (*StashBoxValidationResult, error) {
 	box := models.StashBox{Endpoint: input.Endpoint, APIKey: input.APIKey}
-	client := stashbox.NewClient(box, r.stashboxRepository())
+	client := r.newStashBoxClient(box)
 
 	user, err := client.GetUser(ctx)
 

internal/api/resolver_query_find_performer.go

@@ -32,6 +32,11 @@ func (r *queryResolver) FindPerformers(ctx context.Context, performerFilter *mod
 		}
 	}
 
+	// #5682 - convert JSON numbers to float64 or int64
+	if performerFilter != nil {
+		performerFilter.CustomFields = convertCustomFieldCriterionInputJSONNumbers(performerFilter.CustomFields)
+	}
+
 	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
 		var performers []*models.Performer
 		var err error

internal/api/resolver_query_find_scene_marker.go

@@ -4,14 +4,31 @@ import (
 	"context"
 
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
 )
 
-func (r *queryResolver) FindSceneMarkers(ctx context.Context, sceneMarkerFilter *models.SceneMarkerFilterType, filter *models.FindFilterType) (ret *FindSceneMarkersResultType, err error) {
+func (r *queryResolver) FindSceneMarkers(ctx context.Context, sceneMarkerFilter *models.SceneMarkerFilterType, filter *models.FindFilterType, ids []string) (ret *FindSceneMarkersResultType, err error) {
+	idInts, err := stringslice.StringSliceToIntSlice(ids)
+	if err != nil {
+		return nil, err
+	}
+
 	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
-		sceneMarkers, total, err := r.repository.SceneMarker.Query(ctx, sceneMarkerFilter, filter)
+		var sceneMarkers []*models.SceneMarker
+		var err error
+		var total int
+
+		if len(idInts) > 0 {
+			sceneMarkers, err = r.repository.SceneMarker.FindMany(ctx, idInts)
+			total = len(sceneMarkers)
+		} else {
+			sceneMarkers, total, err = r.repository.SceneMarker.Query(ctx, sceneMarkerFilter, filter)
+		}
+
 		if err != nil {
 			return err
 		}
+
 		ret = &FindSceneMarkersResultType{
 			Count:        total,
 			SceneMarkers: sceneMarkers,

internal/api/resolver_query_find_tag.go

@@ -62,7 +62,11 @@ func (r *queryResolver) FindTags(ctx context.Context, tagFilter *models.TagFilte
 func (r *queryResolver) AllTags(ctx context.Context) (ret []*models.Tag, err error) {
 	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
 		ret, err = r.repository.Tag.All(ctx)
-		return err
+		if err != nil {
+			return err
+		}
+
+		return nil
 	}); err != nil {
 		return nil, err
 	}

internal/api/resolver_query_scraper.go

@@ -4,15 +4,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"regexp"
+	"slices"
 	"strconv"
-	"strings"
 
-	"github.com/stashapp/stash/internal/manager"
-	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scraper"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
 	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
 )
@@ -34,7 +31,7 @@ func (r *queryResolver) ScrapePerformerURL(ctx context.Context, url string) (*mo
 	return marshalScrapedPerformer(content)
 }
 
-func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string, query string) ([]*scraper.ScrapedScene, error) {
+func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string, query string) ([]*models.ScrapedScene, error) {
 	if query == "" {
 		return nil, nil
 	}
@@ -49,119 +46,10 @@ func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string,
 		return nil, err
 	}
 
-	filterSceneTags(ret)
 	return ret, nil
 }
 
-func compileRegexps(patterns []string) []*regexp.Regexp {
-	excludePatterns := patterns
-	var excludeRegexps []*regexp.Regexp
-
-	for _, excludePattern := range excludePatterns {
-		reg, err := regexp.Compile(strings.ToLower(excludePattern))
-		if err != nil {
-			logger.Errorf("Invalid tag exclusion pattern: %v", err)
-		} else {
-			excludeRegexps = append(excludeRegexps, reg)
-		}
-	}
-
-	return excludeRegexps
-}
-
-// filterSceneTags removes tags matching excluded tag patterns from the provided scraped scenes
-func filterTags(excludeRegexps []*regexp.Regexp, tags []*models.ScrapedTag) (newTags []*models.ScrapedTag, ignoredTags []string) {
-	if len(excludeRegexps) == 0 {
-		return tags, nil
-	}
-
-	for _, t := range tags {
-		ignore := false
-		for _, reg := range excludeRegexps {
-			if reg.MatchString(strings.ToLower(t.Name)) {
-				ignore = true
-				ignoredTags = sliceutil.AppendUnique(ignoredTags, t.Name)
-				break
-			}
-		}
-
-		if !ignore {
-			newTags = append(newTags, t)
-		}
-	}
-
-	return
-}
-
-// filterSceneTags removes tags matching excluded tag patterns from the provided scraped scenes
-func filterSceneTags(scenes []*scraper.ScrapedScene) {
-	excludeRegexps := compileRegexps(manager.GetInstance().Config.GetScraperExcludeTagPatterns())
-
-	var ignoredTags []string
-
-	for _, s := range scenes {
-		var ignored []string
-		s.Tags, ignored = filterTags(excludeRegexps, s.Tags)
-		ignoredTags = sliceutil.AppendUniques(ignoredTags, ignored)
-	}
-
-	if len(ignoredTags) > 0 {
-		logger.Debugf("Scraping ignored tags: %s", strings.Join(ignoredTags, ", "))
-	}
-}
-
-// filterGalleryTags removes tags matching excluded tag patterns from the provided scraped galleries
-func filterGalleryTags(g []*scraper.ScrapedGallery) {
-	excludeRegexps := compileRegexps(manager.GetInstance().Config.GetScraperExcludeTagPatterns())
-
-	var ignoredTags []string
-
-	for _, s := range g {
-		var ignored []string
-		s.Tags, ignored = filterTags(excludeRegexps, s.Tags)
-		ignoredTags = sliceutil.AppendUniques(ignoredTags, ignored)
-	}
-
-	if len(ignoredTags) > 0 {
-		logger.Debugf("Scraping ignored tags: %s", strings.Join(ignoredTags, ", "))
-	}
-}
-
-// filterGalleryTags removes tags matching excluded tag patterns from the provided scraped galleries
-func filterPerformerTags(p []*models.ScrapedPerformer) {
-	excludeRegexps := compileRegexps(manager.GetInstance().Config.GetScraperExcludeTagPatterns())
-
-	var ignoredTags []string
-
-	for _, s := range p {
-		var ignored []string
-		s.Tags, ignored = filterTags(excludeRegexps, s.Tags)
-		ignoredTags = sliceutil.AppendUniques(ignoredTags, ignored)
-	}
-
-	if len(ignoredTags) > 0 {
-		logger.Debugf("Scraping ignored tags: %s", strings.Join(ignoredTags, ", "))
-	}
-}
-
-// filterGroupTags removes tags matching excluded tag patterns from the provided scraped movies
-func filterGroupTags(p []*models.ScrapedMovie) {
-	excludeRegexps := compileRegexps(manager.GetInstance().Config.GetScraperExcludeTagPatterns())
-
-	var ignoredTags []string
-
-	for _, s := range p {
-		var ignored []string
-		s.Tags, ignored = filterTags(excludeRegexps, s.Tags)
-		ignoredTags = sliceutil.AppendUniques(ignoredTags, ignored)
-	}
-
-	if len(ignoredTags) > 0 {
-		logger.Debugf("Scraping ignored tags: %s", strings.Join(ignoredTags, ", "))
-	}
-}
-
-func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*scraper.ScrapedScene, error) {
+func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models.ScrapedScene, error) {
 	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeScene)
 	if err != nil {
 		return nil, err
@@ -172,14 +60,10 @@ func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*scrape
 		return nil, err
 	}
 
-	if ret != nil {
-		filterSceneTags([]*scraper.ScrapedScene{ret})
-	}
-
 	return ret, nil
 }
 
-func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*scraper.ScrapedGallery, error) {
+func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*models.ScrapedGallery, error) {
 	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeGallery)
 	if err != nil {
 		return nil, err
@@ -190,11 +74,16 @@ func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*scra
 		return nil, err
 	}
 
-	if ret != nil {
-		filterGalleryTags([]*scraper.ScrapedGallery{ret})
+	return ret, nil
+}
+
+func (r *queryResolver) ScrapeImageURL(ctx context.Context, url string) (*models.ScrapedImage, error) {
+	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeImage)
+	if err != nil {
+		return nil, err
 	}
 
-	return ret, nil
+	return marshalScrapedImage(content)
 }
 
 func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models.ScrapedMovie, error) {
@@ -208,24 +97,20 @@ func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models
 		return nil, err
 	}
 
-	filterGroupTags([]*models.ScrapedMovie{ret})
-
 	return ret, nil
 }
 
 func (r *queryResolver) ScrapeGroupURL(ctx context.Context, url string) (*models.ScrapedGroup, error) {
-	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeMovie)
+	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeGroup)
 	if err != nil {
 		return nil, err
 	}
 
-	ret, err := marshalScrapedMovie(content)
+	ret, err := marshalScrapedGroup(content)
 	if err != nil {
 		return nil, err
 	}
 
-	filterGroupTags([]*models.ScrapedMovie{ret})
-
 	// convert to scraped group
 	group := &models.ScrapedGroup{
 		StoredID:   ret.StoredID,
@@ -246,8 +131,8 @@ func (r *queryResolver) ScrapeGroupURL(ctx context.Context, url string) (*models
 	return group, nil
 }
 
-func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.Source, input ScrapeSingleSceneInput) ([]*scraper.ScrapedScene, error) {
-	var ret []*scraper.ScrapedScene
+func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.Source, input ScrapeSingleSceneInput) ([]*models.ScrapedScene, error) {
+	var ret []*models.ScrapedScene
 
 	var sceneID int
 	if input.SceneID != nil {
@@ -299,9 +184,14 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.So
 
 		switch {
 		case input.SceneID != nil:
-			ret, err = client.FindStashBoxSceneByFingerprints(ctx, sceneID)
+			var fps []models.Fingerprints
+			fps, err = r.getScenesFingerprints(ctx, []int{sceneID})
+			if err != nil {
+				return nil, err
+			}
+			ret, err = client.FindSceneByFingerprints(ctx, fps[0])
 		case input.Query != nil:
-			ret, err = client.QueryStashBoxScene(ctx, *input.Query)
+			ret, err = client.QueryScene(ctx, *input.Query)
 		default:
 			return nil, fmt.Errorf("%w: scene_id or query must be set", ErrInput)
 		}
@@ -309,16 +199,23 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.So
 		if err != nil {
 			return nil, err
 		}
+
+		// TODO - this should happen after any scene is scraped
+		if err := r.matchScenesRelationships(ctx, ret, *source.StashBoxEndpoint); err != nil {
+			return nil, err
+		}
 	default:
 		return nil, fmt.Errorf("%w: scraper_id or stash_box_index must be set", ErrInput)
 	}
 
-	filterSceneTags(ret)
+	for i := range ret {
+		slices.SortFunc(ret[i].Tags, models.ScrapedTagSortFunction)
+	}
 
 	return ret, nil
 }
 
-func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source scraper.Source, input ScrapeMultiScenesInput) ([][]*scraper.ScrapedScene, error) {
+func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source scraper.Source, input ScrapeMultiScenesInput) ([][]*models.ScrapedScene, error) {
 	if source.ScraperID != nil {
 		return nil, ErrNotImplemented
 	} else if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
@@ -334,12 +231,89 @@ func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source scraper.So
 			return nil, err
 		}
 
-		return client.FindStashBoxScenesByFingerprints(ctx, sceneIDs)
+		fps, err := r.getScenesFingerprints(ctx, sceneIDs)
+		if err != nil {
+			return nil, err
+		}
+
+		ret, err := client.FindScenesByFingerprints(ctx, fps)
+		if err != nil {
+			return nil, err
+		}
+
+		// match relationships - this mutates the existing scenes so we can
+		// just flatten the slice and pass it in
+		flat := sliceutil.Flatten(ret)
+
+		if err := r.matchScenesRelationships(ctx, flat, *source.StashBoxEndpoint); err != nil {
+			return nil, err
+		}
+
+		return ret, nil
 	}
 
 	return nil, errors.New("scraper_id or stash_box_index must be set")
 }
 
+func (r *queryResolver) getScenesFingerprints(ctx context.Context, ids []int) ([]models.Fingerprints, error) {
+	fingerprints := make([]models.Fingerprints, len(ids))
+
+	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+		qb := r.repository.Scene
+
+		for i, sceneID := range ids {
+			scene, err := qb.Find(ctx, sceneID)
+			if err != nil {
+				return err
+			}
+
+			if scene == nil {
+				return fmt.Errorf("scene with id %d not found", sceneID)
+			}
+
+			if err := scene.LoadFiles(ctx, qb); err != nil {
+				return err
+			}
+
+			var sceneFPs models.Fingerprints
+
+			for _, f := range scene.Files.List() {
+				sceneFPs = append(sceneFPs, f.Fingerprints...)
+			}
+
+			fingerprints[i] = sceneFPs
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return fingerprints, nil
+}
+
+// matchSceneRelationships accepts scraped scenes and attempts to match its relationships to existing stash models.
+func (r *queryResolver) matchScenesRelationships(ctx context.Context, ss []*models.ScrapedScene, endpoint string) error {
+	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+		matcher := match.SceneRelationships{
+			PerformerFinder: r.repository.Performer,
+			TagFinder:       r.repository.Tag,
+			StudioFinder:    r.repository.Studio,
+		}
+
+		for _, s := range ss {
+			if err := matcher.MatchRelationships(ctx, s, endpoint); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (r *queryResolver) ScrapeSingleStudio(ctx context.Context, source scraper.Source, input ScrapeSingleStudioInput) ([]*models.ScrapedStudio, error) {
 	if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
 		b, err := resolveStashBox(source.StashBoxIndex, source.StashBoxEndpoint)
@@ -350,7 +324,7 @@ func (r *queryResolver) ScrapeSingleStudio(ctx context.Context, source scraper.S
 		client := r.newStashBoxClient(*b)
 
 		var ret []*models.ScrapedStudio
-		out, err := client.FindStashBoxStudio(ctx, *input.Query)
+		out, err := client.FindStudio(ctx, *input.Query)
 
 		if err != nil {
 			return nil, err
@@ -359,6 +333,17 @@ func (r *queryResolver) ScrapeSingleStudio(ctx context.Context, source scraper.S
 		}
 
 		if len(ret) > 0 {
+			if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+				for _, studio := range ret {
+					if err := match.ScrapedStudioHierarchy(ctx, r.repository.Studio, studio, *source.StashBoxEndpoint); err != nil {
+						return err
+					}
+				}
+
+				return nil
+			}); err != nil {
+				return nil, err
+			}
 			return ret, nil
 		}
 
@@ -404,29 +389,33 @@ func (r *queryResolver) ScrapeSinglePerformer(ctx context.Context, source scrape
 
 		client := r.newStashBoxClient(*b)
 
-		var res []*stashbox.StashBoxPerformerQueryResult
+		var query string
 		switch {
 		case input.PerformerID != nil:
-			res, err = client.FindStashBoxPerformersByNames(ctx, []string{*input.PerformerID})
+			names, err := r.findPerformerNames(ctx, []string{*input.PerformerID})
+			if err != nil {
+				return nil, err
+			}
+
+			query = names[0]
 		case input.Query != nil:
-			res, err = client.QueryStashBoxPerformer(ctx, *input.Query)
+			query = *input.Query
 		default:
 			return nil, ErrNotImplemented
 		}
 
+		if query == "" {
+			return nil, nil
+		}
+		ret, err = client.QueryPerformer(ctx, query)
+
 		if err != nil {
 			return nil, err
 		}
-
-		if len(res) > 0 {
-			ret = res[0].Results
-		}
 	default:
 		return nil, errors.New("scraper_id or stash_box_index must be set")
 	}
 
-	filterPerformerTags(ret)
-
 	return ret, nil
 }
 
@@ -434,6 +423,11 @@ func (r *queryResolver) ScrapeMultiPerformers(ctx context.Context, source scrape
 	if source.ScraperID != nil {
 		return nil, ErrNotImplemented
 	} else if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
+		names, err := r.findPerformerNames(ctx, input.PerformerIds)
+		if err != nil {
+			return nil, err
+		}
+
 		b, err := resolveStashBox(source.StashBoxIndex, source.StashBoxEndpoint)
 		if err != nil {
 			return nil, err
@@ -441,14 +435,40 @@ func (r *queryResolver) ScrapeMultiPerformers(ctx context.Context, source scrape
 
 		client := r.newStashBoxClient(*b)
 
-		return client.FindStashBoxPerformersByPerformerNames(ctx, input.PerformerIds)
+		return client.QueryPerformers(ctx, names)
 	}
 
 	return nil, errors.New("scraper_id or stash_box_index must be set")
 }
 
-func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.Source, input ScrapeSingleGalleryInput) ([]*scraper.ScrapedGallery, error) {
-	var ret []*scraper.ScrapedGallery
+func (r *queryResolver) findPerformerNames(ctx context.Context, performerIDs []string) ([]string, error) {
+	ids, err := stringslice.StringSliceToIntSlice(performerIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	names := make([]string, len(ids))
+
+	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+		p, err := r.repository.Performer.FindMany(ctx, ids)
+		if err != nil {
+			return err
+		}
+
+		for i, pp := range p {
+			names[i] = pp.Name
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return names, nil
+}
+
+func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.Source, input ScrapeSingleGalleryInput) ([]*models.ScrapedGallery, error) {
+	var ret []*models.ScrapedGallery
 
 	if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
 		return nil, ErrNotSupported
@@ -487,10 +507,42 @@ func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.
 		return nil, ErrNotImplemented
 	}
 
-	filterGalleryTags(ret)
 	return ret, nil
 }
 
+func (r *queryResolver) ScrapeSingleImage(ctx context.Context, source scraper.Source, input ScrapeSingleImageInput) ([]*models.ScrapedImage, error) {
+	if source.StashBoxIndex != nil {
+		return nil, ErrNotSupported
+	}
+
+	if source.ScraperID == nil {
+		return nil, fmt.Errorf("%w: scraper_id must be set", ErrInput)
+	}
+
+	var c scraper.ScrapedContent
+
+	switch {
+	case input.ImageID != nil:
+		imageID, err := strconv.Atoi(*input.ImageID)
+		if err != nil {
+			return nil, fmt.Errorf("%w: image id is not an integer: '%s'", ErrInput, *input.ImageID)
+		}
+		c, err = r.scraperCache().ScrapeID(ctx, *source.ScraperID, imageID, scraper.ScrapeContentTypeImage)
+		if err != nil {
+			return nil, err
+		}
+		return marshalScrapedImages([]scraper.ScrapedContent{c})
+	case input.ImageInput != nil:
+		c, err := r.scraperCache().ScrapeFragment(ctx, *source.ScraperID, scraper.Input{Image: input.ImageInput})
+		if err != nil {
+			return nil, err
+		}
+		return marshalScrapedImages([]scraper.ScrapedContent{c})
+	default:
+		return nil, ErrNotImplemented
+	}
+}
+
 func (r *queryResolver) ScrapeSingleMovie(ctx context.Context, source scraper.Source, input ScrapeSingleMovieInput) ([]*models.ScrapedMovie, error) {
 	return nil, ErrNotSupported
 }

internal/api/scraped_content.go

@@ -9,8 +9,8 @@ import (
 
 // marshalScrapedScenes converts ScrapedContent into ScrapedScene. If conversion fails, an
 // error is returned to the caller.
-func marshalScrapedScenes(content []scraper.ScrapedContent) ([]*scraper.ScrapedScene, error) {
-	var ret []*scraper.ScrapedScene
+func marshalScrapedScenes(content []scraper.ScrapedContent) ([]*models.ScrapedScene, error) {
+	var ret []*models.ScrapedScene
 	for _, c := range content {
 		if c == nil {
 			// graphql schema requires scenes to be non-nil
@@ -18,9 +18,9 @@ func marshalScrapedScenes(content []scraper.ScrapedContent) ([]*scraper.ScrapedS
 		}
 
 		switch s := c.(type) {
-		case *scraper.ScrapedScene:
+		case *models.ScrapedScene:
 			ret = append(ret, s)
-		case scraper.ScrapedScene:
+		case models.ScrapedScene:
 			ret = append(ret, &s)
 		default:
 			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedScene", models.ErrConversion)
@@ -55,8 +55,8 @@ func marshalScrapedPerformers(content []scraper.ScrapedContent) ([]*models.Scrap
 
 // marshalScrapedGalleries converts ScrapedContent into ScrapedGallery. If
 // conversion fails, an error is returned.
-func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*scraper.ScrapedGallery, error) {
-	var ret []*scraper.ScrapedGallery
+func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*models.ScrapedGallery, error) {
+	var ret []*models.ScrapedGallery
 	for _, c := range content {
 		if c == nil {
 			// graphql schema requires galleries to be non-nil
@@ -64,9 +64,9 @@ func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*scraper.Scrap
 		}
 
 		switch g := c.(type) {
-		case *scraper.ScrapedGallery:
+		case *models.ScrapedGallery:
 			ret = append(ret, g)
-		case scraper.ScrapedGallery:
+		case models.ScrapedGallery:
 			ret = append(ret, &g)
 		default:
 			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedGallery", models.ErrConversion)
@@ -76,6 +76,27 @@ func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*scraper.Scrap
 	return ret, nil
 }
 
+func marshalScrapedImages(content []scraper.ScrapedContent) ([]*models.ScrapedImage, error) {
+	var ret []*models.ScrapedImage
+	for _, c := range content {
+		if c == nil {
+			// graphql schema requires images to be non-nil
+			continue
+		}
+
+		switch g := c.(type) {
+		case *models.ScrapedImage:
+			ret = append(ret, g)
+		case models.ScrapedImage:
+			ret = append(ret, &g)
+		default:
+			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedImage", models.ErrConversion)
+		}
+	}
+
+	return ret, nil
+}
+
 // marshalScrapedMovies converts ScrapedContent into ScrapedMovie. If conversion
 // fails, an error is returned.
 func marshalScrapedMovies(content []scraper.ScrapedContent) ([]*models.ScrapedMovie, error) {
@@ -92,7 +113,30 @@ func marshalScrapedMovies(content []scraper.ScrapedContent) ([]*models.ScrapedMo
 		case models.ScrapedMovie:
 			ret = append(ret, &m)
 		default:
-			return nil, fmt.Errorf("%w: cannot turn ScrapedConetnt into ScrapedMovie", models.ErrConversion)
+			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedMovie", models.ErrConversion)
+		}
+	}
+
+	return ret, nil
+}
+
+// marshalScrapedMovies converts ScrapedContent into ScrapedMovie. If conversion
+// fails, an error is returned.
+func marshalScrapedGroups(content []scraper.ScrapedContent) ([]*models.ScrapedGroup, error) {
+	var ret []*models.ScrapedGroup
+	for _, c := range content {
+		if c == nil {
+			// graphql schema requires groups to be non-nil
+			continue
+		}
+
+		switch m := c.(type) {
+		case *models.ScrapedGroup:
+			ret = append(ret, m)
+		case models.ScrapedGroup:
+			ret = append(ret, &m)
+		default:
+			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedGroup", models.ErrConversion)
 		}
 	}
 
@@ -110,7 +154,7 @@ func marshalScrapedPerformer(content scraper.ScrapedContent) (*models.ScrapedPer
 }
 
 // marshalScrapedScene will marshal a single scraped scene
-func marshalScrapedScene(content scraper.ScrapedContent) (*scraper.ScrapedScene, error) {
+func marshalScrapedScene(content scraper.ScrapedContent) (*models.ScrapedScene, error) {
 	s, err := marshalScrapedScenes([]scraper.ScrapedContent{content})
 	if err != nil {
 		return nil, err
@@ -120,7 +164,7 @@ func marshalScrapedScene(content scraper.ScrapedContent) (*scraper.ScrapedScene,
 }
 
 // marshalScrapedGallery will marshal a single scraped gallery
-func marshalScrapedGallery(content scraper.ScrapedContent) (*scraper.ScrapedGallery, error) {
+func marshalScrapedGallery(content scraper.ScrapedContent) (*models.ScrapedGallery, error) {
 	g, err := marshalScrapedGalleries([]scraper.ScrapedContent{content})
 	if err != nil {
 		return nil, err
@@ -129,6 +173,16 @@ func marshalScrapedGallery(content scraper.ScrapedContent) (*scraper.ScrapedGall
 	return g[0], nil
 }
 
+// marshalScrapedImage will marshal a single scraped image
+func marshalScrapedImage(content scraper.ScrapedContent) (*models.ScrapedImage, error) {
+	g, err := marshalScrapedImages([]scraper.ScrapedContent{content})
+	if err != nil {
+		return nil, err
+	}
+
+	return g[0], nil
+}
+
 // marshalScrapedMovie will marshal a single scraped movie
 func marshalScrapedMovie(content scraper.ScrapedContent) (*models.ScrapedMovie, error) {
 	m, err := marshalScrapedMovies([]scraper.ScrapedContent{content})
@@ -138,3 +192,13 @@ func marshalScrapedMovie(content scraper.ScrapedContent) (*models.ScrapedMovie,
 
 	return m[0], nil
 }
+
+// marshalScrapedMovie will marshal a single scraped movie
+func marshalScrapedGroup(content scraper.ScrapedContent) (*models.ScrapedGroup, error) {
+	m, err := marshalScrapedGroups([]scraper.ScrapedContent{content})
+	if err != nil {
+		return nil, err
+	}
+
+	return m[0], nil
+}

internal/api/server.go

@@ -41,10 +41,11 @@ import (
 )
 
 const (
-	loginEndpoint      = "/login"
-	logoutEndpoint     = "/logout"
-	gqlEndpoint        = "/graphql"
-	playgroundEndpoint = "/playground"
+	loginEndpoint       = "/login"
+	loginLocaleEndpoint = loginEndpoint + "/locale"
+	logoutEndpoint      = "/logout"
+	gqlEndpoint         = "/graphql"
+	playgroundEndpoint  = "/playground"
 )
 
 type Server struct {
@@ -228,6 +229,7 @@ func Initialize() (*Server, error) {
 	r.Get(loginEndpoint, handleLogin())
 	r.Post(loginEndpoint, handleLoginPost())
 	r.Get(logoutEndpoint, handleLogout())
+	r.Get(loginLocaleEndpoint, handleLoginLocale(cfg))
 	r.HandleFunc(loginEndpoint+"/*", func(w http.ResponseWriter, r *http.Request) {
 		r.URL.Path = strings.TrimPrefix(r.URL.Path, loginEndpoint)
 		w.Header().Set("Cache-Control", "no-cache")

internal/api/session.go

@@ -17,7 +17,11 @@ import (
 	"github.com/stashapp/stash/ui"
 )
 
-const returnURLParam = "returnURL"
+const (
+	returnURLParam = "returnURL"
+
+	defaultLocale = "en-GB"
+)
 
 func getLoginPage() []byte {
 	data, err := fs.ReadFile(ui.LoginUIBox, "login.html")
@@ -58,6 +62,47 @@ func serveLoginPage(w http.ResponseWriter, r *http.Request, returnURL string, lo
 	utils.ServeStaticContent(w, r, buffer.Bytes())
 }
 
+func handleLoginLocale(cfg *config.Config) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// get the locale from the config
+		lang := cfg.GetLanguage()
+		if lang == "" {
+			lang = defaultLocale
+		}
+
+		data, err := getLoginLocale(lang)
+		if err != nil {
+			logger.Debugf("Failed to load login locale file for language %s: %v", lang, err)
+			// try again with the default language
+			if lang != defaultLocale {
+				data, err = getLoginLocale(defaultLocale)
+				if err != nil {
+					logger.Errorf("Failed to load login locale file for default language %s: %v", defaultLocale, err)
+				}
+			}
+
+			// if there's still an error, response with an internal server error
+			if err != nil {
+				http.Error(w, "Failed to load login locale file", http.StatusInternalServerError)
+				return
+			}
+		}
+
+		// write a script to set the locale string map as a global variable
+		localeScript := fmt.Sprintf("var localeStrings = %s;", data)
+		w.Header().Set("Content-Type", "application/javascript")
+		_, _ = w.Write([]byte(localeScript))
+	}
+}
+
+func getLoginLocale(lang string) ([]byte, error) {
+	data, err := fs.ReadFile(ui.LoginUIBox, "locales/"+lang+".json")
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
 func handleLogin() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		returnURL := r.URL.Query().Get(returnURLParam)
@@ -78,31 +123,26 @@ func handleLogin() http.HandlerFunc {
 
 func handleLoginPost() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		url := r.FormValue(returnURLParam)
-		if url == "" {
-			url = getProxyPrefix(r) + "/"
-		}
-
 		err := manager.GetInstance().SessionStore.Login(w, r)
 		if err != nil {
 			// always log the error
-			logger.Errorf("Error logging in: %v", err)
+			logger.Errorf("Error logging in: %v from IP: %s", err, r.RemoteAddr)
 		}
 
 		var invalidCredentialsError *session.InvalidCredentialsError
 
 		if errors.As(err, &invalidCredentialsError) {
-			// serve login page with an error
-			serveLoginPage(w, r, url, "Username or password is invalid")
+			http.Error(w, "Username or password is invalid", http.StatusUnauthorized)
 			return
 		}
 
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+			// don't expose the error to the user
+			http.Error(w, "An unexpected error occurred. See logs", http.StatusInternalServerError)
 			return
 		}
 
-		http.Redirect(w, r, url, http.StatusFound)
+		w.WriteHeader(http.StatusOK)
 	}
 }
 

internal/api/stash_box.go

@@ -4,13 +4,14 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/stashapp/stash/internal/manager"
 	"github.com/stashapp/stash/internal/manager/config"
 	"github.com/stashapp/stash/pkg/models"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
+	"github.com/stashapp/stash/pkg/stashbox"
 )
 
 func (r *Resolver) newStashBoxClient(box models.StashBox) *stashbox.Client {
-	return stashbox.NewClient(box, r.stashboxRepository())
+	return stashbox.NewClient(box, stashbox.ExcludeTagPatterns(manager.GetInstance().Config.GetScraperExcludeTagPatterns()))
 }
 
 func resolveStashBoxFn(indexField, endpointField string) func(index *int, endpoint *string) (*models.StashBox, error) {

internal/identify/identify.go

@@ -13,7 +13,6 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
-	"github.com/stashapp/stash/pkg/scraper"
 	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/txn"
 	"github.com/stashapp/stash/pkg/utils"
@@ -32,7 +31,7 @@ func (e *MultipleMatchesFoundError) Error() string {
 }
 
 type SceneScraper interface {
-	ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error)
+	ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error)
 }
 
 type SceneUpdatePostHookExecutor interface {
@@ -96,7 +95,7 @@ func (t *SceneIdentifier) Identify(ctx context.Context, scene *models.Scene) err
 }
 
 type scrapeResult struct {
-	result *scraper.ScrapedScene
+	result *models.ScrapedScene
 	source ScraperSource
 }
 
@@ -374,7 +373,7 @@ func getFieldOptions(options []MetadataOptions) map[string]*FieldOptions {
 	return ret
 }
 
-func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOptions map[string]*FieldOptions, setOrganized bool) models.ScenePartial {
+func getScenePartial(scene *models.Scene, scraped *models.ScrapedScene, fieldOptions map[string]*FieldOptions, setOrganized bool) models.ScenePartial {
 	partial := models.ScenePartial{}
 
 	if scraped.Title != nil && (scene.Title != *scraped.Title) {

internal/identify/identify_test.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/mocks"
-	"github.com/stashapp/stash/pkg/scraper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 )
@@ -19,10 +18,10 @@ var testCtx = context.Background()
 
 type mockSceneScraper struct {
 	errIDs  []int
-	results map[int][]*scraper.ScrapedScene
+	results map[int][]*models.ScrapedScene
 }
 
-func (s mockSceneScraper) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
+func (s mockSceneScraper) ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
 	if slices.Contains(s.errIDs, sceneID) {
 		return nil, errors.New("scrape scene error")
 	}
@@ -70,7 +69,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
 		{
 			Scraper: mockSceneScraper{
 				errIDs: []int{errID1},
-				results: map[int][]*scraper.ScrapedScene{
+				results: map[int][]*models.ScrapedScene{
 					found1ID: {{
 						Title: &scrapedTitle,
 					}},
@@ -80,7 +79,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
 		{
 			Scraper: mockSceneScraper{
 				errIDs: []int{errID2},
-				results: map[int][]*scraper.ScrapedScene{
+				results: map[int][]*models.ScrapedScene{
 					found2ID: {{
 						Title: &scrapedTitle,
 					}},
@@ -250,7 +249,7 @@ func TestSceneIdentifier_modifyScene(t *testing.T) {
 					StashIDs:     models.NewRelatedStashIDs([]models.StashID{}),
 				},
 				&scrapeResult{
-					result: &scraper.ScrapedScene{},
+					result: &models.ScrapedScene{},
 					source: ScraperSource{
 						Options: defaultOptions,
 					},
@@ -386,14 +385,14 @@ func Test_getScenePartial(t *testing.T) {
 		Mode:   models.RelationshipUpdateModeSet,
 	}
 
-	scrapedScene := &scraper.ScrapedScene{
+	scrapedScene := &models.ScrapedScene{
 		Title:   &scrapedTitle,
 		Date:    &scrapedDate,
 		Details: &scrapedDetails,
 		URLs:    []string{scrapedURL},
 	}
 
-	scrapedUnchangedScene := &scraper.ScrapedScene{
+	scrapedUnchangedScene := &models.ScrapedScene{
 		Title:   &originalTitle,
 		Date:    &originalDate,
 		Details: &originalDetails,
@@ -423,7 +422,7 @@ func Test_getScenePartial(t *testing.T) {
 
 	type args struct {
 		scene        *models.Scene
-		scraped      *scraper.ScrapedScene
+		scraped      *models.ScrapedScene
 		fieldOptions map[string]*FieldOptions
 		setOrganized bool
 	}

internal/identify/scene_test.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/mocks"
-	"github.com/stashapp/stash/pkg/scraper"
 	"github.com/stashapp/stash/pkg/utils"
 	"github.com/stretchr/testify/mock"
 )
@@ -125,7 +124,7 @@ func Test_sceneRelationships_studio(t *testing.T) {
 				source: ScraperSource{
 					RemoteSite: "endpoint",
 				},
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					Studio: tt.result,
 				},
 			}
@@ -315,7 +314,7 @@ func Test_sceneRelationships_performers(t *testing.T) {
 			tr.scene = tt.scene
 			tr.fieldOptions["performers"] = tt.fieldOptions
 			tr.result = &scrapeResult{
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					Performers: tt.scraped,
 				},
 			}
@@ -507,7 +506,7 @@ func Test_sceneRelationships_tags(t *testing.T) {
 			tr.scene = tt.scene
 			tr.fieldOptions["tags"] = tt.fieldOptions
 			tr.result = &scrapeResult{
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					Tags: tt.scraped,
 				},
 			}
@@ -727,7 +726,7 @@ func Test_sceneRelationships_stashIDs(t *testing.T) {
 				source: ScraperSource{
 					RemoteSite: tt.endpoint,
 				},
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					RemoteSiteID: tt.remoteSiteID,
 				},
 			}
@@ -827,7 +826,7 @@ func Test_sceneRelationships_cover(t *testing.T) {
 				ID: tt.sceneID,
 			}
 			tr.result = &scrapeResult{
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					Image: tt.image,
 				},
 			}

internal/manager/config/config.go

@@ -284,7 +284,7 @@ const (
 
 // slice default values
 var (
-	defaultVideoExtensions   = []string{"m4v", "mp4", "mov", "wmv", "avi", "mpg", "mpeg", "rmvb", "rm", "flv", "asf", "mkv", "webm"}
+	defaultVideoExtensions   = []string{"m4v", "mp4", "mov", "wmv", "avi", "mpg", "mpeg", "rmvb", "rm", "flv", "asf", "mkv", "webm", "f4v"}
 	defaultImageExtensions   = []string{"png", "jpg", "jpeg", "gif", "webp"}
 	defaultGalleryExtensions = []string{"zip", "cbz"}
 	defaultMenuItems         = []string{"scenes", "images", "movies", "markers", "galleries", "performers", "studios", "tags"}
@@ -1105,9 +1105,10 @@ func stashBoxValidate(str string) bool {
 }
 
 type StashBoxInput struct {
-	Endpoint string `json:"endpoint"`
-	APIKey   string `json:"api_key"`
-	Name     string `json:"name"`
+	Endpoint             string `json:"endpoint"`
+	APIKey               string `json:"api_key"`
+	Name                 string `json:"name"`
+	MaxRequestsPerMinute int    `json:"max_requests_per_minute"`
 }
 
 func (i *Config) ValidateStashBoxes(boxes []*StashBoxInput) error {

internal/manager/repository.go

@@ -14,6 +14,9 @@ type SceneService interface {
 	AssignFile(ctx context.Context, sceneID int, fileID models.FileID) error
 	Merge(ctx context.Context, sourceIDs []int, destinationID int, fileDeleter *scene.FileDeleter, options scene.MergeOptions) error
 	Destroy(ctx context.Context, scene *models.Scene, fileDeleter *scene.FileDeleter, deleteGenerated, deleteFile bool) error
+
+	FindByIDs(ctx context.Context, ids []int, load ...scene.LoadRelationshipOption) ([]*models.Scene, error)
+	sceneFingerprintGetter
 }
 
 type ImageService interface {

internal/manager/task/migrate.go

@@ -42,8 +42,8 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error
 
 	logger.Infof("Migrating database from %d to %d", schemaInfo.CurrentSchemaVersion, schemaInfo.RequiredSchemaVersion)
 
-	// set the number of tasks = required steps + optimise
-	progress.SetTotal(int(schemaInfo.StepsRequired + 1))
+	// set the number of tasks = backup + required steps + optimise
+	progress.SetTotal(int(schemaInfo.StepsRequired + 2))
 
 	database := s.Database
 
@@ -61,12 +61,20 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error
 		}
 	}
 
-	// perform database backup
-	if err := database.Backup(backupPath); err != nil {
+	progress.ExecuteTask("Backing up database", func() {
+		defer progress.Increment()
+
+		// perform database backup
+		err = database.Backup(backupPath)
+	})
+
+	if err != nil {
 		return fmt.Errorf("error backing up database: %s", err)
 	}
 
-	if err := s.runMigrations(ctx, progress); err != nil {
+	err = s.runMigrations(ctx, progress)
+
+	if err != nil {
 		errStr := fmt.Sprintf("error performing migration: %s", err)
 
 		// roll back to the backed up version
@@ -87,6 +95,11 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error
 		}
 	}
 
+	// reinitialise the database
+	if err := database.ReInitialise(); err != nil {
+		return fmt.Errorf("error reinitialising database: %s", err)
+	}
+
 	logger.Infof("Database migration complete")
 
 	return nil
@@ -124,6 +137,8 @@ func (s *MigrateJob) runMigrations(ctx context.Context, progress *job.Progress)
 
 	defer m.Close()
 
+	logger.Info("Running migrations")
+
 	for {
 		currentSchemaVersion := m.CurrentSchemaVersion()
 		targetSchemaVersion := m.RequiredSchemaVersion()
@@ -144,21 +159,15 @@ func (s *MigrateJob) runMigrations(ctx context.Context, progress *job.Progress)
 		progress.Increment()
 	}
 
-	// reinitialise the database
-	if err := database.ReInitialise(); err != nil {
-		return fmt.Errorf("error reinitialising database: %s", err)
-	}
-
-	// optimise the database
+	// perform post-migrate analyze using the migrator connection
 	progress.ExecuteTask("Optimising database", func() {
-		err = database.Optimise(ctx)
+		err = m.PostMigrate(ctx)
+		progress.Increment()
 	})
 
 	if err != nil {
 		return fmt.Errorf("error optimising database: %s", err)
 	}
 
-	progress.Increment()
-
 	return nil
 }

internal/manager/task_export.go

@@ -1042,23 +1042,43 @@ func (t *ExportTask) ExportTags(ctx context.Context, workers int) {
 	logger.Info("[tags] exporting")
 	startTime := time.Now()
 
-	jobCh := make(chan *models.Tag, workers*2) // make a buffered channel to feed workers
-
-	for w := 0; w < workers; w++ { // create export Tag workers
-		tagsWg.Add(1)
-		go t.exportTag(ctx, &tagsWg, jobCh)
+	tagIdx := 0
+	if t.tags != nil {
+		tagIdx = len(t.tags.IDs)
 	}
 
-	for i, tag := range tags {
-		index := i + 1
-		logger.Progressf("[tags] %d of %d", index, len(tags))
+	for {
+		jobCh := make(chan *models.Tag, workers*2) // make a buffered channel to feed workers
 
-		jobCh <- tag // feed workers
+		for w := 0; w < workers; w++ { // create export Tag workers
+			tagsWg.Add(1)
+			go t.exportTag(ctx, &tagsWg, jobCh)
+		}
+
+		for i, tag := range tags {
+			index := i + 1 + tagIdx
+			logger.Progressf("[tags] %d of %d", index, len(tags)+tagIdx)
+
+			jobCh <- tag // feed workers
+		}
+
+		close(jobCh)
+		tagsWg.Wait()
+
+		// if more tags were added, we need to export those too
+		if t.tags == nil || len(t.tags.IDs) == tagIdx {
+			break
+		}
+
+		newTags, err := reader.FindMany(ctx, t.tags.IDs[tagIdx:])
+		if err != nil {
+			logger.Errorf("[tags] failed to fetch tags: %v", err)
+		}
+
+		tags = newTags
+		tagIdx = len(t.tags.IDs)
 	}
 
-	close(jobCh)
-	tagsWg.Wait()
-
 	logger.Infof("[tags] export complete in %s. %d workers used.", time.Since(startTime), workers)
 }
 
@@ -1075,6 +1095,15 @@ func (t *ExportTask) exportTag(ctx context.Context, wg *sync.WaitGroup, jobChan
 			continue
 		}
 
+		if t.includeDependencies {
+			tagIDs, err := tag.GetDependentTagIDs(ctx, tagReader, thisTag)
+			if err != nil {
+				logger.Errorf("[tags] <%s> error getting dependent tags: %v", thisTag.Name, err)
+				continue
+			}
+			t.tags.IDs = sliceutil.AppendUniques(t.tags.IDs, tagIDs)
+		}
+
 		fn := newTagJSON.Filename()
 
 		if err := t.json.saveTag(fn, newTagJSON); err != nil {

internal/manager/task_generate_markers.go

@@ -105,11 +105,11 @@ func (t *GenerateMarkersTask) generateSceneMarkers(ctx context.Context) {
 
 func (t *GenerateMarkersTask) generateMarker(videoFile *models.VideoFile, scene *models.Scene, sceneMarker *models.SceneMarker) {
 	sceneHash := scene.GetHash(t.fileNamingAlgorithm)
-	seconds := int(sceneMarker.Seconds)
+	seconds := float64(sceneMarker.Seconds)
 
 	g := t.generator
 
-	if err := g.MarkerPreviewVideo(context.TODO(), videoFile.Path, sceneHash, seconds, instance.Config.GetPreviewAudio()); err != nil {
+	if err := g.MarkerPreviewVideo(context.TODO(), videoFile.Path, sceneHash, seconds, sceneMarker.EndSeconds, instance.Config.GetPreviewAudio()); err != nil {
 		logger.Errorf("[generator] failed to generate marker video: %v", err)
 		logErrorOutput(err)
 	}

internal/manager/task_identify.go

@@ -9,11 +9,13 @@ import (
 	"github.com/stashapp/stash/internal/identify"
 	"github.com/stashapp/stash/pkg/job"
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/scraper"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
+	"github.com/stashapp/stash/pkg/stashbox"
+	"github.com/stashapp/stash/pkg/txn"
 )
 
 var ErrInput = errors.New("invalid request input")
@@ -169,12 +171,20 @@ func (j *IdentifyJob) getSources() ([]identify.ScraperSource, error) {
 
 		var src identify.ScraperSource
 		if stashBox != nil {
-			stashboxRepository := stashbox.NewRepository(instance.Repository)
+			matcher := match.SceneRelationships{
+				PerformerFinder: instance.Repository.Performer,
+				TagFinder:       instance.Repository.Tag,
+				StudioFinder:    instance.Repository.Studio,
+			}
+
 			src = identify.ScraperSource{
 				Name: "stash-box: " + stashBox.Endpoint,
 				Scraper: stashboxSource{
-					stashbox.NewClient(*stashBox, stashboxRepository),
-					stashBox.Endpoint,
+					Client:                 stashbox.NewClient(*stashBox, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns())),
+					endpoint:               stashBox.Endpoint,
+					txnManager:             instance.Repository.TxnManager,
+					sceneFingerprintGetter: instance.SceneService,
+					matcher:                matcher,
 				},
 				RemoteSite: stashBox.Endpoint,
 			}
@@ -247,14 +257,42 @@ func resolveStashBox(sb []*models.StashBox, source scraper.Source) (*models.Stas
 type stashboxSource struct {
 	*stashbox.Client
 	endpoint string
+
+	txnManager             models.TxnManager
+	sceneFingerprintGetter sceneFingerprintGetter
+	matcher                match.SceneRelationships
 }
 
-func (s stashboxSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
-	results, err := s.FindStashBoxSceneByFingerprints(ctx, sceneID)
+type sceneFingerprintGetter interface {
+	GetScenesFingerprints(ctx context.Context, ids []int) ([]models.Fingerprints, error)
+}
+
+func (s stashboxSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
+	var fps []models.Fingerprints
+	if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
+		var err error
+		fps, err = s.sceneFingerprintGetter.GetScenesFingerprints(ctx, []int{sceneID})
+		return err
+	}); err != nil {
+		return nil, fmt.Errorf("error getting scene fingerprints: %w", err)
+	}
+
+	results, err := s.FindSceneByFingerprints(ctx, fps[0])
 	if err != nil {
 		return nil, fmt.Errorf("error querying stash-box using scene ID %d: %w", sceneID, err)
 	}
 
+	if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
+		for _, ret := range results {
+			if err := s.matcher.MatchRelationships(ctx, ret, s.endpoint); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return nil, fmt.Errorf("error matching scene relationships: %w", err)
+	}
+
 	if len(results) > 0 {
 		return results, nil
 	}
@@ -271,7 +309,7 @@ type scraperSource struct {
 	scraperID string
 }
 
-func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
+func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
 	content, err := s.cache.ScrapeID(ctx, s.scraperID, sceneID, scraper.ScrapeContentTypeScene)
 	if err != nil {
 		return nil, err
@@ -282,8 +320,8 @@ func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scrape
 		return nil, nil
 	}
 
-	if scene, ok := content.(scraper.ScrapedScene); ok {
-		return []*scraper.ScrapedScene{&scene}, nil
+	if scene, ok := content.(models.ScrapedScene); ok {
+		return []*models.ScrapedScene{&scene}, nil
 	}
 
 	return nil, errors.New("could not convert content to scene")

internal/manager/task_import.go

@@ -709,6 +709,11 @@ func (t *ImportTask) ImportScenes(ctx context.Context) {
 				return err
 			}
 
+			// skip importing markers if the scene was not created
+			if sceneImporter.ID == 0 {
+				return nil
+			}
+
 			// import the scene markers
 			for _, m := range sceneJSON.Markers {
 				markerImporter := &scene.MarkerImporter{

internal/manager/task_stash_box_tag.go

@@ -6,9 +6,11 @@ import (
 	"strconv"
 
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/performer"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
+	"github.com/stashapp/stash/pkg/sliceutil"
+	"github.com/stashapp/stash/pkg/stashbox"
 	"github.com/stashapp/stash/pkg/studio"
 )
 
@@ -94,8 +96,7 @@ func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*mode
 
 	r := instance.Repository
 
-	stashboxRepository := stashbox.NewRepository(r)
-	client := stashbox.NewClient(*t.box, stashboxRepository)
+	client := stashbox.NewClient(*t.box, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns()))
 
 	if t.refresh {
 		var remoteID string
@@ -118,7 +119,19 @@ func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*mode
 			return nil, err
 		}
 		if remoteID != "" {
-			performer, err = client.FindStashBoxPerformerByID(ctx, remoteID)
+			performer, err = client.FindPerformerByID(ctx, remoteID)
+
+			if performer != nil && performer.RemoteMergedIntoId != nil {
+				mergedPerformer, err := t.handleMergedPerformer(ctx, performer, client)
+				if err != nil {
+					return nil, err
+				}
+
+				if mergedPerformer != nil {
+					logger.Infof("Performer id %s merged into %s, updating local performer", remoteID, *performer.RemoteMergedIntoId)
+					performer = mergedPerformer
+				}
+			}
 		}
 	} else {
 		var name string
@@ -127,12 +140,35 @@ func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*mode
 		} else {
 			name = t.performer.Name
 		}
-		performer, err = client.FindStashBoxPerformerByName(ctx, name)
+		performer, err = client.FindPerformerByName(ctx, name)
+	}
+
+	if performer != nil {
+		if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
+			return match.ScrapedPerformer(ctx, r.Performer, performer, t.box.Endpoint)
+		}); err != nil {
+			return nil, err
+		}
 	}
 
 	return performer, err
 }
 
+func (t *StashBoxBatchTagTask) handleMergedPerformer(ctx context.Context, performer *models.ScrapedPerformer, client *stashbox.Client) (mergedPerformer *models.ScrapedPerformer, err error) {
+	mergedPerformer, err = client.FindPerformerByID(ctx, *performer.RemoteMergedIntoId)
+	if err != nil {
+		return nil, fmt.Errorf("loading merged performer %s from stashbox", *performer.RemoteMergedIntoId)
+	}
+
+	if mergedPerformer.StoredID != nil && *mergedPerformer.StoredID != *performer.StoredID {
+		logger.Warnf("Performer %s merged into %s, but both exist locally, not merging", *performer.StoredID, *mergedPerformer.StoredID)
+		return nil, nil
+	}
+
+	mergedPerformer.StoredID = performer.StoredID
+	return mergedPerformer, nil
+}
+
 func (t *StashBoxBatchTagTask) processMatchedPerformer(ctx context.Context, p *models.ScrapedPerformer, excluded map[string]bool) {
 	// Refreshing an existing performer
 	if t.performer != nil {
@@ -156,6 +192,19 @@ func (t *StashBoxBatchTagTask) processMatchedPerformer(ctx context.Context, p *m
 
 			partial := p.ToPartial(t.box.Endpoint, excluded, existingStashIDs)
 
+			// if we're setting the performer's aliases, and not the name, then filter out the name
+			// from the aliases to avoid duplicates
+			// add the name to the aliases if it's not already there
+			if partial.Aliases != nil && !partial.Name.Set {
+				partial.Aliases.Values = sliceutil.Filter(partial.Aliases.Values, func(s string) bool {
+					return s != t.performer.Name
+				})
+
+				if p.Name != nil && t.performer.Name != *p.Name {
+					partial.Aliases.Values = sliceutil.AppendUnique(partial.Aliases.Values, *p.Name)
+				}
+			}
+
 			if err := performer.ValidateUpdate(ctx, t.performer.ID, partial, qb); err != nil {
 				return err
 			}
@@ -246,8 +295,7 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
 
 	r := instance.Repository
 
-	stashboxRepository := stashbox.NewRepository(r)
-	client := stashbox.NewClient(*t.box, stashboxRepository)
+	client := stashbox.NewClient(*t.box, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns()))
 
 	if t.refresh {
 		var remoteID string
@@ -268,7 +316,7 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
 			return nil, err
 		}
 		if remoteID != "" {
-			studio, err = client.FindStashBoxStudio(ctx, remoteID)
+			studio, err = client.FindStudio(ctx, remoteID)
 		}
 	} else {
 		var name string
@@ -277,7 +325,19 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
 		} else {
 			name = t.studio.Name
 		}
-		studio, err = client.FindStashBoxStudio(ctx, name)
+		studio, err = client.FindStudio(ctx, name)
+	}
+
+	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
+		if studio != nil {
+			if err := match.ScrapedStudioHierarchy(ctx, r.Studio, studio, t.box.Endpoint); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
 	}
 
 	return studio, err

pkg/ffmpeg/stream_segmented.go

@@ -426,9 +426,11 @@ func serveHLSManifest(sm *StreamManager, w http.ResponseWriter, r *http.Request,
 		return
 	}
 
+	prefix := r.Header.Get("X-Forwarded-Prefix")
+
 	baseUrl := *r.URL
 	baseUrl.RawQuery = ""
-	baseURL := baseUrl.String()
+	baseURL := prefix + baseUrl.String()
 
 	urlQuery := url.Values{}
 	apikey := r.URL.Query().Get(apiKeyParamKey)
@@ -559,9 +561,11 @@ func serveDASHManifest(sm *StreamManager, w http.ResponseWriter, r *http.Request
 	mediaDuration := mpd.Duration(time.Duration(probeResult.FileDuration * float64(time.Second)))
 	m := mpd.NewMPD(mpd.DASH_PROFILE_LIVE, mediaDuration.String(), "PT4.0S")
 
+	prefix := r.Header.Get("X-Forwarded-Prefix")
+
 	baseUrl := r.URL.JoinPath("/")
 	baseUrl.RawQuery = ""
-	m.BaseURL = baseUrl.String()
+	m.BaseURL = prefix + baseUrl.String()
 
 	video, _ := m.AddNewAdaptationSetVideo(MimeWebmVideo, "progressive", true, 1)
 

pkg/fsutil/file.go

@@ -1,6 +1,8 @@
 package fsutil
 
 import (
+	"crypto/sha1"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
@@ -151,7 +153,12 @@ var (
 )
 
 // SanitiseBasename returns a file basename removing any characters that are illegal or problematic to use in the filesystem.
+// It appends a short hash of the original string to ensure uniqueness.
 func SanitiseBasename(v string) string {
+	// Generate a short hash for uniqueness
+	hash := sha1.Sum([]byte(v))
+	shortHash := hex.EncodeToString(hash[:4]) // Use the first 4 bytes of the hash
+
 	v = strings.TrimSpace(v)
 
 	// replace illegal filename characters with -
@@ -163,7 +170,7 @@ func SanitiseBasename(v string) string {
 	// remove multiple hyphens
 	v = multiHyphenRE.ReplaceAllString(v, "-")
 
-	return strings.TrimSpace(v)
+	return strings.TrimSpace(v) + "-" + shortHash
 }
 
 // GetExeName returns the name of the given executable for the current platform.

pkg/fsutil/file_test.go

@@ -8,13 +8,13 @@ func TestSanitiseBasename(t *testing.T) {
 		v    string
 		want string
 	}{
-		{"basic", "basic", "basic"},
-		{"spaces", `spaced name`, "spaced-name"},
-		{"leading/trailing spaces", `  spaced name  `, "spaced-name"},
-		{"hyphen name", `hyphened-name`, "hyphened-name"},
-		{"multi-hyphen", `hyphened--name`, "hyphened-name"},
-		{"replaced characters", `a&b=c\d/:e*"f?_ g`, "a-b-c-d-e-f-g"},
-		{"removed characters", `foo!!bar@@and, more`, "foobarand-more"},
+		{"basic", "basic", "basic-61a7508e"},
+		{"spaces", `spaced name`, "spaced-name-b297cf60"},
+		{"leading/trailing spaces", `  spaced name  `, "spaced-name-175433e9"},
+		{"hyphen name", `hyphened-name`, "hyphened-name-789c55f2"},
+		{"multi-hyphen", `hyphened--name`, "hyphened-name-2da2a58f"},
+		{"replaced characters", `a&b=c\d/:e*"f?_ g`, "a-b-c-d-e-f-g-ffca6fb0"},
+		{"removed characters", `foo!!bar@@and, more`, "foobarand-more-7cee02ab"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/match/scraped.go

@@ -20,18 +20,52 @@ type GroupNamesFinder interface {
 	FindByNames(ctx context.Context, names []string, nocase bool) ([]*models.Group, error)
 }
 
+type SceneRelationships struct {
+	PerformerFinder PerformerFinder
+	TagFinder       models.TagQueryer
+	StudioFinder    StudioFinder
+}
+
+// MatchRelationships accepts a scraped scene and attempts to match its relationships to existing stash models.
+func (r SceneRelationships) MatchRelationships(ctx context.Context, s *models.ScrapedScene, endpoint string) error {
+	thisStudio := s.Studio
+	for thisStudio != nil {
+		if err := ScrapedStudio(ctx, r.StudioFinder, thisStudio, endpoint); err != nil {
+			return err
+		}
+
+		thisStudio = thisStudio.Parent
+	}
+
+	for _, p := range s.Performers {
+		err := ScrapedPerformer(ctx, r.PerformerFinder, p, endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, t := range s.Tags {
+		err := ScrapedTag(ctx, r.TagFinder, t)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // ScrapedPerformer matches the provided performer with the
 // performers in the database and sets the ID field if one is found.
-func ScrapedPerformer(ctx context.Context, qb PerformerFinder, p *models.ScrapedPerformer, stashBoxEndpoint *string) error {
+func ScrapedPerformer(ctx context.Context, qb PerformerFinder, p *models.ScrapedPerformer, stashBoxEndpoint string) error {
 	if p.StoredID != nil || p.Name == nil {
 		return nil
 	}
 
 	// Check if a performer with the StashID already exists
-	if stashBoxEndpoint != nil && p.RemoteSiteID != nil {
+	if stashBoxEndpoint != "" && p.RemoteSiteID != nil {
 		performers, err := qb.FindByStashID(ctx, models.StashID{
 			StashID:  *p.RemoteSiteID,
-			Endpoint: *stashBoxEndpoint,
+			Endpoint: stashBoxEndpoint,
 		})
 		if err != nil {
 			return err
@@ -73,16 +107,16 @@ type StudioFinder interface {
 
 // ScrapedStudio matches the provided studio with the studios
 // in the database and sets the ID field if one is found.
-func ScrapedStudio(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio, stashBoxEndpoint *string) error {
+func ScrapedStudio(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio, stashBoxEndpoint string) error {
 	if s.StoredID != nil {
 		return nil
 	}
 
 	// Check if a studio with the StashID already exists
-	if stashBoxEndpoint != nil && s.RemoteSiteID != nil {
+	if stashBoxEndpoint != "" && s.RemoteSiteID != nil {
 		studios, err := qb.FindByStashID(ctx, models.StashID{
 			StashID:  *s.RemoteSiteID,
-			Endpoint: *stashBoxEndpoint,
+			Endpoint: stashBoxEndpoint,
 		})
 		if err != nil {
 			return err
@@ -118,6 +152,19 @@ func ScrapedStudio(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio
 	return nil
 }
 
+// ScrapedStudioHierarchy executes ScrapedStudio for the provided studio and its parents recursively.
+func ScrapedStudioHierarchy(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio, stashBoxEndpoint string) error {
+	if err := ScrapedStudio(ctx, qb, s, stashBoxEndpoint); err != nil {
+		return err
+	}
+
+	if s.Parent == nil {
+		return nil
+	}
+
+	return ScrapedStudioHierarchy(ctx, qb, s.Parent, stashBoxEndpoint)
+}
+
 // ScrapedGroup matches the provided movie with the movies
 // in the database and returns the ID field if one is found.
 func ScrapedGroup(ctx context.Context, qb GroupNamesFinder, storedID *string, name *string) (matchedID *string, err error) {

pkg/models/fingerprint.go

@@ -26,6 +26,20 @@ func (f *Fingerprint) Value() string {
 	}
 }
 
+// String returns the string representation of the Fingerprint.
+// It will return an empty string if the Fingerprint is not a string.
+func (f Fingerprint) String() string {
+	s, _ := f.Fingerprint.(string)
+	return s
+}
+
+// Int64 returns the int64 representation of the Fingerprint.
+// It will return 0 if the Fingerprint is not an int64.
+func (f Fingerprint) Int64() int64 {
+	v, _ := f.Fingerprint.(int64)
+	return v
+}
+
 type Fingerprints []Fingerprint
 
 func (f Fingerprints) Remove(type_ string) Fingerprints {
@@ -102,33 +116,27 @@ func (f Fingerprints) For(type_ string) *Fingerprint {
 }
 
 func (f Fingerprints) Get(type_ string) interface{} {
-	for _, fp := range f {
-		if fp.Type == type_ {
-			return fp.Fingerprint
-		}
+	fp := f.For(type_)
+	if fp == nil {
+		return nil
 	}
-
-	return nil
+	return fp.Fingerprint
 }
 
 func (f Fingerprints) GetString(type_ string) string {
-	fp := f.Get(type_)
-	if fp != nil {
-		s, _ := fp.(string)
-		return s
+	fp := f.For(type_)
+	if fp == nil {
+		return ""
 	}
-
-	return ""
+	return fp.String()
 }
 
 func (f Fingerprints) GetInt64(type_ string) int64 {
-	fp := f.Get(type_)
+	fp := f.For(type_)
 	if fp != nil {
-		v, _ := fp.(int64)
-		return v
+		return 0
 	}
-
-	return 0
+	return fp.Int64()
 }
 
 // AppendUnique appends a fingerprint to the list if a Fingerprint of the same type does not already exist in the list. If one does, then it is updated with o's Fingerprint value.

pkg/models/image.go

@@ -63,6 +63,28 @@ type ImageFilterType struct {
 	UpdatedAt *TimestampCriterionInput `json:"updated_at"`
 }
 
+type ImageUpdateInput struct {
+	ClientMutationID *string  `json:"clientMutationId"`
+	ID               string   `json:"id"`
+	Title            *string  `json:"title"`
+	Code             *string  `json:"code"`
+	Urls             []string `json:"urls"`
+	Date             *string  `json:"date"`
+	Details          *string  `json:"details"`
+	Photographer     *string  `json:"photographer"`
+	Rating100        *int     `json:"rating100"`
+	Organized        *bool    `json:"organized"`
+	SceneIds         []string `json:"scene_ids"`
+	StudioID         *string  `json:"studio_id"`
+	TagIds           []string `json:"tag_ids"`
+	PerformerIds     []string `json:"performer_ids"`
+	GalleryIds       []string `json:"gallery_ids"`
+	PrimaryFileID    *string  `json:"primary_file_id"`
+
+	// deprecated
+	URL *string `json:"url"`
+}
+
 type ImageDestroyInput struct {
 	ID              string `json:"id"`
 	DeleteFile      *bool  `json:"delete_file"`

pkg/models/jsonschema/scene.go

@@ -14,6 +14,7 @@ import (
 type SceneMarker struct {
 	Title      string        `json:"title,omitempty"`
 	Seconds    string        `json:"seconds,omitempty"`
+	EndSeconds string        `json:"end_seconds,omitempty"`
 	PrimaryTag string        `json:"primary_tag,omitempty"`
 	Tags       []string      `json:"tags,omitempty"`
 	CreatedAt  json.JSONTime `json:"created_at,omitempty"`

pkg/models/jsonschema/tag.go

@@ -11,6 +11,7 @@ import (
 
 type Tag struct {
 	Name          string        `json:"name,omitempty"`
+	SortName      string        `json:"sort_name,omitempty"`
 	Description   string        `json:"description,omitempty"`
 	Favorite      bool          `json:"favorite,omitempty"`
 	Aliases       []string      `json:"aliases,omitempty"`

pkg/models/mocks/SceneReaderWriter.go

@@ -549,6 +549,29 @@ func (_m *SceneReaderWriter) FindByGroupID(ctx context.Context, groupID int) ([]
 	return r0, r1
 }
 
+// FindByIDs provides a mock function with given fields: ctx, ids
+func (_m *SceneReaderWriter) FindByIDs(ctx context.Context, ids []int) ([]*models.Scene, error) {
+	ret := _m.Called(ctx, ids)
+
+	var r0 []*models.Scene
+	if rf, ok := ret.Get(0).(func(context.Context, []int) []*models.Scene); ok {
+		r0 = rf(ctx, ids)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*models.Scene)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, []int) error); ok {
+		r1 = rf(ctx, ids)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // FindByOSHash provides a mock function with given fields: ctx, oshash
 func (_m *SceneReaderWriter) FindByOSHash(ctx context.Context, oshash string) ([]*models.Scene, error) {
 	ret := _m.Called(ctx, oshash)

pkg/models/mocks/query.go

@@ -18,6 +18,10 @@ func (s *sceneResolver) FindMany(ctx context.Context, ids []int) ([]*models.Scen
 	return s.scenes, nil
 }
 
+func (s *sceneResolver) FindByIDs(ctx context.Context, ids []int) ([]*models.Scene, error) {
+	return s.scenes, nil
+}
+
 func SceneQueryResult(scenes []*models.Scene, count int) *models.SceneQueryResult {
 	ret := models.NewSceneQueryResult(&sceneResolver{
 		scenes: scenes,

pkg/models/model_scraped_item.go

@@ -3,6 +3,7 @@ package models
 import (
 	"context"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
@@ -128,13 +129,15 @@ type ScrapedPerformer struct {
 	Aliases        *string       `json:"aliases"`
 	Tags           []*ScrapedTag `json:"tags"`
 	// This should be a base64 encoded data URL
-	Image        *string  `json:"image"` // deprecated: use Images
-	Images       []string `json:"images"`
-	Details      *string  `json:"details"`
-	DeathDate    *string  `json:"death_date"`
-	HairColor    *string  `json:"hair_color"`
-	Weight       *string  `json:"weight"`
-	RemoteSiteID *string  `json:"remote_site_id"`
+	Image              *string  `json:"image"` // deprecated: use Images
+	Images             []string `json:"images"`
+	Details            *string  `json:"details"`
+	DeathDate          *string  `json:"death_date"`
+	HairColor          *string  `json:"hair_color"`
+	Weight             *string  `json:"weight"`
+	RemoteSiteID       *string  `json:"remote_site_id"`
+	RemoteDeleted      bool     `json:"remote_deleted"`
+	RemoteMergedIntoId *string  `json:"remote_merged_into_id"`
 }
 
 func (ScrapedPerformer) IsScrapedContent() {}
@@ -398,6 +401,10 @@ type ScrapedTag struct {
 
 func (ScrapedTag) IsScrapedContent() {}
 
+func ScrapedTagSortFunction(a, b *ScrapedTag) int {
+	return strings.Compare(strings.ToLower(a.Name), strings.ToLower(b.Name))
+}
+
 // A movie from a scraping operation...
 type ScrapedMovie struct {
 	StoredID *string        `json:"stored_id"`
@@ -490,3 +497,88 @@ func (g ScrapedGroup) ScrapedMovie() ScrapedMovie {
 
 	return ret
 }
+
+type ScrapedScene struct {
+	Title    *string  `json:"title"`
+	Code     *string  `json:"code"`
+	Details  *string  `json:"details"`
+	Director *string  `json:"director"`
+	URL      *string  `json:"url"`
+	URLs     []string `json:"urls"`
+	Date     *string  `json:"date"`
+	// This should be a base64 encoded data URL
+	Image        *string                `json:"image"`
+	File         *SceneFileType         `json:"file"`
+	Studio       *ScrapedStudio         `json:"studio"`
+	Tags         []*ScrapedTag          `json:"tags"`
+	Performers   []*ScrapedPerformer    `json:"performers"`
+	Groups       []*ScrapedGroup        `json:"groups"`
+	Movies       []*ScrapedMovie        `json:"movies"`
+	RemoteSiteID *string                `json:"remote_site_id"`
+	Duration     *int                   `json:"duration"`
+	Fingerprints []*StashBoxFingerprint `json:"fingerprints"`
+}
+
+func (ScrapedScene) IsScrapedContent() {}
+
+type ScrapedSceneInput struct {
+	Title        *string  `json:"title"`
+	Code         *string  `json:"code"`
+	Details      *string  `json:"details"`
+	Director     *string  `json:"director"`
+	URL          *string  `json:"url"`
+	URLs         []string `json:"urls"`
+	Date         *string  `json:"date"`
+	RemoteSiteID *string  `json:"remote_site_id"`
+}
+
+type ScrapedImage struct {
+	Title        *string             `json:"title"`
+	Code         *string             `json:"code"`
+	Details      *string             `json:"details"`
+	Photographer *string             `json:"photographer"`
+	URLs         []string            `json:"urls"`
+	Date         *string             `json:"date"`
+	Studio       *ScrapedStudio      `json:"studio"`
+	Tags         []*ScrapedTag       `json:"tags"`
+	Performers   []*ScrapedPerformer `json:"performers"`
+}
+
+func (ScrapedImage) IsScrapedContent() {}
+
+type ScrapedImageInput struct {
+	Title   *string  `json:"title"`
+	Code    *string  `json:"code"`
+	Details *string  `json:"details"`
+	URLs    []string `json:"urls"`
+	Date    *string  `json:"date"`
+}
+
+type ScrapedGallery struct {
+	Title        *string             `json:"title"`
+	Code         *string             `json:"code"`
+	Details      *string             `json:"details"`
+	Photographer *string             `json:"photographer"`
+	URLs         []string            `json:"urls"`
+	Date         *string             `json:"date"`
+	Studio       *ScrapedStudio      `json:"studio"`
+	Tags         []*ScrapedTag       `json:"tags"`
+	Performers   []*ScrapedPerformer `json:"performers"`
+
+	// deprecated
+	URL *string `json:"url"`
+}
+
+func (ScrapedGallery) IsScrapedContent() {}
+
+type ScrapedGalleryInput struct {
+	Title        *string  `json:"title"`
+	Code         *string  `json:"code"`
+	Details      *string  `json:"details"`
+	Photographer *string  `json:"photographer"`
+	URLs         []string `json:"urls"`
+	Date         *string  `json:"date"`
+
+	// deprecated
+	URL *string `json:"url"`
+}

pkg/models/model_tag.go

@@ -8,6 +8,7 @@ import (
 type Tag struct {
 	ID            int       `json:"id"`
 	Name          string    `json:"name"`
+	SortName      string    `json:"sort_name"`
 	Favorite      bool      `json:"favorite"`
 	Description   string    `json:"description"`
 	IgnoreAutoTag bool      `json:"ignore_auto_tag"`
@@ -47,6 +48,7 @@ func (s *Tag) LoadChildIDs(ctx context.Context, l TagRelationLoader) error {
 
 type TagPartial struct {
 	Name          OptionalString
+	SortName      OptionalString
 	Description   OptionalString
 	Favorite      OptionalBool
 	IgnoreAutoTag OptionalBool

pkg/models/repository_scene.go

@@ -10,6 +10,9 @@ type SceneGetter interface {
 	// TODO - rename this to Find and remove existing method
 	FindMany(ctx context.Context, ids []int) ([]*Scene, error)
 	Find(ctx context.Context, id int) (*Scene, error)
+	// FindByIDs works the same way as FindMany, but it ignores any scenes not found
+	// Scenes are not guaranteed to be in the same order as the input
+	FindByIDs(ctx context.Context, ids []int) ([]*Scene, error)
 }
 
 // SceneFinder provides methods to find scenes.

pkg/models/stash_box.go

@@ -7,7 +7,8 @@ type StashBoxFingerprint struct {
 }
 
 type StashBox struct {
-	Endpoint string `json:"endpoint"`
-	APIKey   string `json:"api_key"`
-	Name     string `json:"name"`
+	Endpoint             string `json:"endpoint"`
+	APIKey               string `json:"api_key"`
+	Name                 string `json:"name"`
+	MaxRequestsPerMinute int    `json:"max_requests_per_minute" koanf:"max_requests_per_minute"`
 }

pkg/models/tag.go

@@ -4,6 +4,8 @@ type TagFilterType struct {
 	OperatorFilter[TagFilterType]
 	// Filter by tag name
 	Name *StringCriterionInput `json:"name"`
+	// Filter by tag sort_name
+	SortName *StringCriterionInput `json:"sort_name"`
 	// Filter by tag aliases
 	Aliases *StringCriterionInput `json:"aliases"`
 	// Filter by tag favorites

pkg/plugin/examples/react-component/src/testReact.tsx

@@ -14,7 +14,11 @@ interface IPluginApi {
       Button: React.FC<any>;
       Nav: React.FC<any> & {
         Link: React.FC<any>;
+        Item: React.FC<any>;
       };
+      Tab: React.FC<any> & {
+        Pane: React.FC<any>;
+      }
     },
     FontAwesomeSolid: {
       faEthernet: any;
@@ -45,7 +49,7 @@ interface IPluginApi {
   const React = PluginApi.React;
   const GQL = PluginApi.GQL;
 
-  const { Button } = PluginApi.libraries.Bootstrap;
+  const { Button, Nav, Tab } = PluginApi.libraries.Bootstrap;
   const { faEthernet } = PluginApi.libraries.FontAwesomeSolid;
   const {
     Link,
@@ -144,6 +148,10 @@ interface IPluginApi {
     return <><Overlays />{original({...props})}</>;
   });
 
+  PluginApi.patch.instead("FrontPage", function (props: any, _: any, original: (props: any) => any) {  
+    return <><p>Hello from Test React!</p>{original({...props})}</>;
+  });
+
   const TestPage: React.FC = () => {
     const componentsToLoad = [
       PluginApi.loadableComponents.SceneCard,
@@ -237,5 +245,37 @@ interface IPluginApi {
         )
       }
     ]
-  })
+  });
+
+  PluginApi.patch.before("ScenePage.Tabs", function (props: any) {
+    return [
+      {
+        children: (
+          <>
+            {props.children}
+            <Nav.Item>
+              <Nav.Link eventKey="test-react-tab">
+                Test React tab
+              </Nav.Link>
+            </Nav.Item>
+          </>
+        ),
+      },
+    ];
+  });
+
+  PluginApi.patch.before("ScenePage.TabContent", function (props: any) {
+    return [
+      {
+        children: (
+          <>
+            {props.children}
+            <Tab.Pane eventKey="test-react-tab">
+              Test React tab content {props.scene.id}
+            </Tab.Pane>
+          </>
+        ),
+      },
+    ];
+  });
 })();

pkg/scene/export.go

@@ -235,6 +235,10 @@ func GetSceneMarkersJSON(ctx context.Context, markerReader models.SceneMarkerFin
 			UpdatedAt:  json.JSONTime{Time: sceneMarker.UpdatedAt},
 		}
 
+		if sceneMarker.EndSeconds != nil {
+			sceneMarkerJSON.EndSeconds = getDecimalString(*sceneMarker.EndSeconds)
+		}
+
 		results = append(results, sceneMarkerJSON)
 	}
 

pkg/scene/find.go

@@ -0,0 +1,90 @@
+package scene
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/stashapp/stash/pkg/models"
+)
+
+type LoadRelationshipOption func(context.Context, *models.Scene, models.SceneReader) error
+
+func LoadURLs(ctx context.Context, scene *models.Scene, r models.SceneReader) error {
+	if err := scene.LoadURLs(ctx, r); err != nil {
+		return fmt.Errorf("loading scene URLs: %w", err)
+	}
+
+	return nil
+}
+
+func LoadStashIDs(ctx context.Context, scene *models.Scene, r models.SceneReader) error {
+	if err := scene.LoadStashIDs(ctx, r); err != nil {
+		return fmt.Errorf("failed to load stash IDs for scene %d: %w", scene.ID, err)
+	}
+
+	return nil
+}
+
+func LoadFiles(ctx context.Context, scene *models.Scene, r models.SceneReader) error {
+	if err := scene.LoadFiles(ctx, r); err != nil {
+		return fmt.Errorf("failed to load files for scene %d: %w", scene.ID, err)
+	}
+
+	return nil
+}
+
+// FindByIDs retrieves multiple scenes by their IDs.
+// Missing scenes will be ignored, and the returned scenes are unsorted.
+// This method will load the specified relationships for each scene.
+func (s *Service) FindByIDs(ctx context.Context, ids []int, load ...LoadRelationshipOption) ([]*models.Scene, error) {
+	var scenes []*models.Scene
+	qb := s.Repository
+
+	var err error
+	scenes, err = qb.FindByIDs(ctx, ids)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO - we should bulk load these relationships
+	for _, scene := range scenes {
+		if err := s.LoadRelationships(ctx, scene, load...); err != nil {
+			return nil, err
+		}
+	}
+
+	return scenes, nil
+}
+
+// FindMany retrieves multiple scenes by their IDs. Return value is guaranteed to be in the same order as the input.
+// Missing scenes will return an error.
+// This method will load the specified relationships for each scene.
+func (s *Service) FindMany(ctx context.Context, ids []int, load ...LoadRelationshipOption) ([]*models.Scene, error) {
+	var scenes []*models.Scene
+	qb := s.Repository
+
+	var err error
+	scenes, err = qb.FindMany(ctx, ids)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO - we should bulk load these relationships
+	for _, scene := range scenes {
+		if err := s.LoadRelationships(ctx, scene, load...); err != nil {
+			return nil, err
+		}
+	}
+
+	return scenes, nil
+}
+
+func (s *Service) LoadRelationships(ctx context.Context, scene *models.Scene, load ...LoadRelationshipOption) error {
+	for _, l := range load {
+		if err := l(ctx, scene, s.Repository); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/scene/fingerprints.go

@@ -0,0 +1,40 @@
+package scene
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/stashapp/stash/pkg/models"
+)
+
+// GetFingerprints returns the fingerprints for the given scene ids.
+func (s *Service) GetScenesFingerprints(ctx context.Context, ids []int) ([]models.Fingerprints, error) {
+	fingerprints := make([]models.Fingerprints, len(ids))
+
+	qb := s.Repository
+
+	for i, sceneID := range ids {
+		scene, err := qb.Find(ctx, sceneID)
+		if err != nil {
+			return nil, err
+		}
+
+		if scene == nil {
+			return nil, fmt.Errorf("scene with id %d not found", sceneID)
+		}
+
+		if err := scene.LoadFiles(ctx, qb); err != nil {
+			return nil, err
+		}
+
+		var sceneFPs models.Fingerprints
+
+		for _, f := range scene.Files.List() {
+			sceneFPs = append(sceneFPs, f.Fingerprints...)
+		}
+
+		fingerprints[i] = sceneFPs
+	}
+
+	return fingerprints, nil
+}

pkg/scene/generate/marker_preview.go

@@ -11,7 +11,7 @@ import (
 
 const (
 	markerPreviewWidth        = 640
-	markerPreviewDuration     = 20
+	maxMarkerPreviewDuration  = 20
 	markerPreviewAudioBitrate = "64k"
 
 	markerImageDuration = 5
@@ -20,20 +20,28 @@ const (
 	markerScreenshotQuality = 2
 )
 
-func (g Generator) MarkerPreviewVideo(ctx context.Context, input string, hash string, seconds int, includeAudio bool) error {
+func (g Generator) MarkerPreviewVideo(ctx context.Context, input string, hash string, seconds float64, endSeconds *float64, includeAudio bool) error {
 	lockCtx := g.LockManager.ReadLock(ctx, input)
 	defer lockCtx.Cancel()
 
-	output := g.MarkerPaths.GetVideoPreviewPath(hash, seconds)
+	output := g.MarkerPaths.GetVideoPreviewPath(hash, int(seconds))
 	if !g.Overwrite {
 		if exists, _ := fsutil.FileExists(output); exists {
 			return nil
 		}
 	}
 
+	duration := float64(maxMarkerPreviewDuration)
+
+	// don't allow preview to exceed max duration
+	if endSeconds != nil && *endSeconds-seconds < maxMarkerPreviewDuration {
+		duration = float64(*endSeconds) - seconds
+	}
+
 	if err := g.generateFile(lockCtx, g.MarkerPaths, mp4Pattern, output, g.markerPreviewVideo(input, sceneMarkerOptions{
-		Seconds: seconds,
-		Audio:   includeAudio,
+		Seconds:  seconds,
+		Duration: duration,
+		Audio:    includeAudio,
 	})); err != nil {
 		return err
 	}
@@ -44,8 +52,9 @@ func (g Generator) MarkerPreviewVideo(ctx context.Context, input string, hash st
 }
 
 type sceneMarkerOptions struct {
-	Seconds int
-	Audio   bool
+	Seconds  float64
+	Duration float64
+	Audio    bool
 }
 
 func (g Generator) markerPreviewVideo(input string, options sceneMarkerOptions) generateFn {
@@ -69,8 +78,8 @@ func (g Generator) markerPreviewVideo(input string, options sceneMarkerOptions)
 		)
 
 		trimOptions := transcoder.TranscodeOptions{
-			Duration:   markerPreviewDuration,
-			StartTime:  float64(options.Seconds),
+			Duration:   options.Duration,
+			StartTime:  options.Seconds,
 			OutputPath: tmpFn,
 			VideoCodec: ffmpeg.VideoCodecLibX264,
 			VideoArgs:  videoArgs,
@@ -90,11 +99,11 @@ func (g Generator) markerPreviewVideo(input string, options sceneMarkerOptions)
 	}
 }
 
-func (g Generator) SceneMarkerWebp(ctx context.Context, input string, hash string, seconds int) error {
+func (g Generator) SceneMarkerWebp(ctx context.Context, input string, hash string, seconds float64) error {
 	lockCtx := g.LockManager.ReadLock(ctx, input)
 	defer lockCtx.Cancel()
 
-	output := g.MarkerPaths.GetWebpPreviewPath(hash, seconds)
+	output := g.MarkerPaths.GetWebpPreviewPath(hash, int(seconds))
 	if !g.Overwrite {
 		if exists, _ := fsutil.FileExists(output); exists {
 			return nil
@@ -143,11 +152,11 @@ func (g Generator) sceneMarkerWebp(input string, options sceneMarkerOptions) gen
 	}
 }
 
-func (g Generator) SceneMarkerScreenshot(ctx context.Context, input string, hash string, seconds int, width int) error {
+func (g Generator) SceneMarkerScreenshot(ctx context.Context, input string, hash string, seconds float64, width int) error {
 	lockCtx := g.LockManager.ReadLock(ctx, input)
 	defer lockCtx.Cancel()
 
-	output := g.MarkerPaths.GetScreenshotPath(hash, seconds)
+	output := g.MarkerPaths.GetScreenshotPath(hash, int(seconds))
 	if !g.Overwrite {
 		if exists, _ := fsutil.FileExists(output); exists {
 			return nil
@@ -167,7 +176,7 @@ func (g Generator) SceneMarkerScreenshot(ctx context.Context, input string, hash
 }
 
 type SceneMarkerScreenshotOptions struct {
-	Seconds int
+	Seconds float64
 	Width   int
 }
 
@@ -180,7 +189,7 @@ func (g Generator) sceneMarkerScreenshot(input string, options SceneMarkerScreen
 			Width:      options.Width,
 		}
 
-		args := transcoder.ScreenshotTime(input, float64(options.Seconds), ssOptions)
+		args := transcoder.ScreenshotTime(input, options.Seconds, ssOptions)
 
 		return g.generate(lockCtx, args)
 	}

pkg/scene/marker_import.go

@@ -27,12 +27,20 @@ type MarkerImporter struct {
 
 func (i *MarkerImporter) PreImport(ctx context.Context) error {
 	seconds, _ := strconv.ParseFloat(i.Input.Seconds, 64)
+
+	var endSeconds *float64
+	if i.Input.EndSeconds != "" {
+		parsedEndSeconds, _ := strconv.ParseFloat(i.Input.EndSeconds, 64)
+		endSeconds = &parsedEndSeconds
+	}
+
 	i.marker = models.SceneMarker{
-		Title:     i.Input.Title,
-		Seconds:   seconds,
-		SceneID:   i.SceneID,
-		CreatedAt: i.Input.CreatedAt.GetTime(),
-		UpdatedAt: i.Input.UpdatedAt.GetTime(),
+		Title:      i.Input.Title,
+		Seconds:    seconds,
+		EndSeconds: endSeconds,
+		SceneID:    i.SceneID,
+		CreatedAt:  i.Input.CreatedAt.GetTime(),
+		UpdatedAt:  i.Input.UpdatedAt.GetTime(),
 	}
 
 	if err := i.populateTags(ctx); err != nil {

pkg/scraper/action.go

@@ -29,8 +29,9 @@ type scraperActionImpl interface {
 	scrapeByName(ctx context.Context, name string, ty ScrapeContentType) ([]ScrapedContent, error)
 	scrapeByFragment(ctx context.Context, input Input) (ScrapedContent, error)
 
-	scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error)
-	scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error)
+	scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error)
+	scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error)
+	scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error)
 }
 
 func (c config) getScraper(scraper scraperTypeConfig, client *http.Client, globalConfig GlobalConfig) scraperActionImpl {

pkg/scraper/autotag.go

@@ -89,8 +89,8 @@ func autotagMatchTags(ctx context.Context, path string, tagReader models.TagAuto
 	return ret, nil
 }
 
-func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scene *models.Scene) (*ScrapedScene, error) {
-	var ret *ScrapedScene
+func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scene *models.Scene) (*models.ScrapedScene, error) {
+	var ret *models.ScrapedScene
 	const trimExt = false
 
 	// populate performers, studio and tags based on scene path
@@ -115,7 +115,7 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
 		}
 
 		if len(performers) > 0 || studio != nil || len(tags) > 0 {
-			ret = &ScrapedScene{
+			ret = &models.ScrapedScene{
 				Performers: performers,
 				Studio:     studio,
 				Tags:       tags,
@@ -130,7 +130,7 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
 	return ret, nil
 }
 
-func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	path := gallery.Path
 	if path == "" {
 		// not valid for non-path-based galleries
@@ -140,7 +140,7 @@ func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, ga
 	// only trim extension if gallery is file-based
 	trimExt := gallery.PrimaryFileID != nil
 
-	var ret *ScrapedGallery
+	var ret *models.ScrapedGallery
 
 	// populate performers, studio and tags based on scene path
 	if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
@@ -160,7 +160,7 @@ func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, ga
 		}
 
 		if len(performers) > 0 || studio != nil || len(tags) > 0 {
-			ret = &ScrapedGallery{
+			ret = &models.ScrapedGallery{
 				Performers: performers,
 				Studio:     studio,
 				Tags:       tags,

pkg/scraper/cache.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -15,6 +16,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/txn"
 )
 
@@ -40,6 +42,7 @@ type GlobalConfig interface {
 	GetScraperCertCheck() bool
 	GetPythonPath() string
 	GetProxy() string
+	GetScraperExcludeTagPatterns() []string
 }
 
 func isCDPPathHTTP(c GlobalConfig) bool {
@@ -77,11 +80,18 @@ type GalleryFinder interface {
 	models.URLLoader
 }
 
+type ImageFinder interface {
+	models.ImageGetter
+	models.FileLoader
+	models.URLLoader
+}
+
 type Repository struct {
 	TxnManager models.TxnManager
 
 	SceneFinder     SceneFinder
 	GalleryFinder   GalleryFinder
+	ImageFinder     ImageFinder
 	TagFinder       TagFinder
 	PerformerFinder PerformerFinder
 	GroupFinder     match.GroupNamesFinder
@@ -93,6 +103,7 @@ func NewRepository(repo models.Repository) Repository {
 		TxnManager:      repo.TxnManager,
 		SceneFinder:     repo.Scene,
 		GalleryFinder:   repo.Gallery,
+		ImageFinder:     repo.Image,
 		TagFinder:       repo.Tag,
 		PerformerFinder: repo.Performer,
 		GroupFinder:     repo.Group,
@@ -227,6 +238,10 @@ func (c Cache) findScraper(scraperID string) scraper {
 	return nil
 }
 
+func (c Cache) compileExcludeTagPatterns() []*regexp.Regexp {
+	return CompileExclusionRegexps(c.globalConfig.GetScraperExcludeTagPatterns())
+}
+
 func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeContentType) ([]ScrapedContent, error) {
 	// find scraper with the provided id
 	s := c.findScraper(id)
@@ -247,13 +262,20 @@ func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeConten
 		return nil, fmt.Errorf("error while name scraping with scraper %s: %w", id, err)
 	}
 
+	ignoredRegex := c.compileExcludeTagPatterns()
+
+	var ignoredTags []string
 	for i, cc := range content {
-		content[i], err = c.postScrape(ctx, cc)
+		var thisIgnoredTags []string
+		content[i], thisIgnoredTags, err = c.postScrape(ctx, cc, ignoredRegex)
 		if err != nil {
 			return nil, fmt.Errorf("error while post-scraping with scraper %s: %w", id, err)
 		}
+		ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
 	}
 
+	LogIgnoredTags(ignoredTags)
+
 	return content, nil
 }
 
@@ -277,7 +299,7 @@ func (c Cache) ScrapeFragment(ctx context.Context, id string, input Input) (Scra
 		return nil, fmt.Errorf("error while fragment scraping with scraper %s: %w", id, err)
 	}
 
-	return c.postScrape(ctx, content)
+	return c.postScrapeSingle(ctx, content)
 }
 
 // ScrapeURL scrapes a given url for the given content. Searches the scraper cache
@@ -299,7 +321,7 @@ func (c Cache) ScrapeURL(ctx context.Context, url string, ty ScrapeContentType)
 				return ret, nil
 			}
 
-			return c.postScrape(ctx, ret)
+			return c.postScrapeSingle(ctx, ret)
 		}
 	}
 
@@ -357,12 +379,34 @@ func (c Cache) ScrapeID(ctx context.Context, scraperID string, id int, ty Scrape
 			return nil, fmt.Errorf("scraper %s: %w", scraperID, err)
 		}
 
+		if scraped != nil {
+			ret = scraped
+		}
+
+	case ScrapeContentTypeImage:
+		is, ok := s.(imageScraper)
+		if !ok {
+			return nil, fmt.Errorf("%w: cannot use scraper %s as a image scraper", ErrNotSupported, scraperID)
+		}
+
+		scene, err := c.getImage(ctx, id)
+		if err != nil {
+			return nil, fmt.Errorf("scraper %s: unable to load image id %v: %w", scraperID, id, err)
+		}
+
+		// don't assign nil concrete pointer to ret interface, otherwise nil
+		// detection is harder
+		scraped, err := is.viaImage(ctx, c.client, scene)
+		if err != nil {
+			return nil, fmt.Errorf("scraper %s: %w", scraperID, err)
+		}
+
 		if scraped != nil {
 			ret = scraped
 		}
 	}
 
-	return c.postScrape(ctx, ret)
+	return c.postScrapeSingle(ctx, ret)
 }
 
 func (c Cache) getScene(ctx context.Context, sceneID int) (*models.Scene, error) {
@@ -426,3 +470,31 @@ func (c Cache) getGallery(ctx context.Context, galleryID int) (*models.Gallery,
 	}
 	return ret, nil
 }
+
+func (c Cache) getImage(ctx context.Context, imageID int) (*models.Image, error) {
+	var ret *models.Image
+	r := c.repository
+	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
+		qb := r.ImageFinder
+
+		var err error
+		ret, err = qb.Find(ctx, imageID)
+		if err != nil {
+			return err
+		}
+
+		if ret == nil {
+			return fmt.Errorf("image with id %d not found", imageID)
+		}
+
+		err = ret.LoadFiles(ctx, qb)
+		if err != nil {
+			return err
+		}
+
+		return ret.LoadURLs(ctx, qb)
+	}); err != nil {
+		return nil, err
+	}
+	return ret, nil
+}

pkg/scraper/config.go

@@ -45,8 +45,16 @@ type config struct {
 	// Configuration for querying a gallery by a URL
 	GalleryByURL []*scrapeByURLConfig `yaml:"galleryByURL"`
 
+	// Configuration for querying an image by a URL
+	ImageByURL []*scrapeByURLConfig `yaml:"imageByURL"`
+
+	// Configuration for querying image by an Image fragment
+	ImageByFragment *scraperTypeConfig `yaml:"imageByFragment"`
+
 	// Configuration for querying a movie by a URL - deprecated, use GroupByURL
 	MovieByURL []*scrapeByURLConfig `yaml:"movieByURL"`
+
+	// Configuration for querying a group by a URL
 	GroupByURL []*scrapeByURLConfig `yaml:"groupByURL"`
 
 	// Scraper debugging options
@@ -295,6 +303,21 @@ func (c config) spec() Scraper {
 		ret.Gallery = &gallery
 	}
 
+	image := ScraperSpec{}
+	if c.ImageByFragment != nil {
+		image.SupportedScrapes = append(image.SupportedScrapes, ScrapeTypeFragment)
+	}
+	if len(c.ImageByURL) > 0 {
+		image.SupportedScrapes = append(image.SupportedScrapes, ScrapeTypeURL)
+		for _, v := range c.ImageByURL {
+			image.Urls = append(image.Urls, v.URL...)
+		}
+	}
+
+	if len(image.SupportedScrapes) > 0 {
+		ret.Image = &image
+	}
+
 	group := ScraperSpec{}
 	if len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0 {
 		group.SupportedScrapes = append(group.SupportedScrapes, ScrapeTypeURL)
@@ -319,6 +342,8 @@ func (c config) supports(ty ScrapeContentType) bool {
 		return (c.SceneByName != nil && c.SceneByQueryFragment != nil) || c.SceneByFragment != nil || len(c.SceneByURL) > 0
 	case ScrapeContentTypeGallery:
 		return c.GalleryByFragment != nil || len(c.GalleryByURL) > 0
+	case ScrapeContentTypeImage:
+		return c.ImageByFragment != nil || len(c.ImageByURL) > 0
 	case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
 		return len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0
 	}
@@ -346,7 +371,18 @@ func (c config) matchesURL(url string, ty ScrapeContentType) bool {
 				return true
 			}
 		}
+	case ScrapeContentTypeImage:
+		for _, scraper := range c.ImageByURL {
+			if scraper.matchesURL(url) {
+				return true
+			}
+		}
 	case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
+		for _, scraper := range c.GroupByURL {
+			if scraper.matchesURL(url) {
+				return true
+			}
+		}
 		for _, scraper := range c.MovieByURL {
 			if scraper.matchesURL(url) {
 				return true

pkg/scraper/gallery.go

@@ -1,32 +0,0 @@
-package scraper
-
-import "github.com/stashapp/stash/pkg/models"
-
-type ScrapedGallery struct {
-	Title        *string                    `json:"title"`
-	Code         *string                    `json:"code"`
-	Details      *string                    `json:"details"`
-	Photographer *string                    `json:"photographer"`
-	URLs         []string                   `json:"urls"`
-	Date         *string                    `json:"date"`
-	Studio       *models.ScrapedStudio      `json:"studio"`
-	Tags         []*models.ScrapedTag       `json:"tags"`
-	Performers   []*models.ScrapedPerformer `json:"performers"`
-
-	// deprecated
-	URL *string `json:"url"`
-}
-
-func (ScrapedGallery) IsScrapedContent() {}
-
-type ScrapedGalleryInput struct {
-	Title        *string  `json:"title"`
-	Code         *string  `json:"code"`
-	Details      *string  `json:"details"`
-	Photographer *string  `json:"photographer"`
-	URLs         []string `json:"urls"`
-	Date         *string  `json:"date"`
-
-	// deprecated
-	URL *string `json:"url"`
-}

pkg/scraper/group.go

@@ -33,6 +33,9 @@ func (g group) fragmentScraper(input Input) *scraperTypeConfig {
 	case input.Gallery != nil:
 		// TODO - this should be galleryByQueryFragment
 		return g.config.GalleryByFragment
+	case input.Image != nil:
+		// TODO - this should be imageByImageFragment
+		return g.config.ImageByFragment
 	case input.Scene != nil:
 		return g.config.SceneByQueryFragment
 	}
@@ -57,7 +60,7 @@ func (g group) viaFragment(ctx context.Context, client *http.Client, input Input
 	return s.scrapeByFragment(ctx, input)
 }
 
-func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error) {
+func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*models.ScrapedScene, error) {
 	if g.config.SceneByFragment == nil {
 		return nil, ErrNotSupported
 	}
@@ -66,7 +69,7 @@ func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.
 	return s.scrapeSceneByScene(ctx, scene)
 }
 
-func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	if g.config.GalleryByFragment == nil {
 		return nil, ErrNotSupported
 	}
@@ -75,6 +78,15 @@ func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *mod
 	return s.scrapeGalleryByGallery(ctx, gallery)
 }
 
+func (g group) viaImage(ctx context.Context, client *http.Client, gallery *models.Image) (*models.ScrapedImage, error) {
+	if g.config.ImageByFragment == nil {
+		return nil, ErrNotSupported
+	}
+
+	s := g.config.getScraper(*g.config.ImageByFragment, client, g.globalConf)
+	return s.scrapeImageByImage(ctx, gallery)
+}
+
 func loadUrlCandidates(c config, ty ScrapeContentType) []*scrapeByURLConfig {
 	switch ty {
 	case ScrapeContentTypePerformer:
@@ -85,6 +97,8 @@ func loadUrlCandidates(c config, ty ScrapeContentType) []*scrapeByURLConfig {
 		return append(c.MovieByURL, c.GroupByURL...)
 	case ScrapeContentTypeGallery:
 		return c.GalleryByURL
+	case ScrapeContentTypeImage:
+		return c.ImageByURL
 	}
 
 	panic("loadUrlCandidates: unreachable")

pkg/scraper/image.go

@@ -37,7 +37,7 @@ func setPerformerImage(ctx context.Context, client *http.Client, p *models.Scrap
 	return nil
 }
 
-func setSceneImage(ctx context.Context, client *http.Client, s *ScrapedScene, globalConfig GlobalConfig) error {
+func setSceneImage(ctx context.Context, client *http.Client, s *models.ScrapedScene, globalConfig GlobalConfig) error {
 	// don't try to get the image if it doesn't appear to be a URL
 	if s.Image == nil || !strings.HasPrefix(*s.Image, "http") {
 		// nothing to do

pkg/scraper/json.go

@@ -102,6 +102,12 @@ func (s *jsonScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCont
 			return nil, err
 		}
 		return ret, nil
+	case ScrapeContentTypeImage:
+		ret, err := scraper.scrapeImage(ctx, q)
+		if err != nil || ret == nil {
+			return nil, err
+		}
+		return ret, nil
 	case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
 		ret, err := scraper.scrapeGroup(ctx, q)
 		if err != nil || ret == nil {
@@ -166,7 +172,7 @@ func (s *jsonScraper) scrapeByName(ctx context.Context, name string, ty ScrapeCo
 	return nil, ErrNotSupported
 }
 
-func (s *jsonScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
+func (s *jsonScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromScene(scene)
 	if s.scraper.QueryURLReplacements != nil {
@@ -225,7 +231,31 @@ func (s *jsonScraper) scrapeByFragment(ctx context.Context, input Input) (Scrape
 	return scraper.scrapeScene(ctx, q)
 }
 
-func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
+	// construct the URL
+	queryURL := queryURLParametersFromImage(image)
+	if s.scraper.QueryURLReplacements != nil {
+		queryURL.applyReplacements(s.scraper.QueryURLReplacements)
+	}
+	url := queryURL.constructURL(s.scraper.QueryURL)
+
+	scraper := s.getJsonScraper()
+
+	if scraper == nil {
+		return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config")
+	}
+
+	doc, err := s.loadURL(ctx, url)
+
+	if err != nil {
+		return nil, err
+	}
+
+	q := s.getJsonQuery(doc)
+	return scraper.scrapeImage(ctx, q)
+}
+
+func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromGallery(gallery)
 	if s.scraper.QueryURLReplacements != nil {

pkg/scraper/mapped.go

@@ -43,7 +43,9 @@ func (s mappedConfig) applyCommon(c commonMappedConfig, src string) string {
 	return ret
 }
 
-func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonMappedConfig) mappedResults {
+type isMultiFunc func(key string) bool
+
+func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonMappedConfig, isMulti isMultiFunc) mappedResults {
 	var ret mappedResults
 
 	for k, attrConfig := range s {
@@ -51,7 +53,7 @@ func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonM
 		if attrConfig.Fixed != "" {
 			// TODO - not sure if this needs to set _all_ indexes for the key
 			const i = 0
-			ret = ret.setKey(i, k, attrConfig.Fixed)
+			ret = ret.setSingleValue(i, k, attrConfig.Fixed)
 		} else {
 			selector := attrConfig.Selector
 			selector = s.applyCommon(common, selector)
@@ -63,8 +65,15 @@ func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonM
 
 			if len(found) > 0 {
 				result := s.postProcess(ctx, q, attrConfig, found)
-				for i, text := range result {
-					ret = ret.setKey(i, k, text)
+
+				// HACK - if the key is URLs, then we need to set the value as a multi-value
+				isMulti := isMulti != nil && isMulti(k)
+				if isMulti {
+					ret = ret.setMultiValue(0, k, result)
+				} else {
+					for i, text := range result {
+						ret = ret.setSingleValue(i, k, text)
+					}
 				}
 			}
 		}
@@ -181,6 +190,7 @@ type mappedGalleryScraperConfig struct {
 	Performers mappedConfig `yaml:"Performers"`
 	Studio     mappedConfig `yaml:"Studio"`
 }
+
 type _mappedGalleryScraperConfig mappedGalleryScraperConfig
 
 func (s *mappedGalleryScraperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -228,6 +238,60 @@ func (s *mappedGalleryScraperConfig) UnmarshalYAML(unmarshal func(interface{}) e
 	return nil
 }
 
+type mappedImageScraperConfig struct {
+	mappedConfig
+
+	Tags       mappedConfig `yaml:"Tags"`
+	Performers mappedConfig `yaml:"Performers"`
+	Studio     mappedConfig `yaml:"Studio"`
+}
+type _mappedImageScraperConfig mappedImageScraperConfig
+
+func (s *mappedImageScraperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// HACK - unmarshal to map first, then remove known scene sub-fields, then
+	// remarshal to yaml and pass that down to the base map
+	parentMap := make(map[string]interface{})
+	if err := unmarshal(parentMap); err != nil {
+		return err
+	}
+
+	// move the known sub-fields to a separate map
+	thisMap := make(map[string]interface{})
+
+	thisMap[mappedScraperConfigSceneTags] = parentMap[mappedScraperConfigSceneTags]
+	thisMap[mappedScraperConfigScenePerformers] = parentMap[mappedScraperConfigScenePerformers]
+	thisMap[mappedScraperConfigSceneStudio] = parentMap[mappedScraperConfigSceneStudio]
+
+	delete(parentMap, mappedScraperConfigSceneTags)
+	delete(parentMap, mappedScraperConfigScenePerformers)
+	delete(parentMap, mappedScraperConfigSceneStudio)
+
+	// re-unmarshal the sub-fields
+	yml, err := yaml.Marshal(thisMap)
+	if err != nil {
+		return err
+	}
+
+	// needs to be a different type to prevent infinite recursion
+	c := _mappedImageScraperConfig{}
+	if err := yaml.Unmarshal(yml, &c); err != nil {
+		return err
+	}
+
+	*s = mappedImageScraperConfig(c)
+
+	yml, err = yaml.Marshal(parentMap)
+	if err != nil {
+		return err
+	}
+
+	if err := yaml.Unmarshal(yml, &s.mappedConfig); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 type mappedPerformerScraperConfig struct {
 	mappedConfig
 
@@ -785,41 +849,80 @@ type mappedScraper struct {
 	Common    commonMappedConfig            `yaml:"common"`
 	Scene     *mappedSceneScraperConfig     `yaml:"scene"`
 	Gallery   *mappedGalleryScraperConfig   `yaml:"gallery"`
+	Image     *mappedImageScraperConfig     `yaml:"image"`
 	Performer *mappedPerformerScraperConfig `yaml:"performer"`
-	Movie     *mappedMovieScraperConfig     `yaml:"movie"`
+	Group     *mappedMovieScraperConfig     `yaml:"group"`
+
+	// deprecated
+	Movie *mappedMovieScraperConfig `yaml:"movie"`
 }
 
-type mappedResult map[string]string
+type mappedResult map[string]interface{}
 type mappedResults []mappedResult
 
 func (r mappedResult) apply(dest interface{}) {
-	destVal := reflect.ValueOf(dest)
-
-	// dest should be a pointer
-	destVal = destVal.Elem()
+	destVal := reflect.ValueOf(dest).Elem()
 
+	// all fields are either string pointers or string slices
 	for key, value := range r {
-		field := destVal.FieldByName(key)
-
-		if field.IsValid() {
-			var reflectValue reflect.Value
-			if field.Kind() == reflect.Ptr {
-				// need to copy the value, otherwise everything is set to the
-				// same pointer
-				localValue := value
-				reflectValue = reflect.ValueOf(&localValue)
-			} else {
-				reflectValue = reflect.ValueOf(value)
-			}
-
-			field.Set(reflectValue)
-		} else {
-			logger.Errorf("Field %s does not exist in %T", key, dest)
+		if err := mapFieldValue(destVal, key, value); err != nil {
+			logger.Errorf("Error mapping field %s in %T: %v", key, dest, err)
 		}
 	}
 }
 
-func (r mappedResults) setKey(index int, key string, value string) mappedResults {
+func mapFieldValue(destVal reflect.Value, key string, value interface{}) error {
+	field := destVal.FieldByName(key)
+	fieldType := field.Type()
+
+	if field.IsValid() && field.CanSet() {
+		switch v := value.(type) {
+		case string:
+			// if the field is a pointer to a string, then we need to convert the string to a pointer
+			// if the field is a string slice, then we need to convert the string to a slice
+			switch {
+			case fieldType.Kind() == reflect.String:
+				field.SetString(v)
+			case fieldType.Kind() == reflect.Ptr && fieldType.Elem().Kind() == reflect.String:
+				ptr := reflect.New(fieldType.Elem())
+				ptr.Elem().SetString(v)
+				field.Set(ptr)
+			case fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.String:
+				field.Set(reflect.ValueOf([]string{v}))
+			default:
+				return fmt.Errorf("cannot convert %T to %s", value, fieldType)
+			}
+		case []string:
+			// expect the field to be a string slice
+			if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.String {
+				field.Set(reflect.ValueOf(v))
+			} else {
+				return fmt.Errorf("cannot convert %T to %s", value, fieldType)
+			}
+		default:
+			// fallback to reflection
+			reflectValue := reflect.ValueOf(value)
+			reflectValueType := reflectValue.Type()
+
+			switch {
+			case reflectValueType.ConvertibleTo(fieldType):
+				field.Set(reflectValue.Convert(fieldType))
+			case fieldType.Kind() == reflect.Pointer && reflectValueType.ConvertibleTo(fieldType.Elem()):
+				ptr := reflect.New(fieldType.Elem())
+				ptr.Elem().Set(reflectValue.Convert(fieldType.Elem()))
+				field.Set(ptr)
+			default:
+				return fmt.Errorf("cannot convert %T to %s", value, fieldType)
+			}
+		}
+	} else {
+		return fmt.Errorf("field does not exist or cannot be set")
+	}
+
+	return nil
+}
+
+func (r mappedResults) setSingleValue(index int, key string, value string) mappedResults {
 	if index >= len(r) {
 		r = append(r, make(mappedResult))
 	}
@@ -829,6 +932,20 @@ func (r mappedResults) setKey(index int, key string, value string) mappedResults
 	return r
 }
 
+func (r mappedResults) setMultiValue(index int, key string, value []string) mappedResults {
+	if index >= len(r) {
+		r = append(r, make(mappedResult))
+	}
+
+	logger.Debugf(`[%d][%s] = %s`, index, key, value)
+	r[index][key] = value
+	return r
+}
+
+func urlsIsMulti(key string) bool {
+	return key == "URLs"
+}
+
 func (s mappedScraper) scrapePerformer(ctx context.Context, q mappedQuery) (*models.ScrapedPerformer, error) {
 	var ret models.ScrapedPerformer
 
@@ -839,12 +956,12 @@ func (s mappedScraper) scrapePerformer(ctx context.Context, q mappedQuery) (*mod
 
 	performerTagsMap := performerMap.Tags
 
-	results := performerMap.process(ctx, q, s.Common)
+	results := performerMap.process(ctx, q, s.Common, urlsIsMulti)
 
 	// now apply the tags
 	if performerTagsMap != nil {
 		logger.Debug(`Processing performer tags:`)
-		tagResults := performerTagsMap.process(ctx, q, s.Common)
+		tagResults := performerTagsMap.process(ctx, q, s.Common, nil)
 
 		for _, p := range tagResults {
 			tag := &models.ScrapedTag{}
@@ -872,7 +989,8 @@ func (s mappedScraper) scrapePerformers(ctx context.Context, q mappedQuery) ([]*
 		return nil, nil
 	}
 
-	results := performerMap.process(ctx, q, s.Common)
+	// isMulti is nil because it will behave incorrect when scraping multiple performers
+	results := performerMap.process(ctx, q, s.Common, nil)
 	for _, r := range results {
 		var p models.ScrapedPerformer
 		r.apply(&p)
@@ -882,8 +1000,8 @@ func (s mappedScraper) scrapePerformers(ctx context.Context, q mappedQuery) ([]*
 	return ret, nil
 }
 
-// processSceneRelationships sets the relationships on the ScrapedScene. It returns true if any relationships were set.
-func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQuery, resultIndex int, ret *ScrapedScene) bool {
+// processSceneRelationships sets the relationships on the models.ScrapedScene. It returns true if any relationships were set.
+func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQuery, resultIndex int, ret *models.ScrapedScene) bool {
 	sceneScraperConfig := s.Scene
 
 	scenePerformersMap := sceneScraperConfig.Performers
@@ -901,7 +1019,7 @@ func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQu
 
 	if sceneStudioMap != nil {
 		logger.Debug(`Processing scene studio:`)
-		studioResults := sceneStudioMap.process(ctx, q, s.Common)
+		studioResults := sceneStudioMap.process(ctx, q, s.Common, nil)
 
 		if len(studioResults) > 0 && resultIndex < len(studioResults) {
 			studio := &models.ScrapedStudio{}
@@ -925,14 +1043,15 @@ func (s mappedScraper) processPerformers(ctx context.Context, performersMap mapp
 	// now apply the performers and tags
 	if performersMap.mappedConfig != nil {
 		logger.Debug(`Processing performers:`)
-		performerResults := performersMap.process(ctx, q, s.Common)
+		// isMulti is nil because it will behave incorrect when scraping multiple performers
+		performerResults := performersMap.process(ctx, q, s.Common, nil)
 
 		scenePerformerTagsMap := performersMap.Tags
 
 		// process performer tags once
 		var performerTagResults mappedResults
 		if scenePerformerTagsMap != nil {
-			performerTagResults = scenePerformerTagsMap.process(ctx, q, s.Common)
+			performerTagResults = scenePerformerTagsMap.process(ctx, q, s.Common, nil)
 		}
 
 		for _, p := range performerResults {
@@ -955,7 +1074,7 @@ func (s mappedScraper) processPerformers(ctx context.Context, performersMap mapp
 func processRelationships[T any](ctx context.Context, s mappedScraper, relationshipMap mappedConfig, q mappedQuery) []*T {
 	var ret []*T
 
-	results := relationshipMap.process(ctx, q, s.Common)
+	results := relationshipMap.process(ctx, q, s.Common, nil)
 
 	for _, p := range results {
 		var value T
@@ -966,8 +1085,8 @@ func processRelationships[T any](ctx context.Context, s mappedScraper, relations
 	return ret
 }
 
-func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*ScrapedScene, error) {
-	var ret []*ScrapedScene
+func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*models.ScrapedScene, error) {
+	var ret []*models.ScrapedScene
 
 	sceneScraperConfig := s.Scene
 	sceneMap := sceneScraperConfig.mappedConfig
@@ -976,11 +1095,12 @@ func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*Scra
 	}
 
 	logger.Debug(`Processing scenes:`)
-	results := sceneMap.process(ctx, q, s.Common)
+	// urlsIsMulti is nil because it will behave incorrect when scraping multiple scenes
+	results := sceneMap.process(ctx, q, s.Common, nil)
 	for i, r := range results {
 		logger.Debug(`Processing scene:`)
 
-		var thisScene ScrapedScene
+		var thisScene models.ScrapedScene
 		r.apply(&thisScene)
 		s.processSceneRelationships(ctx, q, i, &thisScene)
 		ret = append(ret, &thisScene)
@@ -989,7 +1109,7 @@ func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*Scra
 	return ret, nil
 }
 
-func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*ScrapedScene, error) {
+func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*models.ScrapedScene, error) {
 	sceneScraperConfig := s.Scene
 	if sceneScraperConfig == nil {
 		return nil, nil
@@ -998,9 +1118,9 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*Scraped
 	sceneMap := sceneScraperConfig.mappedConfig
 
 	logger.Debug(`Processing scene:`)
-	results := sceneMap.process(ctx, q, s.Common)
+	results := sceneMap.process(ctx, q, s.Common, urlsIsMulti)
 
-	var ret ScrapedScene
+	var ret models.ScrapedScene
 	if len(results) > 0 {
 		results[0].apply(&ret)
 	}
@@ -1016,49 +1136,37 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*Scraped
 	return nil, nil
 }
 
-func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*ScrapedGallery, error) {
-	var ret ScrapedGallery
+func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*models.ScrapedImage, error) {
+	var ret models.ScrapedImage
 
-	galleryScraperConfig := s.Gallery
-	if galleryScraperConfig == nil {
+	imageScraperConfig := s.Image
+	if imageScraperConfig == nil {
 		return nil, nil
 	}
 
-	galleryMap := galleryScraperConfig.mappedConfig
+	imageMap := imageScraperConfig.mappedConfig
 
-	galleryPerformersMap := galleryScraperConfig.Performers
-	galleryTagsMap := galleryScraperConfig.Tags
-	galleryStudioMap := galleryScraperConfig.Studio
+	imagePerformersMap := imageScraperConfig.Performers
+	imageTagsMap := imageScraperConfig.Tags
+	imageStudioMap := imageScraperConfig.Studio
 
-	logger.Debug(`Processing gallery:`)
-	results := galleryMap.process(ctx, q, s.Common)
+	logger.Debug(`Processing image:`)
+	results := imageMap.process(ctx, q, s.Common, urlsIsMulti)
 
 	// now apply the performers and tags
-	if galleryPerformersMap != nil {
-		logger.Debug(`Processing gallery performers:`)
-		performerResults := galleryPerformersMap.process(ctx, q, s.Common)
-
-		for _, p := range performerResults {
-			performer := &models.ScrapedPerformer{}
-			p.apply(performer)
-			ret.Performers = append(ret.Performers, performer)
-		}
+	if imagePerformersMap != nil {
+		logger.Debug(`Processing image performers:`)
+		ret.Performers = processRelationships[models.ScrapedPerformer](ctx, s, imagePerformersMap, q)
 	}
 
-	if galleryTagsMap != nil {
-		logger.Debug(`Processing gallery tags:`)
-		tagResults := galleryTagsMap.process(ctx, q, s.Common)
-
-		for _, p := range tagResults {
-			tag := &models.ScrapedTag{}
-			p.apply(tag)
-			ret.Tags = append(ret.Tags, tag)
-		}
+	if imageTagsMap != nil {
+		logger.Debug(`Processing image tags:`)
+		ret.Tags = processRelationships[models.ScrapedTag](ctx, s, imageTagsMap, q)
 	}
 
-	if galleryStudioMap != nil {
-		logger.Debug(`Processing gallery studio:`)
-		studioResults := galleryStudioMap.process(ctx, q, s.Common)
+	if imageStudioMap != nil {
+		logger.Debug(`Processing image studio:`)
+		studioResults := imageStudioMap.process(ctx, q, s.Common, nil)
 
 		if len(studioResults) > 0 {
 			studio := &models.ScrapedStudio{}
@@ -1079,24 +1187,92 @@ func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*Scrap
 	return &ret, nil
 }
 
-func (s mappedScraper) scrapeGroup(ctx context.Context, q mappedQuery) (*models.ScrapedMovie, error) {
-	var ret models.ScrapedMovie
+func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*models.ScrapedGallery, error) {
+	var ret models.ScrapedGallery
 
-	movieScraperConfig := s.Movie
-	if movieScraperConfig == nil {
+	galleryScraperConfig := s.Gallery
+	if galleryScraperConfig == nil {
 		return nil, nil
 	}
 
-	movieMap := movieScraperConfig.mappedConfig
+	galleryMap := galleryScraperConfig.mappedConfig
 
-	movieStudioMap := movieScraperConfig.Studio
-	movieTagsMap := movieScraperConfig.Tags
+	galleryPerformersMap := galleryScraperConfig.Performers
+	galleryTagsMap := galleryScraperConfig.Tags
+	galleryStudioMap := galleryScraperConfig.Studio
 
-	results := movieMap.process(ctx, q, s.Common)
+	logger.Debug(`Processing gallery:`)
+	results := galleryMap.process(ctx, q, s.Common, urlsIsMulti)
 
-	if movieStudioMap != nil {
-		logger.Debug(`Processing movie studio:`)
-		studioResults := movieStudioMap.process(ctx, q, s.Common)
+	// now apply the performers and tags
+	if galleryPerformersMap != nil {
+		logger.Debug(`Processing gallery performers:`)
+		performerResults := galleryPerformersMap.process(ctx, q, s.Common, urlsIsMulti)
+
+		for _, p := range performerResults {
+			performer := &models.ScrapedPerformer{}
+			p.apply(performer)
+			ret.Performers = append(ret.Performers, performer)
+		}
+	}
+
+	if galleryTagsMap != nil {
+		logger.Debug(`Processing gallery tags:`)
+		tagResults := galleryTagsMap.process(ctx, q, s.Common, nil)
+
+		for _, p := range tagResults {
+			tag := &models.ScrapedTag{}
+			p.apply(tag)
+			ret.Tags = append(ret.Tags, tag)
+		}
+	}
+
+	if galleryStudioMap != nil {
+		logger.Debug(`Processing gallery studio:`)
+		studioResults := galleryStudioMap.process(ctx, q, s.Common, nil)
+
+		if len(studioResults) > 0 {
+			studio := &models.ScrapedStudio{}
+			studioResults[0].apply(studio)
+			ret.Studio = studio
+		}
+	}
+
+	// if no basic fields are populated, and no relationships, then return nil
+	if len(results) == 0 && len(ret.Performers) == 0 && len(ret.Tags) == 0 && ret.Studio == nil {
+		return nil, nil
+	}
+
+	if len(results) > 0 {
+		results[0].apply(&ret)
+	}
+
+	return &ret, nil
+}
+
+func (s mappedScraper) scrapeGroup(ctx context.Context, q mappedQuery) (*models.ScrapedGroup, error) {
+	var ret models.ScrapedGroup
+
+	// try group scraper first, falling back to movie
+	groupScraperConfig := s.Group
+
+	if groupScraperConfig == nil {
+		groupScraperConfig = s.Movie
+	}
+	if groupScraperConfig == nil {
+		return nil, nil
+	}
+
+	groupMap := groupScraperConfig.mappedConfig
+
+	groupStudioMap := groupScraperConfig.Studio
+	groupTagsMap := groupScraperConfig.Tags
+
+	results := groupMap.process(ctx, q, s.Common, urlsIsMulti)
+
+	if groupStudioMap != nil {
+		logger.Debug(`Processing group studio:`)
+		studioResults := groupStudioMap.process(ctx, q, s.Common, nil)
 
 		if len(studioResults) > 0 {
 			studio := &models.ScrapedStudio{}
@@ -1106,9 +1282,9 @@ func (s mappedScraper) scrapeGroup(ctx context.Context, q mappedQuery) (*models.
 	}
 
 	// now apply the tags
-	if movieTagsMap != nil {
-		logger.Debug(`Processing movie tags:`)
-		tagResults := movieTagsMap.process(ctx, q, s.Common)
+	if groupTagsMap != nil {
+		logger.Debug(`Processing group tags:`)
+		tagResults := groupTagsMap.process(ctx, q, s.Common, nil)
 
 		for _, p := range tagResults {
 			tag := &models.ScrapedTag{}

pkg/scraper/postprocessing.go

@@ -2,56 +2,76 @@ package scraper
 
 import (
 	"context"
+	"regexp"
 
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/utils"
 )
 
 // postScrape handles post-processing of scraped content. If the content
 // requires post-processing, this function fans out to the given content
 // type and post-processes it.
-func (c Cache) postScrape(ctx context.Context, content ScrapedContent) (ScrapedContent, error) {
+func (c Cache) postScrape(ctx context.Context, content ScrapedContent, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
 	// Analyze the concrete type, call the right post-processing function
 	switch v := content.(type) {
 	case *models.ScrapedPerformer:
 		if v != nil {
-			return c.postScrapePerformer(ctx, *v)
+			return c.postScrapePerformer(ctx, *v, excludeTagRE)
 		}
 	case models.ScrapedPerformer:
-		return c.postScrapePerformer(ctx, v)
-	case *ScrapedScene:
+		return c.postScrapePerformer(ctx, v, excludeTagRE)
+	case *models.ScrapedScene:
 		if v != nil {
-			return c.postScrapeScene(ctx, *v)
+			return c.postScrapeScene(ctx, *v, excludeTagRE)
 		}
-	case ScrapedScene:
-		return c.postScrapeScene(ctx, v)
-	case *ScrapedGallery:
+	case models.ScrapedScene:
+		return c.postScrapeScene(ctx, v, excludeTagRE)
+	case *models.ScrapedGallery:
 		if v != nil {
-			return c.postScrapeGallery(ctx, *v)
+			return c.postScrapeGallery(ctx, *v, excludeTagRE)
 		}
-	case ScrapedGallery:
-		return c.postScrapeGallery(ctx, v)
+	case models.ScrapedGallery:
+		return c.postScrapeGallery(ctx, v, excludeTagRE)
+	case *models.ScrapedImage:
+		if v != nil {
+			return c.postScrapeImage(ctx, *v, excludeTagRE)
+		}
+	case models.ScrapedImage:
+		return c.postScrapeImage(ctx, v, excludeTagRE)
 	case *models.ScrapedMovie:
 		if v != nil {
-			return c.postScrapeMovie(ctx, *v)
+			return c.postScrapeMovie(ctx, *v, excludeTagRE)
 		}
 	case models.ScrapedMovie:
-		return c.postScrapeMovie(ctx, v)
+		return c.postScrapeMovie(ctx, v, excludeTagRE)
 	case *models.ScrapedGroup:
 		if v != nil {
-			return c.postScrapeGroup(ctx, *v)
+			return c.postScrapeGroup(ctx, *v, excludeTagRE)
 		}
 	case models.ScrapedGroup:
-		return c.postScrapeGroup(ctx, v)
+		return c.postScrapeGroup(ctx, v, excludeTagRE)
 	}
 
 	// If nothing matches, pass the content through
-	return content, nil
+	return content, nil, nil
 }
 
-func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerformer) (ScrapedContent, error) {
+// postScrapeSingle handles post-processing of a single scraped content item.
+// This is a convenience function that includes logging the ignored tags, as opposed to logging them in the caller.
+func (c Cache) postScrapeSingle(ctx context.Context, content ScrapedContent) (ScrapedContent, error) {
+	ret, ignoredTags, err := c.postScrape(ctx, content, c.compileExcludeTagPatterns())
+	if err != nil {
+		return nil, err
+	}
+
+	LogIgnoredTags(ignoredTags)
+	return ret, nil
+}
+
+func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerformer, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
 	r := c.repository
 	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
 		tqb := r.TagFinder
@@ -60,11 +80,11 @@ func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerforme
 		if err != nil {
 			return err
 		}
-		p.Tags = tags
+		p.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		return nil
 	}); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// post-process - set the image if applicable
@@ -99,10 +119,10 @@ func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerforme
 		}
 	}
 
-	return p, nil
+	return p, ignoredTags, nil
 }
 
-func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie) (ScrapedContent, error) {
+func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
 	r := c.repository
 	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
 		tqb := r.TagFinder
@@ -110,17 +130,17 @@ func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie) (Scra
 		if err != nil {
 			return err
 		}
-		m.Tags = tags
+		m.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		if m.Studio != nil {
-			if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, nil); err != nil {
+			if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, ""); err != nil {
 				return err
 			}
 		}
 
 		return nil
 	}); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// post-process - set the image if applicable
@@ -131,10 +151,10 @@ func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie) (Scra
 		logger.Warnf("could not set back image using URL %s: %v", *m.BackImage, err)
 	}
 
-	return m, nil
+	return m, ignoredTags, nil
 }
 
-func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup) (ScrapedContent, error) {
+func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
 	r := c.repository
 	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
 		tqb := r.TagFinder
@@ -142,17 +162,17 @@ func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup) (Scra
 		if err != nil {
 			return err
 		}
-		m.Tags = tags
+		m.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		if m.Studio != nil {
-			if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, nil); err != nil {
+			if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, ""); err != nil {
 				return err
 			}
 		}
 
 		return nil
 	}); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// post-process - set the image if applicable
@@ -163,24 +183,25 @@ func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup) (Scra
 		logger.Warnf("could not set back image using URL %s: %v", *m.BackImage, err)
 	}
 
-	return m, nil
+	return m, ignoredTags, nil
 }
 
-func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPerformer) error {
+func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPerformer, excludeTagRE []*regexp.Regexp) (ignoredTags []string, err error) {
 	tqb := c.repository.TagFinder
 
 	tags, err := postProcessTags(ctx, tqb, p.Tags)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	p.Tags = tags
+	p.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 	p.Country = resolveCountryName(p.Country)
 
-	return nil
+	return ignoredTags, nil
 }
 
-func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene) (ScrapedContent, error) {
+func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
 	// set the URL/URLs field
 	if scene.URL == nil && len(scene.URLs) > 0 {
 		scene.URL = &scene.URLs[0]
@@ -201,13 +222,16 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene) (Scraped
 				continue
 			}
 
-			if err := c.postScrapeScenePerformer(ctx, *p); err != nil {
+			thisIgnoredTags, err := c.postScrapeScenePerformer(ctx, *p, excludeTagRE)
+			if err != nil {
 				return err
 			}
 
-			if err := match.ScrapedPerformer(ctx, pqb, p, nil); err != nil {
+			if err := match.ScrapedPerformer(ctx, pqb, p, ""); err != nil {
 				return err
 			}
+
+			ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
 		}
 
 		for _, p := range scene.Movies {
@@ -250,10 +274,10 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene) (Scraped
 		if err != nil {
 			return err
 		}
-		scene.Tags = tags
+		scene.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		if scene.Studio != nil {
-			err := match.ScrapedStudio(ctx, sqb, scene.Studio, nil)
+			err := match.ScrapedStudio(ctx, sqb, scene.Studio, "")
 			if err != nil {
 				return err
 			}
@@ -261,7 +285,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene) (Scraped
 
 		return nil
 	}); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// post-process - set the image if applicable
@@ -269,10 +293,10 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene) (Scraped
 		logger.Warnf("Could not set image using URL %s: %v", *scene.Image, err)
 	}
 
-	return scene, nil
+	return scene, ignoredTags, nil
 }
 
-func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery) (ScrapedContent, error) {
+func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
 	// set the URL/URLs field
 	if g.URL == nil && len(g.URLs) > 0 {
 		g.URL = &g.URLs[0]
@@ -288,7 +312,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery) (Scraped
 		sqb := r.StudioFinder
 
 		for _, p := range g.Performers {
-			err := match.ScrapedPerformer(ctx, pqb, p, nil)
+			err := match.ScrapedPerformer(ctx, pqb, p, "")
 			if err != nil {
 				return err
 			}
@@ -298,10 +322,10 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery) (Scraped
 		if err != nil {
 			return err
 		}
-		g.Tags = tags
+		g.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		if g.Studio != nil {
-			err := match.ScrapedStudio(ctx, sqb, g.Studio, nil)
+			err := match.ScrapedStudio(ctx, sqb, g.Studio, "")
 			if err != nil {
 				return err
 			}
@@ -309,22 +333,43 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery) (Scraped
 
 		return nil
 	}); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	return g, nil
+	return g, ignoredTags, nil
 }
 
-func postProcessTags(ctx context.Context, tqb models.TagQueryer, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) {
-	var ret []*models.ScrapedTag
+func (c Cache) postScrapeImage(ctx context.Context, image models.ScrapedImage, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
+	r := c.repository
+	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
+		pqb := r.PerformerFinder
+		tqb := r.TagFinder
+		sqb := r.StudioFinder
 
-	for _, t := range scrapedTags {
-		err := match.ScrapedTag(ctx, tqb, t)
-		if err != nil {
-			return nil, err
+		for _, p := range image.Performers {
+			if err := match.ScrapedPerformer(ctx, pqb, p, ""); err != nil {
+				return err
+			}
 		}
-		ret = append(ret, t)
+
+		tags, err := postProcessTags(ctx, tqb, image.Tags)
+		if err != nil {
+			return err
+		}
+
+		image.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
+
+		if image.Studio != nil {
+			err := match.ScrapedStudio(ctx, sqb, image.Studio, "")
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return nil, nil, err
 	}
 
-	return ret, nil
+	return image, ignoredTags, nil
 }

pkg/scraper/query_url.go

@@ -26,7 +26,7 @@ func queryURLParametersFromScene(scene *models.Scene) queryURLParameters {
 	return ret
 }
 
-func queryURLParametersFromScrapedScene(scene ScrapedSceneInput) queryURLParameters {
+func queryURLParametersFromScrapedScene(scene models.ScrapedSceneInput) queryURLParameters {
 	ret := make(queryURLParameters)
 
 	setField := func(field string, value *string) {
@@ -73,6 +73,24 @@ func queryURLParametersFromGallery(gallery *models.Gallery) queryURLParameters {
 	return ret
 }
 
+func queryURLParametersFromImage(image *models.Image) queryURLParameters {
+	ret := make(queryURLParameters)
+	ret["checksum"] = image.Checksum
+
+	if image.Path != "" {
+		ret["filename"] = filepath.Base(image.Path)
+	}
+	if image.Title != "" {
+		ret["title"] = image.Title
+	}
+
+	if len(image.URLs.List()) > 0 {
+		ret["url"] = image.URLs.List()[0]
+	}
+
+	return ret
+}
+
 func (p queryURLParameters) applyReplacements(r queryURLReplacements) {
 	for k, v := range p {
 		rpl, found := r[k]

pkg/scraper/scene.go

@@ -1,39 +0,0 @@
-package scraper
-
-import (
-	"github.com/stashapp/stash/pkg/models"
-)
-
-type ScrapedScene struct {
-	Title    *string  `json:"title"`
-	Code     *string  `json:"code"`
-	Details  *string  `json:"details"`
-	Director *string  `json:"director"`
-	URL      *string  `json:"url"`
-	URLs     []string `json:"urls"`
-	Date     *string  `json:"date"`
-	// This should be a base64 encoded data URL
-	Image        *string                       `json:"image"`
-	File         *models.SceneFileType         `json:"file"`
-	Studio       *models.ScrapedStudio         `json:"studio"`
-	Tags         []*models.ScrapedTag          `json:"tags"`
-	Performers   []*models.ScrapedPerformer    `json:"performers"`
-	Groups       []*models.ScrapedGroup        `json:"groups"`
-	Movies       []*models.ScrapedMovie        `json:"movies"`
-	RemoteSiteID *string                       `json:"remote_site_id"`
-	Duration     *int                          `json:"duration"`
-	Fingerprints []*models.StashBoxFingerprint `json:"fingerprints"`
-}
-
-func (ScrapedScene) IsScrapedContent() {}
-
-type ScrapedSceneInput struct {
-	Title        *string  `json:"title"`
-	Code         *string  `json:"code"`
-	Details      *string  `json:"details"`
-	Director     *string  `json:"director"`
-	URL          *string  `json:"url"`
-	URLs         []string `json:"urls"`
-	Date         *string  `json:"date"`
-	RemoteSiteID *string  `json:"remote_site_id"`
-}

pkg/scraper/scraper.go

@@ -36,6 +36,7 @@ const (
 	ScrapeContentTypeGroup     ScrapeContentType = "GROUP"
 	ScrapeContentTypePerformer ScrapeContentType = "PERFORMER"
 	ScrapeContentTypeScene     ScrapeContentType = "SCENE"
+	ScrapeContentTypeImage     ScrapeContentType = "IMAGE"
 )
 
 var AllScrapeContentType = []ScrapeContentType{
@@ -44,11 +45,12 @@ var AllScrapeContentType = []ScrapeContentType{
 	ScrapeContentTypeGroup,
 	ScrapeContentTypePerformer,
 	ScrapeContentTypeScene,
+	ScrapeContentTypeImage,
 }
 
 func (e ScrapeContentType) IsValid() bool {
 	switch e {
-	case ScrapeContentTypeGallery, ScrapeContentTypeMovie, ScrapeContentTypeGroup, ScrapeContentTypePerformer, ScrapeContentTypeScene:
+	case ScrapeContentTypeGallery, ScrapeContentTypeMovie, ScrapeContentTypeGroup, ScrapeContentTypePerformer, ScrapeContentTypeScene, ScrapeContentTypeImage:
 		return true
 	}
 	return false
@@ -84,6 +86,8 @@ type Scraper struct {
 	Scene *ScraperSpec `json:"scene"`
 	// Details for gallery scraper
 	Gallery *ScraperSpec `json:"gallery"`
+	// Details for image scraper
+	Image *ScraperSpec `json:"image"`
 	// Details for movie scraper
 	Group *ScraperSpec `json:"group"`
 	// Details for movie scraper
@@ -159,8 +163,9 @@ var (
 // set to nil.
 type Input struct {
 	Performer *ScrapedPerformerInput
-	Scene     *ScrapedSceneInput
-	Gallery   *ScrapedGalleryInput
+	Scene     *models.ScrapedSceneInput
+	Gallery   *models.ScrapedGalleryInput
+	Image     *models.ScrapedImageInput
 }
 
 // populateURL populates the URL field of the input based on the
@@ -222,7 +227,15 @@ type fragmentScraper interface {
 type sceneScraper interface {
 	scraper
 
-	viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error)
+	viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*models.ScrapedScene, error)
+}
+
+// imageScraper is a scraper which supports image scrapes with
+// image data as the input.
+type imageScraper interface {
+	scraper
+
+	viaImage(ctx context.Context, client *http.Client, image *models.Image) (*models.ScrapedImage, error)
 }
 
 // galleryScraper is a scraper which supports gallery scrapes with
@@ -230,5 +243,5 @@ type sceneScraper interface {
 type galleryScraper interface {
 	scraper
 
-	viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error)
+	viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error)
 }

pkg/scraper/script.go

@@ -328,7 +328,7 @@ func (s *scriptScraper) scrapeByName(ctx context.Context, name string, ty Scrape
 			}
 		}
 	case ScrapeContentTypeScene:
-		var scenes []ScrapedScene
+		var scenes []models.ScrapedScene
 		err = s.runScraperScript(ctx, input, &scenes)
 		if err == nil {
 			for _, s := range scenes {
@@ -377,44 +377,62 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
 		err := s.runScraperScript(ctx, input, &performer)
 		return performer, err
 	case ScrapeContentTypeGallery:
-		var gallery *ScrapedGallery
+		var gallery *models.ScrapedGallery
 		err := s.runScraperScript(ctx, input, &gallery)
 		return gallery, err
 	case ScrapeContentTypeScene:
-		var scene *ScrapedScene
+		var scene *models.ScrapedScene
 		err := s.runScraperScript(ctx, input, &scene)
 		return scene, err
 	case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
 		var movie *models.ScrapedMovie
 		err := s.runScraperScript(ctx, input, &movie)
 		return movie, err
+	case ScrapeContentTypeImage:
+		var image *models.ScrapedImage
+		err := s.runScraperScript(ctx, input, &image)
+		return image, err
 	}
 
 	return nil, ErrNotSupported
 }
 
-func (s *scriptScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
+func (s *scriptScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
 	inString, err := json.Marshal(sceneInputFromScene(scene))
 
 	if err != nil {
 		return nil, err
 	}
 
-	var ret *ScrapedScene
+	var ret *models.ScrapedScene
 
 	err = s.runScraperScript(ctx, string(inString), &ret)
 
 	return ret, err
 }
 
-func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	inString, err := json.Marshal(galleryInputFromGallery(gallery))
 
 	if err != nil {
 		return nil, err
 	}
 
-	var ret *ScrapedGallery
+	var ret *models.ScrapedGallery
+
+	err = s.runScraperScript(ctx, string(inString), &ret)
+
+	return ret, err
+}
+
+func (s *scriptScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
+	inString, err := json.Marshal(imageToUpdateInput(image))
+
+	if err != nil {
+		return nil, err
+	}
+
+	var ret *models.ScrapedImage
 
 	err = s.runScraperScript(ctx, string(inString), &ret)
 

pkg/scraper/stash.go

@@ -54,7 +54,7 @@ type stashFindPerformerNamePerformer struct {
 func (p stashFindPerformerNamePerformer) toPerformer() *models.ScrapedPerformer {
 	return &models.ScrapedPerformer{
 		Name: &p.Name,
-		// put id into the URL field
+		// HACK - put id into the URL field
 		URL: &p.ID,
 	}
 }
@@ -107,16 +107,18 @@ func (s *stashScraper) imageGetter() imageGetter {
 }
 
 func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (ScrapedContent, error) {
-	if input.Gallery != nil || input.Scene != nil {
-		return nil, fmt.Errorf("%w: using stash scraper as a fragment scraper", ErrNotSupported)
+	if input.Performer != nil {
+		return s.scrapeByPerformerFragment(ctx, *input.Performer)
 	}
 
-	if input.Performer == nil {
-		return nil, fmt.Errorf("%w: the given performer is nil", ErrNotSupported)
+	if input.Scene != nil {
+		return s.scrapeBySceneFragment(ctx, *input.Scene)
 	}
 
-	scrapedPerformer := input.Performer
+	return nil, fmt.Errorf("%w: using stash scraper as a fragment scraper", ErrNotSupported)
+}
 
+func (s *stashScraper) scrapeByPerformerFragment(ctx context.Context, scrapedPerformer ScrapedPerformerInput) (ScrapedContent, error) {
 	client := s.getStashClient()
 
 	var q struct {
@@ -168,6 +170,45 @@ func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap
 	return &ret, nil
 }
 
+func (s *stashScraper) scrapeBySceneFragment(ctx context.Context, scrapedScene models.ScrapedSceneInput) (ScrapedContent, error) {
+	client := s.getStashClient()
+
+	var q struct {
+		FindScene *scrapedSceneStash `graphql:"findScene(id: $f)"`
+	}
+
+	sceneID := scrapedScene.URLs[0]
+
+	// get the id from the URL field
+	vars := map[string]interface{}{
+		"f": graphql.ID(sceneID),
+	}
+
+	err := client.Query(ctx, &q, vars)
+	if err != nil {
+		return nil, convertGraphqlError(err)
+	}
+
+	if q.FindScene == nil {
+		return nil, nil
+	}
+
+	// need to copy back to a scraped scene
+	ret, err := s.scrapedStashSceneToScrapedScene(ctx, q.FindScene)
+	if err != nil {
+		return nil, err
+	}
+
+	// get the scene image directly
+	ig := s.imageGetter()
+	ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, ig)
+	if err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
 type scrapedStudioStash struct {
 	Name string  `graphql:"name" json:"name"`
 	URL  *string `graphql:"url" json:"url"`
@@ -178,8 +219,8 @@ type stashFindSceneNamesResultType struct {
 	Scenes []*scrapedSceneStash `graphql:"scenes"`
 }
 
-func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scene *scrapedSceneStash) (*ScrapedScene, error) {
-	ret := ScrapedScene{}
+func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scene *scrapedSceneStash) (*models.ScrapedScene, error) {
+	ret := models.ScrapedScene{}
 	err := copier.Copy(&ret, scene)
 	if err != nil {
 		return nil, err
@@ -232,6 +273,10 @@ func (s *stashScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
 			if err != nil {
 				return nil, err
 			}
+
+			// HACK - put id into the URL field
+			// put id into the URL field
+			converted.URLs = []string{scene.ID}
 			ret = append(ret, converted)
 		}
 
@@ -296,7 +341,7 @@ type scrapedSceneStash struct {
 	Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
 }
 
-func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
+func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
 	// query by MD5
 	var q struct {
 		FindScene *scrapedSceneStash `graphql:"findSceneByHash(input: $c)"`
@@ -334,7 +379,7 @@ func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sce
 		return nil, err
 	}
 
-	// get the performer image directly
+	// get the scene image directly
 	ig := s.imageGetter()
 	ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, ig)
 	if err != nil {
@@ -356,7 +401,7 @@ type scrapedGalleryStash struct {
 	Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
 }
 
-func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	var q struct {
 		FindGallery *scrapedGalleryStash `graphql:"findGalleryByHash(input: $c)"`
 	}
@@ -380,7 +425,7 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
 	}
 
 	// need to copy back to a scraped scene
-	ret := ScrapedGallery{}
+	ret := models.ScrapedGallery{}
 	if err := copier.Copy(&ret, q.FindGallery); err != nil {
 		return nil, err
 	}
@@ -388,6 +433,33 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
 	return &ret, nil
 }
 
+func (s *stashScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
+	return nil, ErrNotSupported
+}
+
 func (s *stashScraper) scrapeByURL(_ context.Context, _ string, _ ScrapeContentType) (ScrapedContent, error) {
 	return nil, ErrNotSupported
 }
+
+func imageToUpdateInput(gallery *models.Image) models.ImageUpdateInput {
+	dateToStringPtr := func(s *models.Date) *string {
+		if s != nil {
+			v := s.String()
+			return &v
+		}
+
+		return nil
+	}
+
+	// fallback to file basename if title is empty
+	title := gallery.GetTitle()
+	urls := gallery.URLs.List()
+
+	return models.ImageUpdateInput{
+		ID:      strconv.Itoa(gallery.ID),
+		Title:   &title,
+		Details: &gallery.Details,
+		Urls:    urls,
+		Date:    dateToStringPtr(gallery.Date),
+	}
+}

pkg/scraper/stashbox/models.go

@@ -1,13 +0,0 @@
-package stashbox
-
-import "github.com/stashapp/stash/pkg/models"
-
-type StashBoxStudioQueryResult struct {
-	Query   string                  `json:"query"`
-	Results []*models.ScrapedStudio `json:"results"`
-}
-
-type StashBoxPerformerQueryResult struct {
-	Query   string                     `json:"query"`
-	Results []*models.ScrapedPerformer `json:"results"`
-}

pkg/scraper/tag.go

@@ -0,0 +1,77 @@
+package scraper
+
+import (
+	"context"
+	"regexp"
+	"strings"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/match"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/sliceutil"
+)
+
+func postProcessTags(ctx context.Context, tqb models.TagQueryer, scrapedTags []*models.ScrapedTag) (ret []*models.ScrapedTag, err error) {
+	ret = make([]*models.ScrapedTag, 0, len(scrapedTags))
+
+	for _, t := range scrapedTags {
+		err := match.ScrapedTag(ctx, tqb, t)
+		if err != nil {
+			return nil, err
+		}
+		ret = append(ret, t)
+	}
+
+	return ret, err
+}
+
+// FilterTags removes tags matching excluded tag patterns from the list of scraped tags
+// It returns the filtered list of tags and a list of the excluded tags
+func FilterTags(excludeRegexps []*regexp.Regexp, tags []*models.ScrapedTag) (newTags []*models.ScrapedTag, ignoredTags []string) {
+	if len(excludeRegexps) == 0 {
+		return tags, nil
+	}
+
+	newTags = make([]*models.ScrapedTag, 0, len(tags))
+
+	for _, t := range tags {
+		ignore := false
+		for _, reg := range excludeRegexps {
+			if reg.MatchString(strings.ToLower(t.Name)) {
+				ignore = true
+				ignoredTags = sliceutil.AppendUnique(ignoredTags, t.Name)
+				break
+			}
+		}
+
+		if !ignore {
+			newTags = append(newTags, t)
+		}
+	}
+
+	return newTags, ignoredTags
+}
+
+// CompileExclusionRegexps compiles a list of tag exclusion patterns into a list of regular expressions
+func CompileExclusionRegexps(patterns []string) []*regexp.Regexp {
+	excludePatterns := patterns
+	var excludeRegexps []*regexp.Regexp
+
+	for _, excludePattern := range excludePatterns {
+		reg, err := regexp.Compile(strings.ToLower(excludePattern))
+		if err != nil {
+			logger.Errorf("Invalid tag exclusion pattern: %v", err)
+		} else {
+			excludeRegexps = append(excludeRegexps, reg)
+		}
+	}
+
+	return excludeRegexps
+}
+
+// LogIgnoredTags logs the list of ignored tags
+func LogIgnoredTags(ignoredTags []string) {
+	if len(ignoredTags) > 0 {
+		logger.Debugf("Tags ignored for matching exclusion patterns: %s", strings.Join(ignoredTags, ", "))
+	}
+}

pkg/scraper/xpath.go

@@ -83,6 +83,12 @@ func (s *xpathScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCon
 			return nil, err
 		}
 		return ret, nil
+	case ScrapeContentTypeImage:
+		ret, err := scraper.scrapeImage(ctx, q)
+		if err != nil || ret == nil {
+			return nil, err
+		}
+		return ret, nil
 	case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
 		ret, err := scraper.scrapeGroup(ctx, q)
 		if err != nil || ret == nil {
@@ -145,7 +151,7 @@ func (s *xpathScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
 	return nil, ErrNotSupported
 }
 
-func (s *xpathScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
+func (s *xpathScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromScene(scene)
 	if s.scraper.QueryURLReplacements != nil {
@@ -204,7 +210,7 @@ func (s *xpathScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap
 	return scraper.scrapeScene(ctx, q)
 }
 
-func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromGallery(gallery)
 	if s.scraper.QueryURLReplacements != nil {
@@ -228,6 +234,30 @@ func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
 	return scraper.scrapeGallery(ctx, q)
 }
 
+func (s *xpathScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
+	// construct the URL
+	queryURL := queryURLParametersFromImage(image)
+	if s.scraper.QueryURLReplacements != nil {
+		queryURL.applyReplacements(s.scraper.QueryURLReplacements)
+	}
+	url := queryURL.constructURL(s.scraper.QueryURL)
+
+	scraper := s.getXpathScraper()
+
+	if scraper == nil {
+		return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config")
+	}
+
+	doc, err := s.loadURL(ctx, url)
+
+	if err != nil {
+		return nil, err
+	}
+
+	q := s.getXPathQuery(doc)
+	return scraper.scrapeImage(ctx, q)
+}
+
 func (s *xpathScraper) loadURL(ctx context.Context, url string) (*html.Node, error) {
 	r, err := loadURL(ctx, url, s.client, s.config, s.globalConfig)
 	if err != nil {

pkg/scraper/xpath_test.go

@@ -32,6 +32,7 @@ const htmlDoc1 = `
 							</td>
 							<td class="paramvalue">
 								<a href="/html/m_links/Mia_Malkova/">Mia Malkova</a>&nbsp;
+								<a href="/html/m_links/Mia_Malkova/second_url">Mia Malkova</a>&nbsp;
 							</td>
 						</tr>
 						<tr>
@@ -206,6 +207,8 @@ func makeXPathConfig() mappedPerformerScraperConfig {
 	}
 
 	config.mappedConfig["Name"] = makeSimpleAttrConfig(makeCommonXPath("Babe Name:") + `/a`)
+	config.mappedConfig["URL"] = makeSimpleAttrConfig(makeCommonXPath("Babe Name:") + `/a/@href`)
+	config.mappedConfig["URLs"] = makeSimpleAttrConfig(makeCommonXPath("Babe Name:") + `/a/@href`)
 	config.mappedConfig["Ethnicity"] = makeSimpleAttrConfig(makeCommonXPath("Ethnicity:"))
 	config.mappedConfig["Aliases"] = makeSimpleAttrConfig(makeCommonXPath("Aliases:"))
 	config.mappedConfig["EyeColor"] = makeSimpleAttrConfig(makeCommonXPath("Eye Color:"))
@@ -321,6 +324,8 @@ func TestScrapePerformerXPath(t *testing.T) {
 	}
 
 	const performerName = "Mia Malkova"
+	const url = "/html/m_links/Mia_Malkova/"
+	const secondURL = "/html/m_links/Mia_Malkova/second_url"
 	const ethnicity = "Caucasian"
 	const country = "United States"
 	const birthdate = "1992-07-01"
@@ -338,6 +343,16 @@ func TestScrapePerformerXPath(t *testing.T) {
 	const weight = "57" // 126 lb
 
 	verifyField(t, performerName, performer.Name, "Name")
+	verifyField(t, url, performer.URL, "URL")
+
+	// #5294 - test multiple URLs
+	if len(performer.URLs) != 2 {
+		t.Errorf("Expected 2 URLs, got %d", len(performer.URLs))
+	} else {
+		verifyField(t, url, &performer.URLs[0], "URLs[0]")
+		verifyField(t, secondURL, &performer.URLs[1], "URLs[1]")
+	}
+
 	verifyField(t, gender, performer.Gender, "Gender")
 	verifyField(t, ethnicity, performer.Ethnicity, "Ethnicity")
 	verifyField(t, country, performer.Country, "Country")
@@ -569,7 +584,7 @@ func makeSceneXPathConfig() mappedScraper {
 
 	performerConfig := make(mappedConfig)
 	performerConfig["Name"] = makeSimpleAttrConfig(`$performerElem/@data-mxptext`)
-	performerConfig["URL"] = makeSimpleAttrConfig(`$performerElem/@href`)
+	performerConfig["URLs"] = makeSimpleAttrConfig(`$performerElem/@href`)
 	config.Performers.mappedConfig = performerConfig
 
 	studioConfig := make(mappedConfig)
@@ -653,8 +668,8 @@ func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []strin
 		}
 		if i < len(actualPerformers) {
 			actualName = *actualPerformers[i].Name
-			if actualPerformers[i].URL != nil {
-				actualURL = *actualPerformers[i].URL
+			if len(actualPerformers[i].URLs) == 1 {
+				actualURL = actualPerformers[i].URLs[0]
 			}
 		}
 
@@ -805,7 +820,7 @@ func TestLoadInvalidXPath(t *testing.T) {
 		doc: doc,
 	}
 
-	config.process(context.Background(), q, nil)
+	config.process(context.Background(), q, nil, nil)
 }
 
 type mockGlobalConfig struct{}

pkg/sliceutil/collections.go

@@ -167,3 +167,13 @@ func ValuesToPtrs[T any](vs []T) []*T {
 	}
 	return ret
 }
+
+// Flatten returns a single slice containing all elements of the provided
+// slice of slices.
+func Flatten[T any](vs [][]T) []T {
+	var ret []T
+	for _, v := range vs {
+		ret = append(ret, v...)
+	}
+	return ret
+}

pkg/sqlite/anonymise.go

@@ -816,6 +816,7 @@ func (db *Anonymiser) anonymiseTags(ctx context.Context) error {
 			query := dialect.From(table).Select(
 				table.Col(idColumn),
 				table.Col("name"),
+				table.Col("sort_name"),
 				table.Col("description"),
 			).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
 
@@ -826,12 +827,14 @@ func (db *Anonymiser) anonymiseTags(ctx context.Context) error {
 				var (
 					id          int
 					name        sql.NullString
+					sortName    sql.NullString
 					description sql.NullString
 				)
 
 				if err := rows.Scan(
 					&id,
 					&name,
+					&sortName,
 					&description,
 				); err != nil {
 					return err
@@ -839,6 +842,7 @@ func (db *Anonymiser) anonymiseTags(ctx context.Context) error {
 
 				set := goqu.Record{}
 				db.obfuscateNullString(set, "name", name)
+				db.obfuscateNullString(set, "sort_name", sortName)
 				db.obfuscateNullString(set, "description", description)
 
 				if len(set) > 0 {

pkg/sqlite/database.go

@@ -34,7 +34,7 @@ const (
 	cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE"
 )
 
-var appSchemaVersion uint = 71
+var appSchemaVersion uint = 72
 
 //go:embed migrations/*.sql
 var migrationsBox embed.FS
@@ -430,7 +430,19 @@ func (db *Database) Vacuum(ctx context.Context) error {
 
 // Analyze runs an ANALYZE on the database to improve query performance.
 func (db *Database) Analyze(ctx context.Context) error {
-	_, err := db.writeDB.ExecContext(ctx, "ANALYZE")
+	return analyze(ctx, db.writeDB)
+}
+
+// analyze runs an ANALYZE on the database to improve query performance.
+func analyze(ctx context.Context, db *sqlx.DB) error {
+	_, err := db.ExecContext(ctx, "ANALYZE")
+	return err
+}
+
+// flushWAL flushes the Write-Ahead Log (WAL) to the main database file.
+// It also truncates the WAL file to 0 bytes.
+func flushWAL(ctx context.Context, db *sqlx.DB) error {
+	_, err := db.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
 	return err
 }
 

pkg/sqlite/gallery.go

@@ -155,7 +155,7 @@ var (
 			},
 			fkColumn:     "tag_id",
 			foreignTable: tagTable,
-			orderBy:      "tags.name ASC",
+			orderBy:      "COALESCE(tags.sort_name, tags.name) ASC",
 		},
 		images: joinRepository{
 			repository: repository{

pkg/sqlite/group.go

@@ -122,7 +122,7 @@ var (
 			},
 			fkColumn:     tagIDColumn,
 			foreignTable: tagTable,
-			orderBy:      "tags.name ASC",
+			orderBy:      "COALESCE(tags.sort_name, tags.name) ASC",
 		},
 	}
 )

pkg/sqlite/image.go

@@ -177,7 +177,7 @@ var (
 			},
 			fkColumn:     tagIDColumn,
 			foreignTable: tagTable,
-			orderBy:      "tags.name ASC",
+			orderBy:      "COALESCE(tags.sort_name, tags.name) ASC",
 		},
 	}
 )

pkg/sqlite/migrate.go

@@ -39,6 +39,12 @@ func NewMigrator(db *Database) (*Migrator, error) {
 	m.conn.SetConnMaxIdleTime(dbConnTimeout)
 
 	m.m, err = m.getMigrate()
+
+	// if error encountered, close the connection
+	if err != nil {
+		m.Close()
+	}
+
 	return m, err
 }
 
@@ -124,6 +130,27 @@ func (m *Migrator) runCustomMigration(ctx context.Context, fn customMigrationFun
 	return nil
 }
 
+func (m *Migrator) PostMigrate(ctx context.Context) error {
+	// optimise the database
+	var err error
+	logger.Info("Running database analyze")
+
+	// don't use Optimize/vacuum as this adds a significant amount of time
+	// to the migration
+	err = analyze(ctx, m.conn)
+
+	if err == nil {
+		logger.Debug("Flushing WAL")
+		err = flushWAL(ctx, m.conn)
+	}
+
+	if err != nil {
+		return fmt.Errorf("error optimising database: %s", err)
+	}
+
+	return nil
+}
+
 func (db *Database) getDatabaseSchemaVersion() (uint, error) {
 	m, err := NewMigrator(db)
 	if err != nil {

pkg/sqlite/migrations/72_tag_sort_name.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `tags` ADD COLUMN `sort_name` varchar(255);
+

pkg/sqlite/performer.go

@@ -189,7 +189,7 @@ var (
 			},
 			fkColumn:     tagIDColumn,
 			foreignTable: tagTable,
-			orderBy:      "tags.name ASC",
+			orderBy:      "COALESCE(tags.sort_name, tags.name) ASC",
 		},
 		stashIDs: stashIDRepository{
 			repository{

pkg/sqlite/performer_test.go

@@ -121,12 +121,14 @@ func Test_PerformerStore_Create(t *testing.T) {
 					Aliases:        models.NewRelatedStrings(aliases),
 					StashIDs: models.NewRelatedStashIDs([]models.StashID{
 						{
-							StashID:  stashID1,
-							Endpoint: endpoint1,
+							StashID:   stashID1,
+							Endpoint:  endpoint1,
+							UpdatedAt: epochTime,
 						},
 						{
-							StashID:  stashID2,
-							Endpoint: endpoint2,
+							StashID:   stashID2,
+							Endpoint:  endpoint2,
+							UpdatedAt: epochTime,
 						},
 					}),
 					CreatedAt: createdAt,
@@ -283,12 +285,14 @@ func Test_PerformerStore_Update(t *testing.T) {
 					TagIDs:         models.NewRelatedIDs([]int{tagIDs[tagIdx1WithPerformer], tagIDs[tagIdx1WithDupName]}),
 					StashIDs: models.NewRelatedStashIDs([]models.StashID{
 						{
-							StashID:  stashID1,
-							Endpoint: endpoint1,
+							StashID:   stashID1,
+							Endpoint:  endpoint1,
+							UpdatedAt: epochTime,
 						},
 						{
-							StashID:  stashID2,
-							Endpoint: endpoint2,
+							StashID:   stashID2,
+							Endpoint:  endpoint2,
+							UpdatedAt: epochTime,
 						},
 					}),
 					CreatedAt: createdAt,
@@ -518,12 +522,14 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
 				StashIDs: &models.UpdateStashIDs{
 					StashIDs: []models.StashID{
 						{
-							StashID:  stashID1,
-							Endpoint: endpoint1,
+							StashID:   stashID1,
+							Endpoint:  endpoint1,
+							UpdatedAt: epochTime,
 						},
 						{
-							StashID:  stashID2,
-							Endpoint: endpoint2,
+							StashID:   stashID2,
+							Endpoint:  endpoint2,
+							UpdatedAt: epochTime,
 						},
 					},
 					Mode: models.RelationshipUpdateModeSet,
@@ -560,12 +566,14 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
 				TagIDs:         models.NewRelatedIDs([]int{tagIDs[tagIdx1WithPerformer], tagIDs[tagIdx1WithDupName]}),
 				StashIDs: models.NewRelatedStashIDs([]models.StashID{
 					{
-						StashID:  stashID1,
-						Endpoint: endpoint1,
+						StashID:   stashID1,
+						Endpoint:  endpoint1,
+						UpdatedAt: epochTime,
 					},
 					{
-						StashID:  stashID2,
-						Endpoint: endpoint2,
+						StashID:   stashID2,
+						Endpoint:  endpoint2,
+						UpdatedAt: epochTime,
 					},
 				}),
 				CreatedAt: createdAt,
@@ -2075,8 +2083,9 @@ func testPerformerStashIDs(ctx context.Context, t *testing.T, s *models.Performe
 	const stashIDStr = "stashID"
 	const endpoint = "endpoint"
 	stashID := models.StashID{
-		StashID:  stashIDStr,
-		Endpoint: endpoint,
+		StashID:   stashIDStr,
+		Endpoint:  endpoint,
+		UpdatedAt: epochTime,
 	}
 
 	qb := db.Performer

pkg/sqlite/scene.go

@@ -201,7 +201,7 @@ var (
 			},
 			fkColumn:     tagIDColumn,
 			foreignTable: tagTable,
-			orderBy:      "tags.name ASC",
+			orderBy:      "COALESCE(tags.sort_name, tags.name) ASC",
 		},
 		performers: joinRepository{
 			repository: repository{
@@ -493,8 +493,11 @@ func (qb *SceneStore) Find(ctx context.Context, id int) (*models.Scene, error) {
 	return ret, err
 }
 
-func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene, error) {
-	scenes := make([]*models.Scene, len(ids))
+// FindByIDs finds multiple scenes by their IDs.
+// No check is made to see if the scenes exist, and the order of the returned scenes
+// is not guaranteed to be the same as the order of the input IDs.
+func (qb *SceneStore) FindByIDs(ctx context.Context, ids []int) ([]*models.Scene, error) {
+	scenes := make([]*models.Scene, 0, len(ids))
 
 	table := qb.table()
 	if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
@@ -504,16 +507,29 @@ func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene,
 			return err
 		}
 
-		for _, s := range unsorted {
-			i := slices.Index(ids, s.ID)
-			scenes[i] = s
-		}
+		scenes = append(scenes, unsorted...)
 
 		return nil
 	}); err != nil {
 		return nil, err
 	}
 
+	return scenes, nil
+}
+
+func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene, error) {
+	scenes := make([]*models.Scene, len(ids))
+
+	unsorted, err := qb.FindByIDs(ctx, ids)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, s := range unsorted {
+		i := slices.Index(ids, s.ID)
+		scenes[i] = s
+	}
+
 	for i := range scenes {
 		if scenes[i] == nil {
 			return nil, fmt.Errorf("scene with id %d not found", ids[i])
@@ -1056,6 +1072,7 @@ func (qb *SceneStore) QueryCount(ctx context.Context, sceneFilter *models.SceneF
 var sceneSortOptions = sortOptions{
 	"bitrate",
 	"created_at",
+	"code",
 	"date",
 	"file_count",
 	"filesize",

pkg/sqlite/scene_test.go

@@ -133,12 +133,14 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
 				}),
 				StashIDs: models.NewRelatedStashIDs([]models.StashID{
 					{
-						StashID:  stashID1,
-						Endpoint: endpoint1,
+						StashID:   stashID1,
+						Endpoint:  endpoint1,
+						UpdatedAt: epochTime,
 					},
 					{
-						StashID:  stashID2,
-						Endpoint: endpoint2,
+						StashID:   stashID2,
+						Endpoint:  endpoint2,
+						UpdatedAt: epochTime,
 					},
 				}),
 				ResumeTime:   float64(resumeTime),
@@ -178,12 +180,14 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
 				}),
 				StashIDs: models.NewRelatedStashIDs([]models.StashID{
 					{
-						StashID:  stashID1,
-						Endpoint: endpoint1,
+						StashID:   stashID1,
+						Endpoint:  endpoint1,
+						UpdatedAt: epochTime,
 					},
 					{
-						StashID:  stashID2,
-						Endpoint: endpoint2,
+						StashID:   stashID2,
+						Endpoint:  endpoint2,
+						UpdatedAt: epochTime,
 					},
 				}),
 				ResumeTime:   resumeTime,
@@ -362,12 +366,14 @@ func Test_sceneQueryBuilder_Update(t *testing.T) {
 				}),
 				StashIDs: models.NewRelatedStashIDs([]models.StashID{
 					{
-						StashID:  stashID1,
-						Endpoint: endpoint1,
+						StashID:   stashID1,
+						Endpoint:  endpoint1,
+						UpdatedAt: epochTime,
 					},
 					{
-						StashID:  stashID2,
-						Endpoint: endpoint2,
+						StashID:   stashID2,
+						Endpoint:  endpoint2,
+						UpdatedAt: epochTime,
 					},
 				}),
 				ResumeTime:   resumeTime,
@@ -590,12 +596,14 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
 				StashIDs: &models.UpdateStashIDs{
 					StashIDs: []models.StashID{
 						{
-							StashID:  stashID1,
-							Endpoint: endpoint1,
+							StashID:   stashID1,
+							Endpoint:  endpoint1,
+							UpdatedAt: epochTime,
 						},
 						{
-							StashID:  stashID2,
-							Endpoint: endpoint2,
+							StashID:   stashID2,
+							Endpoint:  endpoint2,
+							UpdatedAt: epochTime,
 						},
 					},
 					Mode: models.RelationshipUpdateModeSet,
@@ -634,12 +642,14 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
 				}),
 				StashIDs: models.NewRelatedStashIDs([]models.StashID{
 					{
-						StashID:  stashID1,
-						Endpoint: endpoint1,
+						StashID:   stashID1,
+						Endpoint:  endpoint1,
+						UpdatedAt: epochTime,
 					},
 					{
-						StashID:  stashID2,
-						Endpoint: endpoint2,
+						StashID:   stashID2,
+						Endpoint:  endpoint2,
+						UpdatedAt: epochTime,
 					},
 				}),
 				ResumeTime:   resumeTime,
@@ -741,12 +751,14 @@ func Test_sceneQueryBuilder_UpdatePartialRelationships(t *testing.T) {
 
 		stashIDs = []models.StashID{
 			{
-				StashID:  stashID1,
-				Endpoint: endpoint1,
+				StashID:   stashID1,
+				Endpoint:  endpoint1,
+				UpdatedAt: epochTime,
 			},
 			{
-				StashID:  stashID2,
-				Endpoint: endpoint2,
+				StashID:   stashID2,
+				Endpoint:  endpoint2,
+				UpdatedAt: epochTime,
 			},
 		}
 	)
@@ -4377,8 +4389,9 @@ func testSceneStashIDs(ctx context.Context, t *testing.T, s *models.Scene) {
 	const stashIDStr = "stashID"
 	const endpoint = "endpoint"
 	stashID := models.StashID{
-		StashID:  stashIDStr,
-		Endpoint: endpoint,
+		StashID:   stashIDStr,
+		Endpoint:  endpoint,
+		UpdatedAt: epochTime,
 	}
 
 	qb := db.Scene