Remove matrix references + add community forum

* Update markdown syntax for Scrapers Development

* Fix typo

.github/workflows/build.yml

@@ -16,7 +16,7 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v2
 

.gitignore

@@ -21,6 +21,9 @@ vendor
 # GraphQL generated output
 internal/api/generated_*.go
 
+# Generated locale files
+ui/login/locales/*
+
 ####
 # Visual Studio
 ####

Makefile

@@ -281,6 +281,10 @@ generate-ui:
 generate-backend: touch-ui
 	go generate ./cmd/stash
 
+.PHONY: generate-login-locale
+generate-login-locale:
+	go generate ./ui
+
 .PHONY: generate-dataloaders
 generate-dataloaders:
 	go generate ./internal/api/loaders
@@ -351,7 +355,10 @@ ifdef STASH_SOURCEMAPS
 endif
 
 .PHONY: ui
-ui: ui-env
+ui: ui-only generate-login-locale
+
+.PHONY: ui-only
+ui-only: ui-env
 	cd ui/v2.5 && yarn build
 
 .PHONY: zip-ui

README.md

@@ -5,7 +5,6 @@
 [![GitHub Sponsors](https://img.shields.io/github/sponsors/stashapp?logo=github)](https://github.com/sponsors/stashapp)
 [![Open Collective backers](https://img.shields.io/opencollective/backers/stashapp?logo=opencollective)](https://opencollective.com/stashapp)
 [![Go Report Card](https://goreportcard.com/badge/github.com/stashapp/stash)](https://goreportcard.com/report/github.com/stashapp/stash)
-[![Matrix](https://img.shields.io/matrix/stashapp:unredacted.org?logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#stashapp:unredacted.org)
 [![Discord](https://img.shields.io/discord/559159668438728723.svg?logo=discord)](https://discord.gg/2TsNFKt)
 [![GitHub release (latest by date)](https://img.shields.io/github/v/release/stashapp/stash?logo=github)](https://github.com/stashapp/stash/releases/latest)
 [![GitHub issues by-label](https://img.shields.io/github/issues-raw/stashapp/stash/bounty)](https://github.com/stashapp/stash/labels/bounty)
@@ -68,19 +67,24 @@ Stash is available in 32 languages (so far!) and it could be in your language to
 
 [![Translation status](https://translate.codeberg.org/widget/stash/stash/multi-auto.svg)](https://translate.codeberg.org/engage/stash/)
 
+## Join Our Community
+
+We are excited to announce that we have a new home for support, feature requests, and discussions related to Stash and its associated projects. Join our community on the [Discourse forum](https://discourse.stashapp.cc) to connect with other users, share your ideas, and get help from fellow enthusiasts.
+
 # Support (FAQ)
 
 Check out our documentation on [Stash-Docs](https://docs.stashapp.cc) for information about the software, questions, guides, add-ons and more. 
 
 For more help you can:
 * Check the in-app documentation, in the top right corner of the app (it's also mirrored on [Stash-Docs](https://docs.stashapp.cc/in-app-manual))
-* Join the [Matrix space](https://matrix.to/#/#stashapp:unredacted.org)
-* Join the [Discord server](https://discord.gg/2TsNFKt), where the community can offer support.
+* Join our [community forum](https://discourse.stashapp.cc)
+* Join the [Discord server](https://discord.gg/2TsNFKt)
 * Start a [discussion on GitHub](https://github.com/stashapp/stash/discussions)
 
 # Customization
 
 ## Themes and CSS Customization
+
 There is a [directory of community-created themes](https://docs.stashapp.cc/themes/list) on Stash-Docs.
 
 You can also change the Stash interface to fit your desired style with various snippets from [Custom CSS snippets](https://docs.stashapp.cc/themes/custom-css-snippets).

cmd/stash/main.go

@@ -152,6 +152,9 @@ func recoverPanic() {
 func exitError(err error) {
 	exitCode = 1
 	logger.Error(err)
+	// #5784 - log to stdout as well as the logger
+	// this does mean that it will log twice if the logger is set to stdout
+	fmt.Println(err)
 	if desktop.IsDesktop() {
 		desktop.FatalError(err)
 	}

docker/build/x86_64/Dockerfile

@@ -1,7 +1,7 @@
 # This dockerfile should be built with `make docker-build` from the stash root.
 
 # Build Frontend
-FROM node:alpine as frontend
+FROM node:20-alpine AS frontend
 RUN apk add --no-cache make git
 ## cache node_modules separately
 COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/
@@ -13,19 +13,22 @@ RUN make pre-ui
 RUN make generate-ui
 ARG GITHASH
 ARG STASH_VERSION
-RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui
+RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui-only
 
 # Build Backend
-FROM golang:1.22-alpine as backend
+FROM golang:1.22.8-alpine AS backend
 RUN apk add --no-cache make alpine-sdk
 WORKDIR /stash
 COPY ./go* ./*.go Makefile gqlgen.yml .gqlgenc.yml /stash/
+COPY ./graphql /stash/graphql/
 COPY ./scripts /stash/scripts/
 COPY ./pkg /stash/pkg/
-COPY ./cmd /stash/cmd
-COPY ./internal /stash/internal
+COPY ./cmd /stash/cmd/
+COPY ./internal /stash/internal/
+# needed for generate-login-locale
+COPY ./ui /stash/ui/
+RUN make generate-backend generate-login-locale
 COPY --from=frontend /stash /stash/
-RUN make generate-backend
 ARG GITHASH
 ARG STASH_VERSION
 RUN make flags-release flags-pie stash

docker/build/x86_64/Dockerfile-CUDA

@@ -1,7 +1,8 @@
 # This dockerfile should be built with `make docker-cuda-build` from the stash root.
+ARG CUDA_VERSION=12.8.0
 
 # Build Frontend
-FROM node:alpine as frontend
+FROM node:20-alpine AS frontend
 RUN apk add --no-cache make git
 ## cache node_modules separately
 COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/
@@ -13,37 +14,47 @@ RUN make pre-ui
 RUN make generate-ui
 ARG GITHASH
 ARG STASH_VERSION
-RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui
+RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui-only
 
 # Build Backend
-FROM golang:1.22-bullseye as backend
+FROM golang:1.22.8-bullseye AS backend
 RUN apt update && apt install -y build-essential golang
 WORKDIR /stash
 COPY ./go* ./*.go Makefile gqlgen.yml .gqlgenc.yml /stash/
+COPY ./graphql /stash/graphql/
 COPY ./scripts /stash/scripts/
 COPY ./pkg /stash/pkg/
 COPY ./cmd /stash/cmd
 COPY ./internal /stash/internal
+# needed for generate-login-locale
+COPY ./ui /stash/ui/
+RUN make generate-backend generate-login-locale
 COPY --from=frontend /stash /stash/
-RUN make generate-backend
 ARG GITHASH
 ARG STASH_VERSION
 RUN make flags-release flags-pie stash
 
 # Final Runnable Image
-FROM nvidia/cuda:12.0.1-base-ubuntu22.04
-RUN apt update && apt upgrade -y && apt install -y ca-certificates libvips-tools ffmpeg wget intel-media-va-driver-non-free vainfo
-RUN rm -rf /var/lib/apt/lists/*
-COPY --from=backend /stash/stash /usr/bin/
+FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu24.04
+RUN apt update && apt upgrade -y && apt install -y \
+    # stash dependencies
+    ca-certificates libvips-tools ffmpeg \
+    # intel dependencies
+    intel-media-va-driver-non-free vainfo \
+    # python tools
+    python3 python3-pip && \
+  # cleanup
+  apt autoremove -y && apt clean && \
+  rm -rf /var/lib/apt/lists/*
+COPY --from=backend --chmod=555 /stash/stash /usr/bin/
 
 # NVENC Patch
 RUN mkdir -p /usr/local/bin /patched-lib
-RUN wget https://raw.githubusercontent.com/keylase/nvidia-patch/master/patch.sh -O /usr/local/bin/patch.sh
-RUN wget https://raw.githubusercontent.com/keylase/nvidia-patch/master/docker-entrypoint.sh -O /usr/local/bin/docker-entrypoint.sh
-RUN chmod +x /usr/local/bin/patch.sh /usr/local/bin/docker-entrypoint.sh /usr/bin/stash
+ADD --chmod=555 https://raw.githubusercontent.com/keylase/nvidia-patch/master/patch.sh /usr/local/bin/patch.sh
+ADD --chmod=555 https://raw.githubusercontent.com/keylase/nvidia-patch/master/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
-ENV LANG C.UTF-8
-ENV NVIDIA_VISIBLE_DEVICES all
+ENV LANG=C.UTF-8
+ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=video,utility
 ENV STASH_CONFIG_FILE=/root/.stash/config.yml
 EXPOSE 9999

docker/build/x86_64/README.md

@@ -4,7 +4,7 @@ This dockerfile is used to build a stash docker container using the current sour
 
 # Building the docker container
 
-From the top-level directory (should contain `main.go` file):
+From the top-level directory (should contain `tools.go` file):
 
 ```
 make docker-build

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/go-chi/cors v1.2.1
 	github.com/go-chi/httplog v0.3.1
 	github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4
-	github.com/golang-jwt/jwt/v4 v4.5.1
+	github.com/golang-jwt/jwt/v4 v4.5.2
 	github.com/golang-migrate/migrate/v4 v4.16.2
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/securecookie v1.1.1
@@ -57,6 +57,7 @@ require (
 	golang.org/x/sys v0.28.0
 	golang.org/x/term v0.27.0
 	golang.org/x/text v0.21.0
+	golang.org/x/time v0.10.0
 	gopkg.in/guregu/null.v4 v4.0.0
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -246,8 +246,8 @@ github.com/gobwas/ws v1.3.0/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/K
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
-github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
+github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA=
 github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
@@ -954,6 +954,8 @@ golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
+golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

gqlgen.yml

@@ -17,7 +17,7 @@ autobind:
   - github.com/stashapp/stash/pkg/scraper
   - github.com/stashapp/stash/internal/identify
   - github.com/stashapp/stash/internal/dlna
-  - github.com/stashapp/stash/pkg/scraper/stashbox
+  - github.com/stashapp/stash/pkg/stashbox
 
 models:
   # Scalars

graphql/schema/types/stash-box.graphql

@@ -2,12 +2,15 @@ type StashBox {
   endpoint: String!
   api_key: String!
   name: String!
+  max_requests_per_minute: Int!
 }
 
 input StashBoxInput {
   endpoint: String!
   api_key: String!
   name: String!
+  # defaults to 240
+  max_requests_per_minute: Int
 }
 
 type StashID {

internal/api/resolver.go

@@ -13,7 +13,6 @@ import (
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/plugin/hook"
 	"github.com/stashapp/stash/pkg/scraper"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
 )
 
 var (
@@ -138,10 +137,6 @@ func (r *Resolver) withReadTxn(ctx context.Context, fn func(ctx context.Context)
 	return r.repository.WithReadTxn(ctx, fn)
 }
 
-func (r *Resolver) stashboxRepository() stashbox.Repository {
-	return stashbox.NewRepository(r.repository)
-}
-
 func (r *queryResolver) MarkerWall(ctx context.Context, q *string) (ret []*models.SceneMarker, err error) {
 	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
 		ret, err = r.repository.SceneMarker.Wall(ctx, q)

internal/api/resolver_mutation_scene.go

@@ -694,6 +694,13 @@ func validateSceneMarkerEndSeconds(seconds, endSeconds float64) error {
 	return nil
 }
 
+func float64OrZero(f *float64) float64 {
+	if f == nil {
+		return 0
+	}
+	return *f
+}
+
 func (r *mutationResolver) SceneMarkerUpdate(ctx context.Context, input SceneMarkerUpdateInput) (*models.SceneMarker, error) {
 	markerID, err := strconv.Atoi(input.ID)
 	if err != nil {
@@ -784,7 +791,7 @@ func (r *mutationResolver) SceneMarkerUpdate(ctx context.Context, input SceneMar
 		}
 
 		// remove the marker preview if the scene changed or if the timestamp was changed
-		if existingMarker.SceneID != newMarker.SceneID || existingMarker.Seconds != newMarker.Seconds || existingMarker.EndSeconds != newMarker.EndSeconds {
+		if existingMarker.SceneID != newMarker.SceneID || existingMarker.Seconds != newMarker.Seconds || float64OrZero(existingMarker.EndSeconds) != float64OrZero(newMarker.EndSeconds) {
 			seconds := int(existingMarker.Seconds)
 			if err := fileDeleter.MarkMarkerFiles(existingScene, seconds); err != nil {
 				return err

internal/api/resolver_mutation_stash_box.go

@@ -7,6 +7,10 @@ import (
 
 	"github.com/stashapp/stash/internal/manager"
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/scene"
+	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
+	"github.com/stashapp/stash/pkg/stashbox"
 )
 
 func (r *mutationResolver) SubmitStashBoxFingerprints(ctx context.Context, input StashBoxFingerprintSubmissionInput) (bool, error) {
@@ -15,8 +19,23 @@ func (r *mutationResolver) SubmitStashBoxFingerprints(ctx context.Context, input
 		return false, err
 	}
 
+	ids, err := stringslice.StringSliceToIntSlice(input.SceneIds)
+	if err != nil {
+		return false, err
+	}
+
 	client := r.newStashBoxClient(*b)
-	return client.SubmitStashBoxFingerprints(ctx, input.SceneIds)
+
+	var scenes []*models.Scene
+
+	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+		scenes, err = r.sceneService.FindByIDs(ctx, ids, scene.LoadStashIDs, scene.LoadFiles)
+		return err
+	}); err != nil {
+		return false, err
+	}
+
+	return client.SubmitFingerprints(ctx, scenes)
 }
 
 func (r *mutationResolver) StashBoxBatchPerformerTag(ctx context.Context, input manager.StashBoxBatchTagInput) (string, error) {
@@ -69,17 +88,76 @@ func (r *mutationResolver) SubmitStashBoxSceneDraft(ctx context.Context, input S
 			logger.Errorf("Error getting scene cover: %v", err)
 		}
 
-		if err := scene.LoadURLs(ctx, r.repository.Scene); err != nil {
-			return fmt.Errorf("loading scene URLs: %w", err)
+		draft, err := r.makeSceneDraft(ctx, scene, cover)
+		if err != nil {
+			return err
 		}
 
-		res, err = client.SubmitSceneDraft(ctx, scene, cover)
+		res, err = client.SubmitSceneDraft(ctx, *draft)
 		return err
 	})
 
 	return res, err
 }
 
+func (r *mutationResolver) makeSceneDraft(ctx context.Context, s *models.Scene, cover []byte) (*stashbox.SceneDraft, error) {
+	if err := s.LoadURLs(ctx, r.repository.Scene); err != nil {
+		return nil, fmt.Errorf("loading scene URLs: %w", err)
+	}
+
+	if err := s.LoadStashIDs(ctx, r.repository.Scene); err != nil {
+		return nil, err
+	}
+
+	draft := &stashbox.SceneDraft{
+		Scene: s,
+	}
+
+	pqb := r.repository.Performer
+	sqb := r.repository.Studio
+
+	if s.StudioID != nil {
+		var err error
+		draft.Studio, err = sqb.Find(ctx, *s.StudioID)
+		if err != nil {
+			return nil, err
+		}
+		if draft.Studio == nil {
+			return nil, fmt.Errorf("studio with id %d not found", *s.StudioID)
+		}
+
+		if err := draft.Studio.LoadStashIDs(ctx, r.repository.Studio); err != nil {
+			return nil, err
+		}
+	}
+
+	// submit all file fingerprints
+	if err := s.LoadFiles(ctx, r.repository.Scene); err != nil {
+		return nil, err
+	}
+
+	scenePerformers, err := pqb.FindBySceneID(ctx, s.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, p := range scenePerformers {
+		if err := p.LoadStashIDs(ctx, pqb); err != nil {
+			return nil, err
+		}
+	}
+	draft.Performers = scenePerformers
+
+	draft.Tags, err = r.repository.Tag.FindBySceneID(ctx, s.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	draft.Cover = cover
+
+	return draft, nil
+}
+
 func (r *mutationResolver) SubmitStashBoxPerformerDraft(ctx context.Context, input StashBoxDraftSubmissionInput) (*string, error) {
 	b, err := resolveStashBox(input.StashBoxIndex, input.StashBoxEndpoint)
 	if err != nil {
@@ -105,7 +183,22 @@ func (r *mutationResolver) SubmitStashBoxPerformerDraft(ctx context.Context, inp
 			return fmt.Errorf("performer with id %d not found", id)
 		}
 
-		res, err = client.SubmitPerformerDraft(ctx, performer)
+		pqb := r.repository.Performer
+		if err := performer.LoadAliases(ctx, pqb); err != nil {
+			return err
+		}
+
+		if err := performer.LoadURLs(ctx, pqb); err != nil {
+			return err
+		}
+
+		if err := performer.LoadStashIDs(ctx, pqb); err != nil {
+			return err
+		}
+
+		img, _ := pqb.GetImage(ctx, performer.ID)
+
+		res, err = client.SubmitPerformerDraft(ctx, performer, img)
 		return err
 	})
 

internal/api/resolver_query_find_tag.go

@@ -62,7 +62,11 @@ func (r *queryResolver) FindTags(ctx context.Context, tagFilter *models.TagFilte
 func (r *queryResolver) AllTags(ctx context.Context) (ret []*models.Tag, err error) {
 	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
 		ret, err = r.repository.Tag.All(ctx)
-		return err
+		if err != nil {
+			return err
+		}
+
+		return nil
 	}); err != nil {
 		return nil, err
 	}

internal/api/resolver_query_scraper.go

@@ -4,11 +4,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"strconv"
 
+	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scraper"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
+	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
 )
 
@@ -29,7 +31,7 @@ func (r *queryResolver) ScrapePerformerURL(ctx context.Context, url string) (*mo
 	return marshalScrapedPerformer(content)
 }
 
-func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string, query string) ([]*scraper.ScrapedScene, error) {
+func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string, query string) ([]*models.ScrapedScene, error) {
 	if query == "" {
 		return nil, nil
 	}
@@ -47,7 +49,7 @@ func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string,
 	return ret, nil
 }
 
-func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*scraper.ScrapedScene, error) {
+func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models.ScrapedScene, error) {
 	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeScene)
 	if err != nil {
 		return nil, err
@@ -61,7 +63,7 @@ func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*scrape
 	return ret, nil
 }
 
-func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*scraper.ScrapedGallery, error) {
+func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*models.ScrapedGallery, error) {
 	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeGallery)
 	if err != nil {
 		return nil, err
@@ -75,7 +77,7 @@ func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*scra
 	return ret, nil
 }
 
-func (r *queryResolver) ScrapeImageURL(ctx context.Context, url string) (*scraper.ScrapedImage, error) {
+func (r *queryResolver) ScrapeImageURL(ctx context.Context, url string) (*models.ScrapedImage, error) {
 	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeImage)
 	if err != nil {
 		return nil, err
@@ -99,12 +101,12 @@ func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models
 }
 
 func (r *queryResolver) ScrapeGroupURL(ctx context.Context, url string) (*models.ScrapedGroup, error) {
-	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeMovie)
+	content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeGroup)
 	if err != nil {
 		return nil, err
 	}
 
-	ret, err := marshalScrapedMovie(content)
+	ret, err := marshalScrapedGroup(content)
 	if err != nil {
 		return nil, err
 	}
@@ -129,8 +131,8 @@ func (r *queryResolver) ScrapeGroupURL(ctx context.Context, url string) (*models
 	return group, nil
 }
 
-func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.Source, input ScrapeSingleSceneInput) ([]*scraper.ScrapedScene, error) {
-	var ret []*scraper.ScrapedScene
+func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.Source, input ScrapeSingleSceneInput) ([]*models.ScrapedScene, error) {
+	var ret []*models.ScrapedScene
 
 	var sceneID int
 	if input.SceneID != nil {
@@ -182,9 +184,14 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.So
 
 		switch {
 		case input.SceneID != nil:
-			ret, err = client.FindStashBoxSceneByFingerprints(ctx, sceneID)
+			var fps []models.Fingerprints
+			fps, err = r.getScenesFingerprints(ctx, []int{sceneID})
+			if err != nil {
+				return nil, err
+			}
+			ret, err = client.FindSceneByFingerprints(ctx, fps[0])
 		case input.Query != nil:
-			ret, err = client.QueryStashBoxScene(ctx, *input.Query)
+			ret, err = client.QueryScene(ctx, *input.Query)
 		default:
 			return nil, fmt.Errorf("%w: scene_id or query must be set", ErrInput)
 		}
@@ -192,14 +199,23 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.So
 		if err != nil {
 			return nil, err
 		}
+
+		// TODO - this should happen after any scene is scraped
+		if err := r.matchScenesRelationships(ctx, ret, *source.StashBoxEndpoint); err != nil {
+			return nil, err
+		}
 	default:
 		return nil, fmt.Errorf("%w: scraper_id or stash_box_index must be set", ErrInput)
 	}
 
+	for i := range ret {
+		slices.SortFunc(ret[i].Tags, models.ScrapedTagSortFunction)
+	}
+
 	return ret, nil
 }
 
-func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source scraper.Source, input ScrapeMultiScenesInput) ([][]*scraper.ScrapedScene, error) {
+func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source scraper.Source, input ScrapeMultiScenesInput) ([][]*models.ScrapedScene, error) {
 	if source.ScraperID != nil {
 		return nil, ErrNotImplemented
 	} else if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
@@ -215,12 +231,89 @@ func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source scraper.So
 			return nil, err
 		}
 
-		return client.FindStashBoxScenesByFingerprints(ctx, sceneIDs)
+		fps, err := r.getScenesFingerprints(ctx, sceneIDs)
+		if err != nil {
+			return nil, err
+		}
+
+		ret, err := client.FindScenesByFingerprints(ctx, fps)
+		if err != nil {
+			return nil, err
+		}
+
+		// match relationships - this mutates the existing scenes so we can
+		// just flatten the slice and pass it in
+		flat := sliceutil.Flatten(ret)
+
+		if err := r.matchScenesRelationships(ctx, flat, *source.StashBoxEndpoint); err != nil {
+			return nil, err
+		}
+
+		return ret, nil
 	}
 
 	return nil, errors.New("scraper_id or stash_box_index must be set")
 }
 
+func (r *queryResolver) getScenesFingerprints(ctx context.Context, ids []int) ([]models.Fingerprints, error) {
+	fingerprints := make([]models.Fingerprints, len(ids))
+
+	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+		qb := r.repository.Scene
+
+		for i, sceneID := range ids {
+			scene, err := qb.Find(ctx, sceneID)
+			if err != nil {
+				return err
+			}
+
+			if scene == nil {
+				return fmt.Errorf("scene with id %d not found", sceneID)
+			}
+
+			if err := scene.LoadFiles(ctx, qb); err != nil {
+				return err
+			}
+
+			var sceneFPs models.Fingerprints
+
+			for _, f := range scene.Files.List() {
+				sceneFPs = append(sceneFPs, f.Fingerprints...)
+			}
+
+			fingerprints[i] = sceneFPs
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return fingerprints, nil
+}
+
+// matchSceneRelationships accepts scraped scenes and attempts to match its relationships to existing stash models.
+func (r *queryResolver) matchScenesRelationships(ctx context.Context, ss []*models.ScrapedScene, endpoint string) error {
+	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+		matcher := match.SceneRelationships{
+			PerformerFinder: r.repository.Performer,
+			TagFinder:       r.repository.Tag,
+			StudioFinder:    r.repository.Studio,
+		}
+
+		for _, s := range ss {
+			if err := matcher.MatchRelationships(ctx, s, endpoint); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (r *queryResolver) ScrapeSingleStudio(ctx context.Context, source scraper.Source, input ScrapeSingleStudioInput) ([]*models.ScrapedStudio, error) {
 	if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
 		b, err := resolveStashBox(source.StashBoxIndex, source.StashBoxEndpoint)
@@ -231,7 +324,7 @@ func (r *queryResolver) ScrapeSingleStudio(ctx context.Context, source scraper.S
 		client := r.newStashBoxClient(*b)
 
 		var ret []*models.ScrapedStudio
-		out, err := client.FindStashBoxStudio(ctx, *input.Query)
+		out, err := client.FindStudio(ctx, *input.Query)
 
 		if err != nil {
 			return nil, err
@@ -240,6 +333,17 @@ func (r *queryResolver) ScrapeSingleStudio(ctx context.Context, source scraper.S
 		}
 
 		if len(ret) > 0 {
+			if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+				for _, studio := range ret {
+					if err := match.ScrapedStudioHierarchy(ctx, r.repository.Studio, studio, *source.StashBoxEndpoint); err != nil {
+						return err
+					}
+				}
+
+				return nil
+			}); err != nil {
+				return nil, err
+			}
 			return ret, nil
 		}
 
@@ -285,23 +389,29 @@ func (r *queryResolver) ScrapeSinglePerformer(ctx context.Context, source scrape
 
 		client := r.newStashBoxClient(*b)
 
-		var res []*stashbox.StashBoxPerformerQueryResult
+		var query string
 		switch {
 		case input.PerformerID != nil:
-			res, err = client.FindStashBoxPerformersByNames(ctx, []string{*input.PerformerID})
+			names, err := r.findPerformerNames(ctx, []string{*input.PerformerID})
+			if err != nil {
+				return nil, err
+			}
+
+			query = names[0]
 		case input.Query != nil:
-			res, err = client.QueryStashBoxPerformer(ctx, *input.Query)
+			query = *input.Query
 		default:
 			return nil, ErrNotImplemented
 		}
 
+		if query == "" {
+			return nil, nil
+		}
+		ret, err = client.QueryPerformer(ctx, query)
+
 		if err != nil {
 			return nil, err
 		}
-
-		if len(res) > 0 {
-			ret = res[0].Results
-		}
 	default:
 		return nil, errors.New("scraper_id or stash_box_index must be set")
 	}
@@ -313,6 +423,11 @@ func (r *queryResolver) ScrapeMultiPerformers(ctx context.Context, source scrape
 	if source.ScraperID != nil {
 		return nil, ErrNotImplemented
 	} else if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
+		names, err := r.findPerformerNames(ctx, input.PerformerIds)
+		if err != nil {
+			return nil, err
+		}
+
 		b, err := resolveStashBox(source.StashBoxIndex, source.StashBoxEndpoint)
 		if err != nil {
 			return nil, err
@@ -320,14 +435,40 @@ func (r *queryResolver) ScrapeMultiPerformers(ctx context.Context, source scrape
 
 		client := r.newStashBoxClient(*b)
 
-		return client.FindStashBoxPerformersByPerformerNames(ctx, input.PerformerIds)
+		return client.QueryPerformers(ctx, names)
 	}
 
 	return nil, errors.New("scraper_id or stash_box_index must be set")
 }
 
-func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.Source, input ScrapeSingleGalleryInput) ([]*scraper.ScrapedGallery, error) {
-	var ret []*scraper.ScrapedGallery
+func (r *queryResolver) findPerformerNames(ctx context.Context, performerIDs []string) ([]string, error) {
+	ids, err := stringslice.StringSliceToIntSlice(performerIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	names := make([]string, len(ids))
+
+	if err := r.withReadTxn(ctx, func(ctx context.Context) error {
+		p, err := r.repository.Performer.FindMany(ctx, ids)
+		if err != nil {
+			return err
+		}
+
+		for i, pp := range p {
+			names[i] = pp.Name
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return names, nil
+}
+
+func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.Source, input ScrapeSingleGalleryInput) ([]*models.ScrapedGallery, error) {
+	var ret []*models.ScrapedGallery
 
 	if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
 		return nil, ErrNotSupported
@@ -369,7 +510,7 @@ func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.
 	return ret, nil
 }
 
-func (r *queryResolver) ScrapeSingleImage(ctx context.Context, source scraper.Source, input ScrapeSingleImageInput) ([]*scraper.ScrapedImage, error) {
+func (r *queryResolver) ScrapeSingleImage(ctx context.Context, source scraper.Source, input ScrapeSingleImageInput) ([]*models.ScrapedImage, error) {
 	if source.StashBoxIndex != nil {
 		return nil, ErrNotSupported
 	}

internal/api/scraped_content.go

@@ -9,8 +9,8 @@ import (
 
 // marshalScrapedScenes converts ScrapedContent into ScrapedScene. If conversion fails, an
 // error is returned to the caller.
-func marshalScrapedScenes(content []scraper.ScrapedContent) ([]*scraper.ScrapedScene, error) {
-	var ret []*scraper.ScrapedScene
+func marshalScrapedScenes(content []scraper.ScrapedContent) ([]*models.ScrapedScene, error) {
+	var ret []*models.ScrapedScene
 	for _, c := range content {
 		if c == nil {
 			// graphql schema requires scenes to be non-nil
@@ -18,9 +18,9 @@ func marshalScrapedScenes(content []scraper.ScrapedContent) ([]*scraper.ScrapedS
 		}
 
 		switch s := c.(type) {
-		case *scraper.ScrapedScene:
+		case *models.ScrapedScene:
 			ret = append(ret, s)
-		case scraper.ScrapedScene:
+		case models.ScrapedScene:
 			ret = append(ret, &s)
 		default:
 			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedScene", models.ErrConversion)
@@ -55,8 +55,8 @@ func marshalScrapedPerformers(content []scraper.ScrapedContent) ([]*models.Scrap
 
 // marshalScrapedGalleries converts ScrapedContent into ScrapedGallery. If
 // conversion fails, an error is returned.
-func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*scraper.ScrapedGallery, error) {
-	var ret []*scraper.ScrapedGallery
+func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*models.ScrapedGallery, error) {
+	var ret []*models.ScrapedGallery
 	for _, c := range content {
 		if c == nil {
 			// graphql schema requires galleries to be non-nil
@@ -64,9 +64,9 @@ func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*scraper.Scrap
 		}
 
 		switch g := c.(type) {
-		case *scraper.ScrapedGallery:
+		case *models.ScrapedGallery:
 			ret = append(ret, g)
-		case scraper.ScrapedGallery:
+		case models.ScrapedGallery:
 			ret = append(ret, &g)
 		default:
 			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedGallery", models.ErrConversion)
@@ -76,8 +76,8 @@ func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*scraper.Scrap
 	return ret, nil
 }
 
-func marshalScrapedImages(content []scraper.ScrapedContent) ([]*scraper.ScrapedImage, error) {
-	var ret []*scraper.ScrapedImage
+func marshalScrapedImages(content []scraper.ScrapedContent) ([]*models.ScrapedImage, error) {
+	var ret []*models.ScrapedImage
 	for _, c := range content {
 		if c == nil {
 			// graphql schema requires images to be non-nil
@@ -85,9 +85,9 @@ func marshalScrapedImages(content []scraper.ScrapedContent) ([]*scraper.ScrapedI
 		}
 
 		switch g := c.(type) {
-		case *scraper.ScrapedImage:
+		case *models.ScrapedImage:
 			ret = append(ret, g)
-		case scraper.ScrapedImage:
+		case models.ScrapedImage:
 			ret = append(ret, &g)
 		default:
 			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedImage", models.ErrConversion)
@@ -113,7 +113,30 @@ func marshalScrapedMovies(content []scraper.ScrapedContent) ([]*models.ScrapedMo
 		case models.ScrapedMovie:
 			ret = append(ret, &m)
 		default:
-			return nil, fmt.Errorf("%w: cannot turn ScrapedConetnt into ScrapedMovie", models.ErrConversion)
+			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedMovie", models.ErrConversion)
+		}
+	}
+
+	return ret, nil
+}
+
+// marshalScrapedMovies converts ScrapedContent into ScrapedMovie. If conversion
+// fails, an error is returned.
+func marshalScrapedGroups(content []scraper.ScrapedContent) ([]*models.ScrapedGroup, error) {
+	var ret []*models.ScrapedGroup
+	for _, c := range content {
+		if c == nil {
+			// graphql schema requires groups to be non-nil
+			continue
+		}
+
+		switch m := c.(type) {
+		case *models.ScrapedGroup:
+			ret = append(ret, m)
+		case models.ScrapedGroup:
+			ret = append(ret, &m)
+		default:
+			return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedGroup", models.ErrConversion)
 		}
 	}
 
@@ -131,7 +154,7 @@ func marshalScrapedPerformer(content scraper.ScrapedContent) (*models.ScrapedPer
 }
 
 // marshalScrapedScene will marshal a single scraped scene
-func marshalScrapedScene(content scraper.ScrapedContent) (*scraper.ScrapedScene, error) {
+func marshalScrapedScene(content scraper.ScrapedContent) (*models.ScrapedScene, error) {
 	s, err := marshalScrapedScenes([]scraper.ScrapedContent{content})
 	if err != nil {
 		return nil, err
@@ -141,7 +164,7 @@ func marshalScrapedScene(content scraper.ScrapedContent) (*scraper.ScrapedScene,
 }
 
 // marshalScrapedGallery will marshal a single scraped gallery
-func marshalScrapedGallery(content scraper.ScrapedContent) (*scraper.ScrapedGallery, error) {
+func marshalScrapedGallery(content scraper.ScrapedContent) (*models.ScrapedGallery, error) {
 	g, err := marshalScrapedGalleries([]scraper.ScrapedContent{content})
 	if err != nil {
 		return nil, err
@@ -151,7 +174,7 @@ func marshalScrapedGallery(content scraper.ScrapedContent) (*scraper.ScrapedGall
 }
 
 // marshalScrapedImage will marshal a single scraped image
-func marshalScrapedImage(content scraper.ScrapedContent) (*scraper.ScrapedImage, error) {
+func marshalScrapedImage(content scraper.ScrapedContent) (*models.ScrapedImage, error) {
 	g, err := marshalScrapedImages([]scraper.ScrapedContent{content})
 	if err != nil {
 		return nil, err
@@ -169,3 +192,13 @@ func marshalScrapedMovie(content scraper.ScrapedContent) (*models.ScrapedMovie,
 
 	return m[0], nil
 }
+
+// marshalScrapedMovie will marshal a single scraped movie
+func marshalScrapedGroup(content scraper.ScrapedContent) (*models.ScrapedGroup, error) {
+	m, err := marshalScrapedGroups([]scraper.ScrapedContent{content})
+	if err != nil {
+		return nil, err
+	}
+
+	return m[0], nil
+}

internal/api/server.go

@@ -41,10 +41,11 @@ import (
 )
 
 const (
-	loginEndpoint      = "/login"
-	logoutEndpoint     = "/logout"
-	gqlEndpoint        = "/graphql"
-	playgroundEndpoint = "/playground"
+	loginEndpoint       = "/login"
+	loginLocaleEndpoint = loginEndpoint + "/locale"
+	logoutEndpoint      = "/logout"
+	gqlEndpoint         = "/graphql"
+	playgroundEndpoint  = "/playground"
 )
 
 type Server struct {
@@ -228,6 +229,7 @@ func Initialize() (*Server, error) {
 	r.Get(loginEndpoint, handleLogin())
 	r.Post(loginEndpoint, handleLoginPost())
 	r.Get(logoutEndpoint, handleLogout())
+	r.Get(loginLocaleEndpoint, handleLoginLocale(cfg))
 	r.HandleFunc(loginEndpoint+"/*", func(w http.ResponseWriter, r *http.Request) {
 		r.URL.Path = strings.TrimPrefix(r.URL.Path, loginEndpoint)
 		w.Header().Set("Cache-Control", "no-cache")

internal/api/session.go

@@ -17,7 +17,11 @@ import (
 	"github.com/stashapp/stash/ui"
 )
 
-const returnURLParam = "returnURL"
+const (
+	returnURLParam = "returnURL"
+
+	defaultLocale = "en-GB"
+)
 
 func getLoginPage() []byte {
 	data, err := fs.ReadFile(ui.LoginUIBox, "login.html")
@@ -58,6 +62,47 @@ func serveLoginPage(w http.ResponseWriter, r *http.Request, returnURL string, lo
 	utils.ServeStaticContent(w, r, buffer.Bytes())
 }
 
+func handleLoginLocale(cfg *config.Config) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// get the locale from the config
+		lang := cfg.GetLanguage()
+		if lang == "" {
+			lang = defaultLocale
+		}
+
+		data, err := getLoginLocale(lang)
+		if err != nil {
+			logger.Debugf("Failed to load login locale file for language %s: %v", lang, err)
+			// try again with the default language
+			if lang != defaultLocale {
+				data, err = getLoginLocale(defaultLocale)
+				if err != nil {
+					logger.Errorf("Failed to load login locale file for default language %s: %v", defaultLocale, err)
+				}
+			}
+
+			// if there's still an error, response with an internal server error
+			if err != nil {
+				http.Error(w, "Failed to load login locale file", http.StatusInternalServerError)
+				return
+			}
+		}
+
+		// write a script to set the locale string map as a global variable
+		localeScript := fmt.Sprintf("var localeStrings = %s;", data)
+		w.Header().Set("Content-Type", "application/javascript")
+		_, _ = w.Write([]byte(localeScript))
+	}
+}
+
+func getLoginLocale(lang string) ([]byte, error) {
+	data, err := fs.ReadFile(ui.LoginUIBox, "locales/"+lang+".json")
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
 func handleLogin() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		returnURL := r.URL.Query().Get(returnURLParam)
@@ -78,31 +123,26 @@ func handleLogin() http.HandlerFunc {
 
 func handleLoginPost() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		url := r.FormValue(returnURLParam)
-		if url == "" {
-			url = getProxyPrefix(r) + "/"
-		}
-
 		err := manager.GetInstance().SessionStore.Login(w, r)
 		if err != nil {
 			// always log the error
-			logger.Errorf("Error logging in: %v", err)
+			logger.Errorf("Error logging in: %v from IP: %s", err, r.RemoteAddr)
 		}
 
 		var invalidCredentialsError *session.InvalidCredentialsError
 
 		if errors.As(err, &invalidCredentialsError) {
-			// serve login page with an error
-			serveLoginPage(w, r, url, "Username or password is invalid")
+			http.Error(w, "Username or password is invalid", http.StatusUnauthorized)
 			return
 		}
 
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+			// don't expose the error to the user
+			http.Error(w, "An unexpected error occurred. See logs", http.StatusInternalServerError)
 			return
 		}
 
-		http.Redirect(w, r, url, http.StatusFound)
+		w.WriteHeader(http.StatusOK)
 	}
 }
 

internal/api/stash_box.go

@@ -7,11 +7,11 @@ import (
 	"github.com/stashapp/stash/internal/manager"
 	"github.com/stashapp/stash/internal/manager/config"
 	"github.com/stashapp/stash/pkg/models"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
+	"github.com/stashapp/stash/pkg/stashbox"
 )
 
 func (r *Resolver) newStashBoxClient(box models.StashBox) *stashbox.Client {
-	return stashbox.NewClient(box, r.stashboxRepository(), manager.GetInstance().Config.GetScraperExcludeTagPatterns())
+	return stashbox.NewClient(box, stashbox.ExcludeTagPatterns(manager.GetInstance().Config.GetScraperExcludeTagPatterns()))
 }
 
 func resolveStashBoxFn(indexField, endpointField string) func(index *int, endpoint *string) (*models.StashBox, error) {

internal/identify/identify.go

@@ -13,7 +13,6 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
-	"github.com/stashapp/stash/pkg/scraper"
 	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/txn"
 	"github.com/stashapp/stash/pkg/utils"
@@ -32,7 +31,7 @@ func (e *MultipleMatchesFoundError) Error() string {
 }
 
 type SceneScraper interface {
-	ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error)
+	ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error)
 }
 
 type SceneUpdatePostHookExecutor interface {
@@ -96,7 +95,7 @@ func (t *SceneIdentifier) Identify(ctx context.Context, scene *models.Scene) err
 }
 
 type scrapeResult struct {
-	result *scraper.ScrapedScene
+	result *models.ScrapedScene
 	source ScraperSource
 }
 
@@ -374,7 +373,7 @@ func getFieldOptions(options []MetadataOptions) map[string]*FieldOptions {
 	return ret
 }
 
-func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOptions map[string]*FieldOptions, setOrganized bool) models.ScenePartial {
+func getScenePartial(scene *models.Scene, scraped *models.ScrapedScene, fieldOptions map[string]*FieldOptions, setOrganized bool) models.ScenePartial {
 	partial := models.ScenePartial{}
 
 	if scraped.Title != nil && (scene.Title != *scraped.Title) {

internal/identify/identify_test.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/mocks"
-	"github.com/stashapp/stash/pkg/scraper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 )
@@ -19,10 +18,10 @@ var testCtx = context.Background()
 
 type mockSceneScraper struct {
 	errIDs  []int
-	results map[int][]*scraper.ScrapedScene
+	results map[int][]*models.ScrapedScene
 }
 
-func (s mockSceneScraper) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
+func (s mockSceneScraper) ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
 	if slices.Contains(s.errIDs, sceneID) {
 		return nil, errors.New("scrape scene error")
 	}
@@ -70,7 +69,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
 		{
 			Scraper: mockSceneScraper{
 				errIDs: []int{errID1},
-				results: map[int][]*scraper.ScrapedScene{
+				results: map[int][]*models.ScrapedScene{
 					found1ID: {{
 						Title: &scrapedTitle,
 					}},
@@ -80,7 +79,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
 		{
 			Scraper: mockSceneScraper{
 				errIDs: []int{errID2},
-				results: map[int][]*scraper.ScrapedScene{
+				results: map[int][]*models.ScrapedScene{
 					found2ID: {{
 						Title: &scrapedTitle,
 					}},
@@ -250,7 +249,7 @@ func TestSceneIdentifier_modifyScene(t *testing.T) {
 					StashIDs:     models.NewRelatedStashIDs([]models.StashID{}),
 				},
 				&scrapeResult{
-					result: &scraper.ScrapedScene{},
+					result: &models.ScrapedScene{},
 					source: ScraperSource{
 						Options: defaultOptions,
 					},
@@ -386,14 +385,14 @@ func Test_getScenePartial(t *testing.T) {
 		Mode:   models.RelationshipUpdateModeSet,
 	}
 
-	scrapedScene := &scraper.ScrapedScene{
+	scrapedScene := &models.ScrapedScene{
 		Title:   &scrapedTitle,
 		Date:    &scrapedDate,
 		Details: &scrapedDetails,
 		URLs:    []string{scrapedURL},
 	}
 
-	scrapedUnchangedScene := &scraper.ScrapedScene{
+	scrapedUnchangedScene := &models.ScrapedScene{
 		Title:   &originalTitle,
 		Date:    &originalDate,
 		Details: &originalDetails,
@@ -423,7 +422,7 @@ func Test_getScenePartial(t *testing.T) {
 
 	type args struct {
 		scene        *models.Scene
-		scraped      *scraper.ScrapedScene
+		scraped      *models.ScrapedScene
 		fieldOptions map[string]*FieldOptions
 		setOrganized bool
 	}

internal/identify/scene_test.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/mocks"
-	"github.com/stashapp/stash/pkg/scraper"
 	"github.com/stashapp/stash/pkg/utils"
 	"github.com/stretchr/testify/mock"
 )
@@ -125,7 +124,7 @@ func Test_sceneRelationships_studio(t *testing.T) {
 				source: ScraperSource{
 					RemoteSite: "endpoint",
 				},
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					Studio: tt.result,
 				},
 			}
@@ -315,7 +314,7 @@ func Test_sceneRelationships_performers(t *testing.T) {
 			tr.scene = tt.scene
 			tr.fieldOptions["performers"] = tt.fieldOptions
 			tr.result = &scrapeResult{
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					Performers: tt.scraped,
 				},
 			}
@@ -507,7 +506,7 @@ func Test_sceneRelationships_tags(t *testing.T) {
 			tr.scene = tt.scene
 			tr.fieldOptions["tags"] = tt.fieldOptions
 			tr.result = &scrapeResult{
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					Tags: tt.scraped,
 				},
 			}
@@ -727,7 +726,7 @@ func Test_sceneRelationships_stashIDs(t *testing.T) {
 				source: ScraperSource{
 					RemoteSite: tt.endpoint,
 				},
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					RemoteSiteID: tt.remoteSiteID,
 				},
 			}
@@ -827,7 +826,7 @@ func Test_sceneRelationships_cover(t *testing.T) {
 				ID: tt.sceneID,
 			}
 			tr.result = &scrapeResult{
-				result: &scraper.ScrapedScene{
+				result: &models.ScrapedScene{
 					Image: tt.image,
 				},
 			}

internal/manager/config/config.go

@@ -1105,9 +1105,10 @@ func stashBoxValidate(str string) bool {
 }
 
 type StashBoxInput struct {
-	Endpoint string `json:"endpoint"`
-	APIKey   string `json:"api_key"`
-	Name     string `json:"name"`
+	Endpoint             string `json:"endpoint"`
+	APIKey               string `json:"api_key"`
+	Name                 string `json:"name"`
+	MaxRequestsPerMinute int    `json:"max_requests_per_minute"`
 }
 
 func (i *Config) ValidateStashBoxes(boxes []*StashBoxInput) error {

internal/manager/repository.go

@@ -14,6 +14,9 @@ type SceneService interface {
 	AssignFile(ctx context.Context, sceneID int, fileID models.FileID) error
 	Merge(ctx context.Context, sourceIDs []int, destinationID int, fileDeleter *scene.FileDeleter, options scene.MergeOptions) error
 	Destroy(ctx context.Context, scene *models.Scene, fileDeleter *scene.FileDeleter, deleteGenerated, deleteFile bool) error
+
+	FindByIDs(ctx context.Context, ids []int, load ...scene.LoadRelationshipOption) ([]*models.Scene, error)
+	sceneFingerprintGetter
 }
 
 type ImageService interface {

internal/manager/task_export.go

@@ -1042,23 +1042,43 @@ func (t *ExportTask) ExportTags(ctx context.Context, workers int) {
 	logger.Info("[tags] exporting")
 	startTime := time.Now()
 
-	jobCh := make(chan *models.Tag, workers*2) // make a buffered channel to feed workers
-
-	for w := 0; w < workers; w++ { // create export Tag workers
-		tagsWg.Add(1)
-		go t.exportTag(ctx, &tagsWg, jobCh)
+	tagIdx := 0
+	if t.tags != nil {
+		tagIdx = len(t.tags.IDs)
 	}
 
-	for i, tag := range tags {
-		index := i + 1
-		logger.Progressf("[tags] %d of %d", index, len(tags))
+	for {
+		jobCh := make(chan *models.Tag, workers*2) // make a buffered channel to feed workers
 
-		jobCh <- tag // feed workers
+		for w := 0; w < workers; w++ { // create export Tag workers
+			tagsWg.Add(1)
+			go t.exportTag(ctx, &tagsWg, jobCh)
+		}
+
+		for i, tag := range tags {
+			index := i + 1 + tagIdx
+			logger.Progressf("[tags] %d of %d", index, len(tags)+tagIdx)
+
+			jobCh <- tag // feed workers
+		}
+
+		close(jobCh)
+		tagsWg.Wait()
+
+		// if more tags were added, we need to export those too
+		if t.tags == nil || len(t.tags.IDs) == tagIdx {
+			break
+		}
+
+		newTags, err := reader.FindMany(ctx, t.tags.IDs[tagIdx:])
+		if err != nil {
+			logger.Errorf("[tags] failed to fetch tags: %v", err)
+		}
+
+		tags = newTags
+		tagIdx = len(t.tags.IDs)
 	}
 
-	close(jobCh)
-	tagsWg.Wait()
-
 	logger.Infof("[tags] export complete in %s. %d workers used.", time.Since(startTime), workers)
 }
 
@@ -1075,6 +1095,15 @@ func (t *ExportTask) exportTag(ctx context.Context, wg *sync.WaitGroup, jobChan
 			continue
 		}
 
+		if t.includeDependencies {
+			tagIDs, err := tag.GetDependentTagIDs(ctx, tagReader, thisTag)
+			if err != nil {
+				logger.Errorf("[tags] <%s> error getting dependent tags: %v", thisTag.Name, err)
+				continue
+			}
+			t.tags.IDs = sliceutil.AppendUniques(t.tags.IDs, tagIDs)
+		}
+
 		fn := newTagJSON.Filename()
 
 		if err := t.json.saveTag(fn, newTagJSON); err != nil {

internal/manager/task_identify.go

@@ -9,11 +9,13 @@ import (
 	"github.com/stashapp/stash/internal/identify"
 	"github.com/stashapp/stash/pkg/job"
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/scraper"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
+	"github.com/stashapp/stash/pkg/stashbox"
+	"github.com/stashapp/stash/pkg/txn"
 )
 
 var ErrInput = errors.New("invalid request input")
@@ -169,12 +171,20 @@ func (j *IdentifyJob) getSources() ([]identify.ScraperSource, error) {
 
 		var src identify.ScraperSource
 		if stashBox != nil {
-			stashboxRepository := stashbox.NewRepository(instance.Repository)
+			matcher := match.SceneRelationships{
+				PerformerFinder: instance.Repository.Performer,
+				TagFinder:       instance.Repository.Tag,
+				StudioFinder:    instance.Repository.Studio,
+			}
+
 			src = identify.ScraperSource{
 				Name: "stash-box: " + stashBox.Endpoint,
 				Scraper: stashboxSource{
-					stashbox.NewClient(*stashBox, stashboxRepository, instance.Config.GetScraperExcludeTagPatterns()),
-					stashBox.Endpoint,
+					Client:                 stashbox.NewClient(*stashBox, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns())),
+					endpoint:               stashBox.Endpoint,
+					txnManager:             instance.Repository.TxnManager,
+					sceneFingerprintGetter: instance.SceneService,
+					matcher:                matcher,
 				},
 				RemoteSite: stashBox.Endpoint,
 			}
@@ -247,14 +257,42 @@ func resolveStashBox(sb []*models.StashBox, source scraper.Source) (*models.Stas
 type stashboxSource struct {
 	*stashbox.Client
 	endpoint string
+
+	txnManager             models.TxnManager
+	sceneFingerprintGetter sceneFingerprintGetter
+	matcher                match.SceneRelationships
 }
 
-func (s stashboxSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
-	results, err := s.FindStashBoxSceneByFingerprints(ctx, sceneID)
+type sceneFingerprintGetter interface {
+	GetScenesFingerprints(ctx context.Context, ids []int) ([]models.Fingerprints, error)
+}
+
+func (s stashboxSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
+	var fps []models.Fingerprints
+	if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
+		var err error
+		fps, err = s.sceneFingerprintGetter.GetScenesFingerprints(ctx, []int{sceneID})
+		return err
+	}); err != nil {
+		return nil, fmt.Errorf("error getting scene fingerprints: %w", err)
+	}
+
+	results, err := s.FindSceneByFingerprints(ctx, fps[0])
 	if err != nil {
 		return nil, fmt.Errorf("error querying stash-box using scene ID %d: %w", sceneID, err)
 	}
 
+	if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
+		for _, ret := range results {
+			if err := s.matcher.MatchRelationships(ctx, ret, s.endpoint); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return nil, fmt.Errorf("error matching scene relationships: %w", err)
+	}
+
 	if len(results) > 0 {
 		return results, nil
 	}
@@ -271,7 +309,7 @@ type scraperSource struct {
 	scraperID string
 }
 
-func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
+func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
 	content, err := s.cache.ScrapeID(ctx, s.scraperID, sceneID, scraper.ScrapeContentTypeScene)
 	if err != nil {
 		return nil, err
@@ -282,8 +320,8 @@ func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scrape
 		return nil, nil
 	}
 
-	if scene, ok := content.(scraper.ScrapedScene); ok {
-		return []*scraper.ScrapedScene{&scene}, nil
+	if scene, ok := content.(models.ScrapedScene); ok {
+		return []*models.ScrapedScene{&scene}, nil
 	}
 
 	return nil, errors.New("could not convert content to scene")

internal/manager/task_import.go

@@ -709,6 +709,11 @@ func (t *ImportTask) ImportScenes(ctx context.Context) {
 				return err
 			}
 
+			// skip importing markers if the scene was not created
+			if sceneImporter.ID == 0 {
+				return nil
+			}
+
 			// import the scene markers
 			for _, m := range sceneJSON.Markers {
 				markerImporter := &scene.MarkerImporter{

internal/manager/task_stash_box_tag.go

@@ -6,10 +6,11 @@ import (
 	"strconv"
 
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/performer"
-	"github.com/stashapp/stash/pkg/scraper/stashbox"
 	"github.com/stashapp/stash/pkg/sliceutil"
+	"github.com/stashapp/stash/pkg/stashbox"
 	"github.com/stashapp/stash/pkg/studio"
 )
 
@@ -95,8 +96,7 @@ func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*mode
 
 	r := instance.Repository
 
-	stashboxRepository := stashbox.NewRepository(r)
-	client := stashbox.NewClient(*t.box, stashboxRepository, instance.Config.GetScraperExcludeTagPatterns())
+	client := stashbox.NewClient(*t.box, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns()))
 
 	if t.refresh {
 		var remoteID string
@@ -119,7 +119,7 @@ func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*mode
 			return nil, err
 		}
 		if remoteID != "" {
-			performer, err = client.FindStashBoxPerformerByID(ctx, remoteID)
+			performer, err = client.FindPerformerByID(ctx, remoteID)
 
 			if performer != nil && performer.RemoteMergedIntoId != nil {
 				mergedPerformer, err := t.handleMergedPerformer(ctx, performer, client)
@@ -140,14 +140,22 @@ func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*mode
 		} else {
 			name = t.performer.Name
 		}
-		performer, err = client.FindStashBoxPerformerByName(ctx, name)
+		performer, err = client.FindPerformerByName(ctx, name)
+	}
+
+	if performer != nil {
+		if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
+			return match.ScrapedPerformer(ctx, r.Performer, performer, t.box.Endpoint)
+		}); err != nil {
+			return nil, err
+		}
 	}
 
 	return performer, err
 }
 
 func (t *StashBoxBatchTagTask) handleMergedPerformer(ctx context.Context, performer *models.ScrapedPerformer, client *stashbox.Client) (mergedPerformer *models.ScrapedPerformer, err error) {
-	mergedPerformer, err = client.FindStashBoxPerformerByID(ctx, *performer.RemoteMergedIntoId)
+	mergedPerformer, err = client.FindPerformerByID(ctx, *performer.RemoteMergedIntoId)
 	if err != nil {
 		return nil, fmt.Errorf("loading merged performer %s from stashbox", *performer.RemoteMergedIntoId)
 	}
@@ -287,8 +295,7 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
 
 	r := instance.Repository
 
-	stashboxRepository := stashbox.NewRepository(r)
-	client := stashbox.NewClient(*t.box, stashboxRepository, instance.Config.GetScraperExcludeTagPatterns())
+	client := stashbox.NewClient(*t.box, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns()))
 
 	if t.refresh {
 		var remoteID string
@@ -309,7 +316,7 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
 			return nil, err
 		}
 		if remoteID != "" {
-			studio, err = client.FindStashBoxStudio(ctx, remoteID)
+			studio, err = client.FindStudio(ctx, remoteID)
 		}
 	} else {
 		var name string
@@ -318,7 +325,19 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
 		} else {
 			name = t.studio.Name
 		}
-		studio, err = client.FindStashBoxStudio(ctx, name)
+		studio, err = client.FindStudio(ctx, name)
+	}
+
+	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
+		if studio != nil {
+			if err := match.ScrapedStudioHierarchy(ctx, r.Studio, studio, t.box.Endpoint); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
 	}
 
 	return studio, err

pkg/ffmpeg/stream_segmented.go

@@ -426,9 +426,11 @@ func serveHLSManifest(sm *StreamManager, w http.ResponseWriter, r *http.Request,
 		return
 	}
 
+	prefix := r.Header.Get("X-Forwarded-Prefix")
+
 	baseUrl := *r.URL
 	baseUrl.RawQuery = ""
-	baseURL := baseUrl.String()
+	baseURL := prefix + baseUrl.String()
 
 	urlQuery := url.Values{}
 	apikey := r.URL.Query().Get(apiKeyParamKey)
@@ -559,9 +561,11 @@ func serveDASHManifest(sm *StreamManager, w http.ResponseWriter, r *http.Request
 	mediaDuration := mpd.Duration(time.Duration(probeResult.FileDuration * float64(time.Second)))
 	m := mpd.NewMPD(mpd.DASH_PROFILE_LIVE, mediaDuration.String(), "PT4.0S")
 
+	prefix := r.Header.Get("X-Forwarded-Prefix")
+
 	baseUrl := r.URL.JoinPath("/")
 	baseUrl.RawQuery = ""
-	m.BaseURL = baseUrl.String()
+	m.BaseURL = prefix + baseUrl.String()
 
 	video, _ := m.AddNewAdaptationSetVideo(MimeWebmVideo, "progressive", true, 1)
 

pkg/fsutil/file.go

@@ -1,6 +1,8 @@
 package fsutil
 
 import (
+	"crypto/sha1"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
@@ -151,7 +153,12 @@ var (
 )
 
 // SanitiseBasename returns a file basename removing any characters that are illegal or problematic to use in the filesystem.
+// It appends a short hash of the original string to ensure uniqueness.
 func SanitiseBasename(v string) string {
+	// Generate a short hash for uniqueness
+	hash := sha1.Sum([]byte(v))
+	shortHash := hex.EncodeToString(hash[:4]) // Use the first 4 bytes of the hash
+
 	v = strings.TrimSpace(v)
 
 	// replace illegal filename characters with -
@@ -163,7 +170,7 @@ func SanitiseBasename(v string) string {
 	// remove multiple hyphens
 	v = multiHyphenRE.ReplaceAllString(v, "-")
 
-	return strings.TrimSpace(v)
+	return strings.TrimSpace(v) + "-" + shortHash
 }
 
 // GetExeName returns the name of the given executable for the current platform.

pkg/fsutil/file_test.go

@@ -8,13 +8,13 @@ func TestSanitiseBasename(t *testing.T) {
 		v    string
 		want string
 	}{
-		{"basic", "basic", "basic"},
-		{"spaces", `spaced name`, "spaced-name"},
-		{"leading/trailing spaces", `  spaced name  `, "spaced-name"},
-		{"hyphen name", `hyphened-name`, "hyphened-name"},
-		{"multi-hyphen", `hyphened--name`, "hyphened-name"},
-		{"replaced characters", `a&b=c\d/:e*"f?_ g`, "a-b-c-d-e-f-g"},
-		{"removed characters", `foo!!bar@@and, more`, "foobarand-more"},
+		{"basic", "basic", "basic-61a7508e"},
+		{"spaces", `spaced name`, "spaced-name-b297cf60"},
+		{"leading/trailing spaces", `  spaced name  `, "spaced-name-175433e9"},
+		{"hyphen name", `hyphened-name`, "hyphened-name-789c55f2"},
+		{"multi-hyphen", `hyphened--name`, "hyphened-name-2da2a58f"},
+		{"replaced characters", `a&b=c\d/:e*"f?_ g`, "a-b-c-d-e-f-g-ffca6fb0"},
+		{"removed characters", `foo!!bar@@and, more`, "foobarand-more-7cee02ab"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/match/scraped.go

@@ -20,18 +20,52 @@ type GroupNamesFinder interface {
 	FindByNames(ctx context.Context, names []string, nocase bool) ([]*models.Group, error)
 }
 
+type SceneRelationships struct {
+	PerformerFinder PerformerFinder
+	TagFinder       models.TagQueryer
+	StudioFinder    StudioFinder
+}
+
+// MatchRelationships accepts a scraped scene and attempts to match its relationships to existing stash models.
+func (r SceneRelationships) MatchRelationships(ctx context.Context, s *models.ScrapedScene, endpoint string) error {
+	thisStudio := s.Studio
+	for thisStudio != nil {
+		if err := ScrapedStudio(ctx, r.StudioFinder, thisStudio, endpoint); err != nil {
+			return err
+		}
+
+		thisStudio = thisStudio.Parent
+	}
+
+	for _, p := range s.Performers {
+		err := ScrapedPerformer(ctx, r.PerformerFinder, p, endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, t := range s.Tags {
+		err := ScrapedTag(ctx, r.TagFinder, t)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // ScrapedPerformer matches the provided performer with the
 // performers in the database and sets the ID field if one is found.
-func ScrapedPerformer(ctx context.Context, qb PerformerFinder, p *models.ScrapedPerformer, stashBoxEndpoint *string) error {
+func ScrapedPerformer(ctx context.Context, qb PerformerFinder, p *models.ScrapedPerformer, stashBoxEndpoint string) error {
 	if p.StoredID != nil || p.Name == nil {
 		return nil
 	}
 
 	// Check if a performer with the StashID already exists
-	if stashBoxEndpoint != nil && p.RemoteSiteID != nil {
+	if stashBoxEndpoint != "" && p.RemoteSiteID != nil {
 		performers, err := qb.FindByStashID(ctx, models.StashID{
 			StashID:  *p.RemoteSiteID,
-			Endpoint: *stashBoxEndpoint,
+			Endpoint: stashBoxEndpoint,
 		})
 		if err != nil {
 			return err
@@ -73,16 +107,16 @@ type StudioFinder interface {
 
 // ScrapedStudio matches the provided studio with the studios
 // in the database and sets the ID field if one is found.
-func ScrapedStudio(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio, stashBoxEndpoint *string) error {
+func ScrapedStudio(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio, stashBoxEndpoint string) error {
 	if s.StoredID != nil {
 		return nil
 	}
 
 	// Check if a studio with the StashID already exists
-	if stashBoxEndpoint != nil && s.RemoteSiteID != nil {
+	if stashBoxEndpoint != "" && s.RemoteSiteID != nil {
 		studios, err := qb.FindByStashID(ctx, models.StashID{
 			StashID:  *s.RemoteSiteID,
-			Endpoint: *stashBoxEndpoint,
+			Endpoint: stashBoxEndpoint,
 		})
 		if err != nil {
 			return err
@@ -118,6 +152,19 @@ func ScrapedStudio(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio
 	return nil
 }
 
+// ScrapedStudioHierarchy executes ScrapedStudio for the provided studio and its parents recursively.
+func ScrapedStudioHierarchy(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio, stashBoxEndpoint string) error {
+	if err := ScrapedStudio(ctx, qb, s, stashBoxEndpoint); err != nil {
+		return err
+	}
+
+	if s.Parent == nil {
+		return nil
+	}
+
+	return ScrapedStudioHierarchy(ctx, qb, s.Parent, stashBoxEndpoint)
+}
+
 // ScrapedGroup matches the provided movie with the movies
 // in the database and returns the ID field if one is found.
 func ScrapedGroup(ctx context.Context, qb GroupNamesFinder, storedID *string, name *string) (matchedID *string, err error) {

pkg/models/fingerprint.go

@@ -26,6 +26,20 @@ func (f *Fingerprint) Value() string {
 	}
 }
 
+// String returns the string representation of the Fingerprint.
+// It will return an empty string if the Fingerprint is not a string.
+func (f Fingerprint) String() string {
+	s, _ := f.Fingerprint.(string)
+	return s
+}
+
+// Int64 returns the int64 representation of the Fingerprint.
+// It will return 0 if the Fingerprint is not an int64.
+func (f Fingerprint) Int64() int64 {
+	v, _ := f.Fingerprint.(int64)
+	return v
+}
+
 type Fingerprints []Fingerprint
 
 func (f Fingerprints) Remove(type_ string) Fingerprints {
@@ -102,33 +116,27 @@ func (f Fingerprints) For(type_ string) *Fingerprint {
 }
 
 func (f Fingerprints) Get(type_ string) interface{} {
-	for _, fp := range f {
-		if fp.Type == type_ {
-			return fp.Fingerprint
-		}
+	fp := f.For(type_)
+	if fp == nil {
+		return nil
 	}
-
-	return nil
+	return fp.Fingerprint
 }
 
 func (f Fingerprints) GetString(type_ string) string {
-	fp := f.Get(type_)
-	if fp != nil {
-		s, _ := fp.(string)
-		return s
+	fp := f.For(type_)
+	if fp == nil {
+		return ""
 	}
-
-	return ""
+	return fp.String()
 }
 
 func (f Fingerprints) GetInt64(type_ string) int64 {
-	fp := f.Get(type_)
+	fp := f.For(type_)
 	if fp != nil {
-		v, _ := fp.(int64)
-		return v
+		return 0
 	}
-
-	return 0
+	return fp.Int64()
 }
 
 // AppendUnique appends a fingerprint to the list if a Fingerprint of the same type does not already exist in the list. If one does, then it is updated with o's Fingerprint value.

pkg/models/jsonschema/scene.go

@@ -14,6 +14,7 @@ import (
 type SceneMarker struct {
 	Title      string        `json:"title,omitempty"`
 	Seconds    string        `json:"seconds,omitempty"`
+	EndSeconds string        `json:"end_seconds,omitempty"`
 	PrimaryTag string        `json:"primary_tag,omitempty"`
 	Tags       []string      `json:"tags,omitempty"`
 	CreatedAt  json.JSONTime `json:"created_at,omitempty"`

pkg/models/mocks/SceneReaderWriter.go

@@ -549,6 +549,29 @@ func (_m *SceneReaderWriter) FindByGroupID(ctx context.Context, groupID int) ([]
 	return r0, r1
 }
 
+// FindByIDs provides a mock function with given fields: ctx, ids
+func (_m *SceneReaderWriter) FindByIDs(ctx context.Context, ids []int) ([]*models.Scene, error) {
+	ret := _m.Called(ctx, ids)
+
+	var r0 []*models.Scene
+	if rf, ok := ret.Get(0).(func(context.Context, []int) []*models.Scene); ok {
+		r0 = rf(ctx, ids)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*models.Scene)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, []int) error); ok {
+		r1 = rf(ctx, ids)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // FindByOSHash provides a mock function with given fields: ctx, oshash
 func (_m *SceneReaderWriter) FindByOSHash(ctx context.Context, oshash string) ([]*models.Scene, error) {
 	ret := _m.Called(ctx, oshash)

pkg/models/mocks/query.go

@@ -18,6 +18,10 @@ func (s *sceneResolver) FindMany(ctx context.Context, ids []int) ([]*models.Scen
 	return s.scenes, nil
 }
 
+func (s *sceneResolver) FindByIDs(ctx context.Context, ids []int) ([]*models.Scene, error) {
+	return s.scenes, nil
+}
+
 func SceneQueryResult(scenes []*models.Scene, count int) *models.SceneQueryResult {
 	ret := models.NewSceneQueryResult(&sceneResolver{
 		scenes: scenes,

pkg/models/model_scraped_item.go

@@ -3,6 +3,7 @@ package models
 import (
 	"context"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
@@ -400,6 +401,10 @@ type ScrapedTag struct {
 
 func (ScrapedTag) IsScrapedContent() {}
 
+func ScrapedTagSortFunction(a, b *ScrapedTag) int {
+	return strings.Compare(strings.ToLower(a.Name), strings.ToLower(b.Name))
+}
+
 // A movie from a scraping operation...
 type ScrapedMovie struct {
 	StoredID *string        `json:"stored_id"`
@@ -492,3 +497,88 @@ func (g ScrapedGroup) ScrapedMovie() ScrapedMovie {
 
 	return ret
 }
+
+type ScrapedScene struct {
+	Title    *string  `json:"title"`
+	Code     *string  `json:"code"`
+	Details  *string  `json:"details"`
+	Director *string  `json:"director"`
+	URL      *string  `json:"url"`
+	URLs     []string `json:"urls"`
+	Date     *string  `json:"date"`
+	// This should be a base64 encoded data URL
+	Image        *string                `json:"image"`
+	File         *SceneFileType         `json:"file"`
+	Studio       *ScrapedStudio         `json:"studio"`
+	Tags         []*ScrapedTag          `json:"tags"`
+	Performers   []*ScrapedPerformer    `json:"performers"`
+	Groups       []*ScrapedGroup        `json:"groups"`
+	Movies       []*ScrapedMovie        `json:"movies"`
+	RemoteSiteID *string                `json:"remote_site_id"`
+	Duration     *int                   `json:"duration"`
+	Fingerprints []*StashBoxFingerprint `json:"fingerprints"`
+}
+
+func (ScrapedScene) IsScrapedContent() {}
+
+type ScrapedSceneInput struct {
+	Title        *string  `json:"title"`
+	Code         *string  `json:"code"`
+	Details      *string  `json:"details"`
+	Director     *string  `json:"director"`
+	URL          *string  `json:"url"`
+	URLs         []string `json:"urls"`
+	Date         *string  `json:"date"`
+	RemoteSiteID *string  `json:"remote_site_id"`
+}
+
+type ScrapedImage struct {
+	Title        *string             `json:"title"`
+	Code         *string             `json:"code"`
+	Details      *string             `json:"details"`
+	Photographer *string             `json:"photographer"`
+	URLs         []string            `json:"urls"`
+	Date         *string             `json:"date"`
+	Studio       *ScrapedStudio      `json:"studio"`
+	Tags         []*ScrapedTag       `json:"tags"`
+	Performers   []*ScrapedPerformer `json:"performers"`
+}
+
+func (ScrapedImage) IsScrapedContent() {}
+
+type ScrapedImageInput struct {
+	Title   *string  `json:"title"`
+	Code    *string  `json:"code"`
+	Details *string  `json:"details"`
+	URLs    []string `json:"urls"`
+	Date    *string  `json:"date"`
+}
+
+type ScrapedGallery struct {
+	Title        *string             `json:"title"`
+	Code         *string             `json:"code"`
+	Details      *string             `json:"details"`
+	Photographer *string             `json:"photographer"`
+	URLs         []string            `json:"urls"`
+	Date         *string             `json:"date"`
+	Studio       *ScrapedStudio      `json:"studio"`
+	Tags         []*ScrapedTag       `json:"tags"`
+	Performers   []*ScrapedPerformer `json:"performers"`
+
+	// deprecated
+	URL *string `json:"url"`
+}
+
+func (ScrapedGallery) IsScrapedContent() {}
+
+type ScrapedGalleryInput struct {
+	Title        *string  `json:"title"`
+	Code         *string  `json:"code"`
+	Details      *string  `json:"details"`
+	Photographer *string  `json:"photographer"`
+	URLs         []string `json:"urls"`
+	Date         *string  `json:"date"`
+
+	// deprecated
+	URL *string `json:"url"`
+}

pkg/models/repository_scene.go

@@ -10,6 +10,9 @@ type SceneGetter interface {
 	// TODO - rename this to Find and remove existing method
 	FindMany(ctx context.Context, ids []int) ([]*Scene, error)
 	Find(ctx context.Context, id int) (*Scene, error)
+	// FindByIDs works the same way as FindMany, but it ignores any scenes not found
+	// Scenes are not guaranteed to be in the same order as the input
+	FindByIDs(ctx context.Context, ids []int) ([]*Scene, error)
 }
 
 // SceneFinder provides methods to find scenes.

pkg/models/stash_box.go

@@ -7,7 +7,8 @@ type StashBoxFingerprint struct {
 }
 
 type StashBox struct {
-	Endpoint string `json:"endpoint"`
-	APIKey   string `json:"api_key"`
-	Name     string `json:"name"`
+	Endpoint             string `json:"endpoint"`
+	APIKey               string `json:"api_key"`
+	Name                 string `json:"name"`
+	MaxRequestsPerMinute int    `json:"max_requests_per_minute" koanf:"max_requests_per_minute"`
 }

pkg/scene/export.go

@@ -235,6 +235,10 @@ func GetSceneMarkersJSON(ctx context.Context, markerReader models.SceneMarkerFin
 			UpdatedAt:  json.JSONTime{Time: sceneMarker.UpdatedAt},
 		}
 
+		if sceneMarker.EndSeconds != nil {
+			sceneMarkerJSON.EndSeconds = getDecimalString(*sceneMarker.EndSeconds)
+		}
+
 		results = append(results, sceneMarkerJSON)
 	}
 

pkg/scene/find.go

@@ -0,0 +1,90 @@
+package scene
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/stashapp/stash/pkg/models"
+)
+
+type LoadRelationshipOption func(context.Context, *models.Scene, models.SceneReader) error
+
+func LoadURLs(ctx context.Context, scene *models.Scene, r models.SceneReader) error {
+	if err := scene.LoadURLs(ctx, r); err != nil {
+		return fmt.Errorf("loading scene URLs: %w", err)
+	}
+
+	return nil
+}
+
+func LoadStashIDs(ctx context.Context, scene *models.Scene, r models.SceneReader) error {
+	if err := scene.LoadStashIDs(ctx, r); err != nil {
+		return fmt.Errorf("failed to load stash IDs for scene %d: %w", scene.ID, err)
+	}
+
+	return nil
+}
+
+func LoadFiles(ctx context.Context, scene *models.Scene, r models.SceneReader) error {
+	if err := scene.LoadFiles(ctx, r); err != nil {
+		return fmt.Errorf("failed to load files for scene %d: %w", scene.ID, err)
+	}
+
+	return nil
+}
+
+// FindByIDs retrieves multiple scenes by their IDs.
+// Missing scenes will be ignored, and the returned scenes are unsorted.
+// This method will load the specified relationships for each scene.
+func (s *Service) FindByIDs(ctx context.Context, ids []int, load ...LoadRelationshipOption) ([]*models.Scene, error) {
+	var scenes []*models.Scene
+	qb := s.Repository
+
+	var err error
+	scenes, err = qb.FindByIDs(ctx, ids)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO - we should bulk load these relationships
+	for _, scene := range scenes {
+		if err := s.LoadRelationships(ctx, scene, load...); err != nil {
+			return nil, err
+		}
+	}
+
+	return scenes, nil
+}
+
+// FindMany retrieves multiple scenes by their IDs. Return value is guaranteed to be in the same order as the input.
+// Missing scenes will return an error.
+// This method will load the specified relationships for each scene.
+func (s *Service) FindMany(ctx context.Context, ids []int, load ...LoadRelationshipOption) ([]*models.Scene, error) {
+	var scenes []*models.Scene
+	qb := s.Repository
+
+	var err error
+	scenes, err = qb.FindMany(ctx, ids)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO - we should bulk load these relationships
+	for _, scene := range scenes {
+		if err := s.LoadRelationships(ctx, scene, load...); err != nil {
+			return nil, err
+		}
+	}
+
+	return scenes, nil
+}
+
+func (s *Service) LoadRelationships(ctx context.Context, scene *models.Scene, load ...LoadRelationshipOption) error {
+	for _, l := range load {
+		if err := l(ctx, scene, s.Repository); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/scene/fingerprints.go

@@ -0,0 +1,40 @@
+package scene
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/stashapp/stash/pkg/models"
+)
+
+// GetFingerprints returns the fingerprints for the given scene ids.
+func (s *Service) GetScenesFingerprints(ctx context.Context, ids []int) ([]models.Fingerprints, error) {
+	fingerprints := make([]models.Fingerprints, len(ids))
+
+	qb := s.Repository
+
+	for i, sceneID := range ids {
+		scene, err := qb.Find(ctx, sceneID)
+		if err != nil {
+			return nil, err
+		}
+
+		if scene == nil {
+			return nil, fmt.Errorf("scene with id %d not found", sceneID)
+		}
+
+		if err := scene.LoadFiles(ctx, qb); err != nil {
+			return nil, err
+		}
+
+		var sceneFPs models.Fingerprints
+
+		for _, f := range scene.Files.List() {
+			sceneFPs = append(sceneFPs, f.Fingerprints...)
+		}
+
+		fingerprints[i] = sceneFPs
+	}
+
+	return fingerprints, nil
+}

pkg/scene/marker_import.go

@@ -27,12 +27,20 @@ type MarkerImporter struct {
 
 func (i *MarkerImporter) PreImport(ctx context.Context) error {
 	seconds, _ := strconv.ParseFloat(i.Input.Seconds, 64)
+
+	var endSeconds *float64
+	if i.Input.EndSeconds != "" {
+		parsedEndSeconds, _ := strconv.ParseFloat(i.Input.EndSeconds, 64)
+		endSeconds = &parsedEndSeconds
+	}
+
 	i.marker = models.SceneMarker{
-		Title:     i.Input.Title,
-		Seconds:   seconds,
-		SceneID:   i.SceneID,
-		CreatedAt: i.Input.CreatedAt.GetTime(),
-		UpdatedAt: i.Input.UpdatedAt.GetTime(),
+		Title:      i.Input.Title,
+		Seconds:    seconds,
+		EndSeconds: endSeconds,
+		SceneID:    i.SceneID,
+		CreatedAt:  i.Input.CreatedAt.GetTime(),
+		UpdatedAt:  i.Input.UpdatedAt.GetTime(),
 	}
 
 	if err := i.populateTags(ctx); err != nil {

pkg/scraper/action.go

@@ -29,9 +29,9 @@ type scraperActionImpl interface {
 	scrapeByName(ctx context.Context, name string, ty ScrapeContentType) ([]ScrapedContent, error)
 	scrapeByFragment(ctx context.Context, input Input) (ScrapedContent, error)
 
-	scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error)
-	scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error)
-	scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error)
+	scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error)
+	scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error)
+	scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error)
 }
 
 func (c config) getScraper(scraper scraperTypeConfig, client *http.Client, globalConfig GlobalConfig) scraperActionImpl {

pkg/scraper/autotag.go

@@ -89,8 +89,8 @@ func autotagMatchTags(ctx context.Context, path string, tagReader models.TagAuto
 	return ret, nil
 }
 
-func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scene *models.Scene) (*ScrapedScene, error) {
-	var ret *ScrapedScene
+func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scene *models.Scene) (*models.ScrapedScene, error) {
+	var ret *models.ScrapedScene
 	const trimExt = false
 
 	// populate performers, studio and tags based on scene path
@@ -115,7 +115,7 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
 		}
 
 		if len(performers) > 0 || studio != nil || len(tags) > 0 {
-			ret = &ScrapedScene{
+			ret = &models.ScrapedScene{
 				Performers: performers,
 				Studio:     studio,
 				Tags:       tags,
@@ -130,7 +130,7 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
 	return ret, nil
 }
 
-func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	path := gallery.Path
 	if path == "" {
 		// not valid for non-path-based galleries
@@ -140,7 +140,7 @@ func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, ga
 	// only trim extension if gallery is file-based
 	trimExt := gallery.PrimaryFileID != nil
 
-	var ret *ScrapedGallery
+	var ret *models.ScrapedGallery
 
 	// populate performers, studio and tags based on scene path
 	if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
@@ -160,7 +160,7 @@ func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, ga
 		}
 
 		if len(performers) > 0 || studio != nil || len(tags) > 0 {
-			ret = &ScrapedGallery{
+			ret = &models.ScrapedGallery{
 				Performers: performers,
 				Studio:     studio,
 				Tags:       tags,

pkg/scraper/config.go

@@ -378,6 +378,11 @@ func (c config) matchesURL(url string, ty ScrapeContentType) bool {
 			}
 		}
 	case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
+		for _, scraper := range c.GroupByURL {
+			if scraper.matchesURL(url) {
+				return true
+			}
+		}
 		for _, scraper := range c.MovieByURL {
 			if scraper.matchesURL(url) {
 				return true

pkg/scraper/gallery.go

@@ -1,32 +0,0 @@
-package scraper
-
-import "github.com/stashapp/stash/pkg/models"
-
-type ScrapedGallery struct {
-	Title        *string                    `json:"title"`
-	Code         *string                    `json:"code"`
-	Details      *string                    `json:"details"`
-	Photographer *string                    `json:"photographer"`
-	URLs         []string                   `json:"urls"`
-	Date         *string                    `json:"date"`
-	Studio       *models.ScrapedStudio      `json:"studio"`
-	Tags         []*models.ScrapedTag       `json:"tags"`
-	Performers   []*models.ScrapedPerformer `json:"performers"`
-
-	// deprecated
-	URL *string `json:"url"`
-}
-
-func (ScrapedGallery) IsScrapedContent() {}
-
-type ScrapedGalleryInput struct {
-	Title        *string  `json:"title"`
-	Code         *string  `json:"code"`
-	Details      *string  `json:"details"`
-	Photographer *string  `json:"photographer"`
-	URLs         []string `json:"urls"`
-	Date         *string  `json:"date"`
-
-	// deprecated
-	URL *string `json:"url"`
-}

pkg/scraper/group.go

@@ -60,7 +60,7 @@ func (g group) viaFragment(ctx context.Context, client *http.Client, input Input
 	return s.scrapeByFragment(ctx, input)
 }
 
-func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error) {
+func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*models.ScrapedScene, error) {
 	if g.config.SceneByFragment == nil {
 		return nil, ErrNotSupported
 	}
@@ -69,7 +69,7 @@ func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.
 	return s.scrapeSceneByScene(ctx, scene)
 }
 
-func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	if g.config.GalleryByFragment == nil {
 		return nil, ErrNotSupported
 	}
@@ -78,7 +78,7 @@ func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *mod
 	return s.scrapeGalleryByGallery(ctx, gallery)
 }
 
-func (g group) viaImage(ctx context.Context, client *http.Client, gallery *models.Image) (*ScrapedImage, error) {
+func (g group) viaImage(ctx context.Context, client *http.Client, gallery *models.Image) (*models.ScrapedImage, error) {
 	if g.config.ImageByFragment == nil {
 		return nil, ErrNotSupported
 	}

pkg/scraper/image.go

@@ -11,28 +11,6 @@ import (
 	"github.com/stashapp/stash/pkg/utils"
 )
 
-type ScrapedImage struct {
-	Title        *string                    `json:"title"`
-	Code         *string                    `json:"code"`
-	Details      *string                    `json:"details"`
-	Photographer *string                    `json:"photographer"`
-	URLs         []string                   `json:"urls"`
-	Date         *string                    `json:"date"`
-	Studio       *models.ScrapedStudio      `json:"studio"`
-	Tags         []*models.ScrapedTag       `json:"tags"`
-	Performers   []*models.ScrapedPerformer `json:"performers"`
-}
-
-func (ScrapedImage) IsScrapedContent() {}
-
-type ScrapedImageInput struct {
-	Title   *string  `json:"title"`
-	Code    *string  `json:"code"`
-	Details *string  `json:"details"`
-	URLs    []string `json:"urls"`
-	Date    *string  `json:"date"`
-}
-
 func setPerformerImage(ctx context.Context, client *http.Client, p *models.ScrapedPerformer, globalConfig GlobalConfig) error {
 	// backwards compatibility: we fetch the image if it's a URL and set it to the first image
 	// Image is deprecated, so only do this if Images is unset
@@ -59,7 +37,7 @@ func setPerformerImage(ctx context.Context, client *http.Client, p *models.Scrap
 	return nil
 }
 
-func setSceneImage(ctx context.Context, client *http.Client, s *ScrapedScene, globalConfig GlobalConfig) error {
+func setSceneImage(ctx context.Context, client *http.Client, s *models.ScrapedScene, globalConfig GlobalConfig) error {
 	// don't try to get the image if it doesn't appear to be a URL
 	if s.Image == nil || !strings.HasPrefix(*s.Image, "http") {
 		// nothing to do

pkg/scraper/json.go

@@ -172,7 +172,7 @@ func (s *jsonScraper) scrapeByName(ctx context.Context, name string, ty ScrapeCo
 	return nil, ErrNotSupported
 }
 
-func (s *jsonScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
+func (s *jsonScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromScene(scene)
 	if s.scraper.QueryURLReplacements != nil {
@@ -231,7 +231,7 @@ func (s *jsonScraper) scrapeByFragment(ctx context.Context, input Input) (Scrape
 	return scraper.scrapeScene(ctx, q)
 }
 
-func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
+func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromImage(image)
 	if s.scraper.QueryURLReplacements != nil {
@@ -255,7 +255,7 @@ func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Imag
 	return scraper.scrapeImage(ctx, q)
 }
 
-func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromGallery(gallery)
 	if s.scraper.QueryURLReplacements != nil {

pkg/scraper/mapped.go

@@ -851,7 +851,10 @@ type mappedScraper struct {
 	Gallery   *mappedGalleryScraperConfig   `yaml:"gallery"`
 	Image     *mappedImageScraperConfig     `yaml:"image"`
 	Performer *mappedPerformerScraperConfig `yaml:"performer"`
-	Movie     *mappedMovieScraperConfig     `yaml:"movie"`
+	Group     *mappedMovieScraperConfig     `yaml:"group"`
+
+	// deprecated
+	Movie *mappedMovieScraperConfig `yaml:"movie"`
 }
 
 type mappedResult map[string]interface{}
@@ -997,8 +1000,8 @@ func (s mappedScraper) scrapePerformers(ctx context.Context, q mappedQuery) ([]*
 	return ret, nil
 }
 
-// processSceneRelationships sets the relationships on the ScrapedScene. It returns true if any relationships were set.
-func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQuery, resultIndex int, ret *ScrapedScene) bool {
+// processSceneRelationships sets the relationships on the models.ScrapedScene. It returns true if any relationships were set.
+func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQuery, resultIndex int, ret *models.ScrapedScene) bool {
 	sceneScraperConfig := s.Scene
 
 	scenePerformersMap := sceneScraperConfig.Performers
@@ -1082,8 +1085,8 @@ func processRelationships[T any](ctx context.Context, s mappedScraper, relations
 	return ret
 }
 
-func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*ScrapedScene, error) {
-	var ret []*ScrapedScene
+func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*models.ScrapedScene, error) {
+	var ret []*models.ScrapedScene
 
 	sceneScraperConfig := s.Scene
 	sceneMap := sceneScraperConfig.mappedConfig
@@ -1097,7 +1100,7 @@ func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*Scra
 	for i, r := range results {
 		logger.Debug(`Processing scene:`)
 
-		var thisScene ScrapedScene
+		var thisScene models.ScrapedScene
 		r.apply(&thisScene)
 		s.processSceneRelationships(ctx, q, i, &thisScene)
 		ret = append(ret, &thisScene)
@@ -1106,7 +1109,7 @@ func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*Scra
 	return ret, nil
 }
 
-func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*ScrapedScene, error) {
+func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*models.ScrapedScene, error) {
 	sceneScraperConfig := s.Scene
 	if sceneScraperConfig == nil {
 		return nil, nil
@@ -1117,7 +1120,7 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*Scraped
 	logger.Debug(`Processing scene:`)
 	results := sceneMap.process(ctx, q, s.Common, urlsIsMulti)
 
-	var ret ScrapedScene
+	var ret models.ScrapedScene
 	if len(results) > 0 {
 		results[0].apply(&ret)
 	}
@@ -1133,8 +1136,8 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*Scraped
 	return nil, nil
 }
 
-func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*ScrapedImage, error) {
-	var ret ScrapedImage
+func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*models.ScrapedImage, error) {
+	var ret models.ScrapedImage
 
 	imageScraperConfig := s.Image
 	if imageScraperConfig == nil {
@@ -1184,8 +1187,8 @@ func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*Scraped
 	return &ret, nil
 }
 
-func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*ScrapedGallery, error) {
-	var ret ScrapedGallery
+func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*models.ScrapedGallery, error) {
+	var ret models.ScrapedGallery
 
 	galleryScraperConfig := s.Gallery
 	if galleryScraperConfig == nil {
@@ -1247,24 +1250,29 @@ func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*Scrap
 	return &ret, nil
 }
 
-func (s mappedScraper) scrapeGroup(ctx context.Context, q mappedQuery) (*models.ScrapedMovie, error) {
-	var ret models.ScrapedMovie
+func (s mappedScraper) scrapeGroup(ctx context.Context, q mappedQuery) (*models.ScrapedGroup, error) {
+	var ret models.ScrapedGroup
 
-	movieScraperConfig := s.Movie
-	if movieScraperConfig == nil {
+	// try group scraper first, falling back to movie
+	groupScraperConfig := s.Group
+
+	if groupScraperConfig == nil {
+		groupScraperConfig = s.Movie
+	}
+	if groupScraperConfig == nil {
 		return nil, nil
 	}
 
-	movieMap := movieScraperConfig.mappedConfig
+	groupMap := groupScraperConfig.mappedConfig
 
-	movieStudioMap := movieScraperConfig.Studio
-	movieTagsMap := movieScraperConfig.Tags
+	groupStudioMap := groupScraperConfig.Studio
+	groupTagsMap := groupScraperConfig.Tags
 
-	results := movieMap.process(ctx, q, s.Common, urlsIsMulti)
+	results := groupMap.process(ctx, q, s.Common, urlsIsMulti)
 
-	if movieStudioMap != nil {
-		logger.Debug(`Processing movie studio:`)
-		studioResults := movieStudioMap.process(ctx, q, s.Common, nil)
+	if groupStudioMap != nil {
+		logger.Debug(`Processing group studio:`)
+		studioResults := groupStudioMap.process(ctx, q, s.Common, nil)
 
 		if len(studioResults) > 0 {
 			studio := &models.ScrapedStudio{}
@@ -1274,9 +1282,9 @@ func (s mappedScraper) scrapeGroup(ctx context.Context, q mappedQuery) (*models.
 	}
 
 	// now apply the tags
-	if movieTagsMap != nil {
-		logger.Debug(`Processing movie tags:`)
-		tagResults := movieTagsMap.process(ctx, q, s.Common, nil)
+	if groupTagsMap != nil {
+		logger.Debug(`Processing group tags:`)
+		tagResults := groupTagsMap.process(ctx, q, s.Common, nil)
 
 		for _, p := range tagResults {
 			tag := &models.ScrapedTag{}

pkg/scraper/postprocessing.go

@@ -23,23 +23,23 @@ func (c Cache) postScrape(ctx context.Context, content ScrapedContent, excludeTa
 		}
 	case models.ScrapedPerformer:
 		return c.postScrapePerformer(ctx, v, excludeTagRE)
-	case *ScrapedScene:
+	case *models.ScrapedScene:
 		if v != nil {
 			return c.postScrapeScene(ctx, *v, excludeTagRE)
 		}
-	case ScrapedScene:
+	case models.ScrapedScene:
 		return c.postScrapeScene(ctx, v, excludeTagRE)
-	case *ScrapedGallery:
+	case *models.ScrapedGallery:
 		if v != nil {
 			return c.postScrapeGallery(ctx, *v, excludeTagRE)
 		}
-	case ScrapedGallery:
+	case models.ScrapedGallery:
 		return c.postScrapeGallery(ctx, v, excludeTagRE)
-	case *ScrapedImage:
+	case *models.ScrapedImage:
 		if v != nil {
 			return c.postScrapeImage(ctx, *v, excludeTagRE)
 		}
-	case ScrapedImage:
+	case models.ScrapedImage:
 		return c.postScrapeImage(ctx, v, excludeTagRE)
 	case *models.ScrapedMovie:
 		if v != nil {
@@ -133,7 +133,7 @@ func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie, exclu
 		m.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		if m.Studio != nil {
-			if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, nil); err != nil {
+			if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, ""); err != nil {
 				return err
 			}
 		}
@@ -165,7 +165,7 @@ func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup, exclu
 		m.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		if m.Studio != nil {
-			if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, nil); err != nil {
+			if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, ""); err != nil {
 				return err
 			}
 		}
@@ -201,7 +201,7 @@ func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPer
 	return ignoredTags, nil
 }
 
-func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
+func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
 	// set the URL/URLs field
 	if scene.URL == nil && len(scene.URLs) > 0 {
 		scene.URL = &scene.URLs[0]
@@ -227,7 +227,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene, excludeT
 				return err
 			}
 
-			if err := match.ScrapedPerformer(ctx, pqb, p, nil); err != nil {
+			if err := match.ScrapedPerformer(ctx, pqb, p, ""); err != nil {
 				return err
 			}
 
@@ -277,7 +277,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene, excludeT
 		scene.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		if scene.Studio != nil {
-			err := match.ScrapedStudio(ctx, sqb, scene.Studio, nil)
+			err := match.ScrapedStudio(ctx, sqb, scene.Studio, "")
 			if err != nil {
 				return err
 			}
@@ -296,7 +296,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene, excludeT
 	return scene, ignoredTags, nil
 }
 
-func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
+func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
 	// set the URL/URLs field
 	if g.URL == nil && len(g.URLs) > 0 {
 		g.URL = &g.URLs[0]
@@ -312,7 +312,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery, excludeT
 		sqb := r.StudioFinder
 
 		for _, p := range g.Performers {
-			err := match.ScrapedPerformer(ctx, pqb, p, nil)
+			err := match.ScrapedPerformer(ctx, pqb, p, "")
 			if err != nil {
 				return err
 			}
@@ -325,7 +325,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery, excludeT
 		g.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		if g.Studio != nil {
-			err := match.ScrapedStudio(ctx, sqb, g.Studio, nil)
+			err := match.ScrapedStudio(ctx, sqb, g.Studio, "")
 			if err != nil {
 				return err
 			}
@@ -339,7 +339,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery, excludeT
 	return g, ignoredTags, nil
 }
 
-func (c Cache) postScrapeImage(ctx context.Context, image ScrapedImage, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
+func (c Cache) postScrapeImage(ctx context.Context, image models.ScrapedImage, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
 	r := c.repository
 	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
 		pqb := r.PerformerFinder
@@ -347,7 +347,7 @@ func (c Cache) postScrapeImage(ctx context.Context, image ScrapedImage, excludeT
 		sqb := r.StudioFinder
 
 		for _, p := range image.Performers {
-			if err := match.ScrapedPerformer(ctx, pqb, p, nil); err != nil {
+			if err := match.ScrapedPerformer(ctx, pqb, p, ""); err != nil {
 				return err
 			}
 		}
@@ -360,7 +360,7 @@ func (c Cache) postScrapeImage(ctx context.Context, image ScrapedImage, excludeT
 		image.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
 
 		if image.Studio != nil {
-			err := match.ScrapedStudio(ctx, sqb, image.Studio, nil)
+			err := match.ScrapedStudio(ctx, sqb, image.Studio, "")
 			if err != nil {
 				return err
 			}

pkg/scraper/query_url.go

@@ -26,7 +26,7 @@ func queryURLParametersFromScene(scene *models.Scene) queryURLParameters {
 	return ret
 }
 
-func queryURLParametersFromScrapedScene(scene ScrapedSceneInput) queryURLParameters {
+func queryURLParametersFromScrapedScene(scene models.ScrapedSceneInput) queryURLParameters {
 	ret := make(queryURLParameters)
 
 	setField := func(field string, value *string) {

pkg/scraper/scene.go

@@ -1,39 +0,0 @@
-package scraper
-
-import (
-	"github.com/stashapp/stash/pkg/models"
-)
-
-type ScrapedScene struct {
-	Title    *string  `json:"title"`
-	Code     *string  `json:"code"`
-	Details  *string  `json:"details"`
-	Director *string  `json:"director"`
-	URL      *string  `json:"url"`
-	URLs     []string `json:"urls"`
-	Date     *string  `json:"date"`
-	// This should be a base64 encoded data URL
-	Image        *string                       `json:"image"`
-	File         *models.SceneFileType         `json:"file"`
-	Studio       *models.ScrapedStudio         `json:"studio"`
-	Tags         []*models.ScrapedTag          `json:"tags"`
-	Performers   []*models.ScrapedPerformer    `json:"performers"`
-	Groups       []*models.ScrapedGroup        `json:"groups"`
-	Movies       []*models.ScrapedMovie        `json:"movies"`
-	RemoteSiteID *string                       `json:"remote_site_id"`
-	Duration     *int                          `json:"duration"`
-	Fingerprints []*models.StashBoxFingerprint `json:"fingerprints"`
-}
-
-func (ScrapedScene) IsScrapedContent() {}
-
-type ScrapedSceneInput struct {
-	Title        *string  `json:"title"`
-	Code         *string  `json:"code"`
-	Details      *string  `json:"details"`
-	Director     *string  `json:"director"`
-	URL          *string  `json:"url"`
-	URLs         []string `json:"urls"`
-	Date         *string  `json:"date"`
-	RemoteSiteID *string  `json:"remote_site_id"`
-}

pkg/scraper/scraper.go

@@ -163,9 +163,9 @@ var (
 // set to nil.
 type Input struct {
 	Performer *ScrapedPerformerInput
-	Scene     *ScrapedSceneInput
-	Gallery   *ScrapedGalleryInput
-	Image     *ScrapedImageInput
+	Scene     *models.ScrapedSceneInput
+	Gallery   *models.ScrapedGalleryInput
+	Image     *models.ScrapedImageInput
 }
 
 // populateURL populates the URL field of the input based on the
@@ -227,7 +227,7 @@ type fragmentScraper interface {
 type sceneScraper interface {
 	scraper
 
-	viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error)
+	viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*models.ScrapedScene, error)
 }
 
 // imageScraper is a scraper which supports image scrapes with
@@ -235,7 +235,7 @@ type sceneScraper interface {
 type imageScraper interface {
 	scraper
 
-	viaImage(ctx context.Context, client *http.Client, image *models.Image) (*ScrapedImage, error)
+	viaImage(ctx context.Context, client *http.Client, image *models.Image) (*models.ScrapedImage, error)
 }
 
 // galleryScraper is a scraper which supports gallery scrapes with
@@ -243,5 +243,5 @@ type imageScraper interface {
 type galleryScraper interface {
 	scraper
 
-	viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error)
+	viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error)
 }

pkg/scraper/script.go

@@ -328,7 +328,7 @@ func (s *scriptScraper) scrapeByName(ctx context.Context, name string, ty Scrape
 			}
 		}
 	case ScrapeContentTypeScene:
-		var scenes []ScrapedScene
+		var scenes []models.ScrapedScene
 		err = s.runScraperScript(ctx, input, &scenes)
 		if err == nil {
 			for _, s := range scenes {
@@ -377,11 +377,11 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
 		err := s.runScraperScript(ctx, input, &performer)
 		return performer, err
 	case ScrapeContentTypeGallery:
-		var gallery *ScrapedGallery
+		var gallery *models.ScrapedGallery
 		err := s.runScraperScript(ctx, input, &gallery)
 		return gallery, err
 	case ScrapeContentTypeScene:
-		var scene *ScrapedScene
+		var scene *models.ScrapedScene
 		err := s.runScraperScript(ctx, input, &scene)
 		return scene, err
 	case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
@@ -389,7 +389,7 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
 		err := s.runScraperScript(ctx, input, &movie)
 		return movie, err
 	case ScrapeContentTypeImage:
-		var image *ScrapedImage
+		var image *models.ScrapedImage
 		err := s.runScraperScript(ctx, input, &image)
 		return image, err
 	}
@@ -397,42 +397,42 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
 	return nil, ErrNotSupported
 }
 
-func (s *scriptScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
+func (s *scriptScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
 	inString, err := json.Marshal(sceneInputFromScene(scene))
 
 	if err != nil {
 		return nil, err
 	}
 
-	var ret *ScrapedScene
+	var ret *models.ScrapedScene
 
 	err = s.runScraperScript(ctx, string(inString), &ret)
 
 	return ret, err
 }
 
-func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	inString, err := json.Marshal(galleryInputFromGallery(gallery))
 
 	if err != nil {
 		return nil, err
 	}
 
-	var ret *ScrapedGallery
+	var ret *models.ScrapedGallery
 
 	err = s.runScraperScript(ctx, string(inString), &ret)
 
 	return ret, err
 }
 
-func (s *scriptScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
+func (s *scriptScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
 	inString, err := json.Marshal(imageToUpdateInput(image))
 
 	if err != nil {
 		return nil, err
 	}
 
-	var ret *ScrapedImage
+	var ret *models.ScrapedImage
 
 	err = s.runScraperScript(ctx, string(inString), &ret)
 

pkg/scraper/stash.go

@@ -170,7 +170,7 @@ func (s *stashScraper) scrapeByPerformerFragment(ctx context.Context, scrapedPer
 	return &ret, nil
 }
 
-func (s *stashScraper) scrapeBySceneFragment(ctx context.Context, scrapedScene ScrapedSceneInput) (ScrapedContent, error) {
+func (s *stashScraper) scrapeBySceneFragment(ctx context.Context, scrapedScene models.ScrapedSceneInput) (ScrapedContent, error) {
 	client := s.getStashClient()
 
 	var q struct {
@@ -219,8 +219,8 @@ type stashFindSceneNamesResultType struct {
 	Scenes []*scrapedSceneStash `graphql:"scenes"`
 }
 
-func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scene *scrapedSceneStash) (*ScrapedScene, error) {
-	ret := ScrapedScene{}
+func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scene *scrapedSceneStash) (*models.ScrapedScene, error) {
+	ret := models.ScrapedScene{}
 	err := copier.Copy(&ret, scene)
 	if err != nil {
 		return nil, err
@@ -341,7 +341,7 @@ type scrapedSceneStash struct {
 	Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
 }
 
-func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
+func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
 	// query by MD5
 	var q struct {
 		FindScene *scrapedSceneStash `graphql:"findSceneByHash(input: $c)"`
@@ -401,7 +401,7 @@ type scrapedGalleryStash struct {
 	Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
 }
 
-func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	var q struct {
 		FindGallery *scrapedGalleryStash `graphql:"findGalleryByHash(input: $c)"`
 	}
@@ -425,7 +425,7 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
 	}
 
 	// need to copy back to a scraped scene
-	ret := ScrapedGallery{}
+	ret := models.ScrapedGallery{}
 	if err := copier.Copy(&ret, q.FindGallery); err != nil {
 		return nil, err
 	}
@@ -433,7 +433,7 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
 	return &ret, nil
 }
 
-func (s *stashScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
+func (s *stashScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
 	return nil, ErrNotSupported
 }
 

pkg/scraper/stashbox/client.go

@@ -1,102 +0,0 @@
-// Package stashbox provides a client interface to a stash-box server instance.
-package stashbox
-
-import (
-	"context"
-	"net/http"
-	"regexp"
-
-	"github.com/Yamashou/gqlgenc/clientv2"
-	"github.com/stashapp/stash/pkg/match"
-	"github.com/stashapp/stash/pkg/models"
-	"github.com/stashapp/stash/pkg/scraper"
-	"github.com/stashapp/stash/pkg/scraper/stashbox/graphql"
-	"github.com/stashapp/stash/pkg/txn"
-)
-
-type SceneReader interface {
-	models.SceneGetter
-	models.StashIDLoader
-	models.VideoFileLoader
-}
-
-type PerformerReader interface {
-	models.PerformerGetter
-	match.PerformerFinder
-	models.AliasLoader
-	models.StashIDLoader
-	models.URLLoader
-	FindBySceneID(ctx context.Context, sceneID int) ([]*models.Performer, error)
-	GetImage(ctx context.Context, performerID int) ([]byte, error)
-}
-
-type StudioReader interface {
-	models.StudioGetter
-	match.StudioFinder
-	models.StashIDLoader
-}
-
-type TagFinder interface {
-	models.TagQueryer
-	FindBySceneID(ctx context.Context, sceneID int) ([]*models.Tag, error)
-}
-
-type Repository struct {
-	TxnManager models.TxnManager
-
-	Scene     SceneReader
-	Performer PerformerReader
-	Tag       TagFinder
-	Studio    StudioReader
-}
-
-func NewRepository(repo models.Repository) Repository {
-	return Repository{
-		TxnManager: repo.TxnManager,
-		Scene:      repo.Scene,
-		Performer:  repo.Performer,
-		Tag:        repo.Tag,
-		Studio:     repo.Studio,
-	}
-}
-
-func (r *Repository) WithReadTxn(ctx context.Context, fn txn.TxnFunc) error {
-	return txn.WithReadTxn(ctx, r.TxnManager, fn)
-}
-
-// Client represents the client interface to a stash-box server instance.
-type Client struct {
-	client     *graphql.Client
-	repository Repository
-	box        models.StashBox
-
-	// tag patterns to be excluded
-	excludeTagRE []*regexp.Regexp
-}
-
-// NewClient returns a new instance of a stash-box client.
-func NewClient(box models.StashBox, repo Repository, excludeTagPatterns []string) *Client {
-	authHeader := func(ctx context.Context, req *http.Request, gqlInfo *clientv2.GQLRequestInfo, res interface{}, next clientv2.RequestInterceptorFunc) error {
-		req.Header.Set("ApiKey", box.APIKey)
-		return next(ctx, req, gqlInfo, res)
-	}
-
-	client := &graphql.Client{
-		Client: clientv2.NewClient(http.DefaultClient, box.Endpoint, nil, authHeader),
-	}
-
-	return &Client{
-		client:       client,
-		repository:   repo,
-		box:          box,
-		excludeTagRE: scraper.CompileExclusionRegexps(excludeTagPatterns),
-	}
-}
-
-func (c Client) getHTTPClient() *http.Client {
-	return c.client.Client.Client
-}
-
-func (c Client) GetUser(ctx context.Context) (*graphql.Me, error) {
-	return c.client.Me(ctx)
-}

pkg/scraper/stashbox/models.go

@@ -1,13 +0,0 @@
-package stashbox
-
-import "github.com/stashapp/stash/pkg/models"
-
-type StashBoxStudioQueryResult struct {
-	Query   string                  `json:"query"`
-	Results []*models.ScrapedStudio `json:"results"`
-}
-
-type StashBoxPerformerQueryResult struct {
-	Query   string                     `json:"query"`
-	Results []*models.ScrapedPerformer `json:"results"`
-}

pkg/scraper/stashbox/scene.go

@@ -1,589 +0,0 @@
-package stashbox
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-
-	"github.com/stashapp/stash/pkg/logger"
-	"github.com/stashapp/stash/pkg/match"
-	"github.com/stashapp/stash/pkg/models"
-	"github.com/stashapp/stash/pkg/scraper"
-	"github.com/stashapp/stash/pkg/scraper/stashbox/graphql"
-	"github.com/stashapp/stash/pkg/sliceutil"
-	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
-	"github.com/stashapp/stash/pkg/utils"
-)
-
-// QueryStashBoxScene queries stash-box for scenes using a query string.
-func (c Client) QueryStashBoxScene(ctx context.Context, queryStr string) ([]*scraper.ScrapedScene, error) {
-	scenes, err := c.client.SearchScene(ctx, queryStr)
-	if err != nil {
-		return nil, err
-	}
-
-	sceneFragments := scenes.SearchScene
-
-	var ret []*scraper.ScrapedScene
-	var ignoredTags []string
-	for _, s := range sceneFragments {
-		ss, err := c.sceneFragmentToScrapedScene(ctx, s)
-		if err != nil {
-			return nil, err
-		}
-
-		var thisIgnoredTags []string
-		ss.Tags, thisIgnoredTags = scraper.FilterTags(c.excludeTagRE, ss.Tags)
-		ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
-
-		ret = append(ret, ss)
-	}
-
-	scraper.LogIgnoredTags(ignoredTags)
-
-	return ret, nil
-}
-
-// FindStashBoxScenesByFingerprints queries stash-box for a scene using the
-// scene's MD5/OSHASH checksum, or PHash.
-func (c Client) FindStashBoxSceneByFingerprints(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
-	res, err := c.FindStashBoxScenesByFingerprints(ctx, []int{sceneID})
-	if len(res) > 0 {
-		return res[0], err
-	}
-	return nil, err
-}
-
-// FindStashBoxScenesByFingerprints queries stash-box for scenes using every
-// scene's MD5/OSHASH checksum, or PHash, and returns results in the same order
-// as the input slice.
-func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, ids []int) ([][]*scraper.ScrapedScene, error) {
-	var fingerprints [][]*graphql.FingerprintQueryInput
-
-	r := c.repository
-	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
-		qb := r.Scene
-
-		for _, sceneID := range ids {
-			scene, err := qb.Find(ctx, sceneID)
-			if err != nil {
-				return err
-			}
-
-			if scene == nil {
-				return fmt.Errorf("scene with id %d not found", sceneID)
-			}
-
-			if err := scene.LoadFiles(ctx, r.Scene); err != nil {
-				return err
-			}
-
-			var sceneFPs []*graphql.FingerprintQueryInput
-
-			for _, f := range scene.Files.List() {
-				checksum := f.Fingerprints.GetString(models.FingerprintTypeMD5)
-				if checksum != "" {
-					sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
-						Hash:      checksum,
-						Algorithm: graphql.FingerprintAlgorithmMd5,
-					})
-				}
-
-				oshash := f.Fingerprints.GetString(models.FingerprintTypeOshash)
-				if oshash != "" {
-					sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
-						Hash:      oshash,
-						Algorithm: graphql.FingerprintAlgorithmOshash,
-					})
-				}
-
-				phash := f.Fingerprints.GetInt64(models.FingerprintTypePhash)
-				if phash != 0 {
-					phashStr := utils.PhashToString(phash)
-					sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
-						Hash:      phashStr,
-						Algorithm: graphql.FingerprintAlgorithmPhash,
-					})
-				}
-			}
-
-			fingerprints = append(fingerprints, sceneFPs)
-		}
-
-		return nil
-	}); err != nil {
-		return nil, err
-	}
-
-	return c.findStashBoxScenesByFingerprints(ctx, fingerprints)
-}
-
-func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, scenes [][]*graphql.FingerprintQueryInput) ([][]*scraper.ScrapedScene, error) {
-	var results [][]*scraper.ScrapedScene
-
-	// filter out nils
-	var validScenes [][]*graphql.FingerprintQueryInput
-	for _, s := range scenes {
-		if len(s) > 0 {
-			validScenes = append(validScenes, s)
-		}
-	}
-
-	var ignoredTags []string
-
-	for i := 0; i < len(validScenes); i += 40 {
-		end := i + 40
-		if end > len(validScenes) {
-			end = len(validScenes)
-		}
-		scenes, err := c.client.FindScenesBySceneFingerprints(ctx, validScenes[i:end])
-
-		if err != nil {
-			return nil, err
-		}
-
-		for _, sceneFragments := range scenes.FindScenesBySceneFingerprints {
-			var sceneResults []*scraper.ScrapedScene
-			for _, scene := range sceneFragments {
-				ss, err := c.sceneFragmentToScrapedScene(ctx, scene)
-				if err != nil {
-					return nil, err
-				}
-
-				var thisIgnoredTags []string
-				ss.Tags, thisIgnoredTags = scraper.FilterTags(c.excludeTagRE, ss.Tags)
-				ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
-
-				sceneResults = append(sceneResults, ss)
-			}
-			results = append(results, sceneResults)
-		}
-	}
-
-	scraper.LogIgnoredTags(ignoredTags)
-
-	// repopulate the results to be the same order as the input
-	ret := make([][]*scraper.ScrapedScene, len(scenes))
-	upTo := 0
-
-	for i, v := range scenes {
-		if len(v) > 0 {
-			ret[i] = results[upTo]
-			upTo++
-		}
-	}
-
-	return ret, nil
-}
-
-func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.SceneFragment) (*scraper.ScrapedScene, error) {
-	stashID := s.ID
-
-	ss := &scraper.ScrapedScene{
-		Title:        s.Title,
-		Code:         s.Code,
-		Date:         s.Date,
-		Details:      s.Details,
-		Director:     s.Director,
-		URL:          findURL(s.Urls, "STUDIO"),
-		Duration:     s.Duration,
-		RemoteSiteID: &stashID,
-		Fingerprints: getFingerprints(s),
-		// Image
-		// stash_id
-	}
-
-	for _, u := range s.Urls {
-		ss.URLs = append(ss.URLs, u.URL)
-	}
-
-	if len(ss.URLs) > 0 {
-		ss.URL = &ss.URLs[0]
-	}
-
-	if len(s.Images) > 0 {
-		// TODO - #454 code sorts images by aspect ratio according to a wanted
-		// orientation. I'm just grabbing the first for now
-		ss.Image = getFirstImage(ctx, c.getHTTPClient(), s.Images)
-	}
-
-	if ss.URL == nil && len(s.Urls) > 0 {
-		// The scene in Stash-box may not have a Studio URL but it does have another URL.
-		// For example it has a www.manyvids.com URL, which is auto set as type ManyVids.
-		// This should be re-visited once Stashapp can support more than one URL.
-		ss.URL = &s.Urls[0].URL
-	}
-
-	r := c.repository
-	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
-		pqb := r.Performer
-		tqb := r.Tag
-
-		if s.Studio != nil {
-			ss.Studio = studioFragmentToScrapedStudio(*s.Studio)
-
-			err := match.ScrapedStudio(ctx, r.Studio, ss.Studio, &c.box.Endpoint)
-			if err != nil {
-				return err
-			}
-
-			var parentStudio *graphql.FindStudio
-			if s.Studio.Parent != nil {
-				parentStudio, err = c.client.FindStudio(ctx, &s.Studio.Parent.ID, nil)
-				if err != nil {
-					return err
-				}
-
-				if parentStudio.FindStudio != nil {
-					ss.Studio.Parent = studioFragmentToScrapedStudio(*parentStudio.FindStudio)
-
-					err = match.ScrapedStudio(ctx, r.Studio, ss.Studio.Parent, &c.box.Endpoint)
-					if err != nil {
-						return err
-					}
-				}
-			}
-		}
-
-		for _, p := range s.Performers {
-			sp := performerFragmentToScrapedPerformer(*p.Performer)
-
-			err := match.ScrapedPerformer(ctx, pqb, sp, &c.box.Endpoint)
-			if err != nil {
-				return err
-			}
-
-			ss.Performers = append(ss.Performers, sp)
-		}
-
-		for _, t := range s.Tags {
-			st := &models.ScrapedTag{
-				Name: t.Name,
-			}
-
-			err := match.ScrapedTag(ctx, tqb, st)
-			if err != nil {
-				return err
-			}
-
-			ss.Tags = append(ss.Tags, st)
-		}
-
-		return nil
-	}); err != nil {
-		return nil, err
-	}
-
-	return ss, nil
-}
-
-func getFirstImage(ctx context.Context, client *http.Client, images []*graphql.ImageFragment) *string {
-	ret, err := fetchImage(ctx, client, images[0].URL)
-	if err != nil && !errors.Is(err, context.Canceled) {
-		logger.Warnf("Error fetching image %s: %s", images[0].URL, err.Error())
-	}
-
-	return ret
-}
-
-func getFingerprints(scene *graphql.SceneFragment) []*models.StashBoxFingerprint {
-	fingerprints := []*models.StashBoxFingerprint{}
-	for _, fp := range scene.Fingerprints {
-		fingerprint := models.StashBoxFingerprint{
-			Algorithm: fp.Algorithm.String(),
-			Hash:      fp.Hash,
-			Duration:  fp.Duration,
-		}
-		fingerprints = append(fingerprints, &fingerprint)
-	}
-	return fingerprints
-}
-
-func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, cover []byte) (*string, error) {
-	draft := graphql.SceneDraftInput{}
-	var image io.Reader
-	r := c.repository
-	pqb := r.Performer
-	sqb := r.Studio
-	endpoint := c.box.Endpoint
-
-	if scene.Title != "" {
-		draft.Title = &scene.Title
-	}
-	if scene.Code != "" {
-		draft.Code = &scene.Code
-	}
-	if scene.Details != "" {
-		draft.Details = &scene.Details
-	}
-	if scene.Director != "" {
-		draft.Director = &scene.Director
-	}
-	// TODO - draft does not accept multiple URLs. Use single URL for now.
-	if len(scene.URLs.List()) > 0 {
-		url := strings.TrimSpace(scene.URLs.List()[0])
-		draft.URL = &url
-	}
-	if scene.Date != nil {
-		v := scene.Date.String()
-		draft.Date = &v
-	}
-
-	if scene.StudioID != nil {
-		studio, err := sqb.Find(ctx, *scene.StudioID)
-		if err != nil {
-			return nil, err
-		}
-		if studio == nil {
-			return nil, fmt.Errorf("studio with id %d not found", *scene.StudioID)
-		}
-
-		studioDraft := graphql.DraftEntityInput{
-			Name: studio.Name,
-		}
-
-		stashIDs, err := sqb.GetStashIDs(ctx, studio.ID)
-		if err != nil {
-			return nil, err
-		}
-		for _, stashID := range stashIDs {
-			c := stashID
-			if stashID.Endpoint == endpoint {
-				studioDraft.ID = &c.StashID
-				break
-			}
-		}
-		draft.Studio = &studioDraft
-	}
-
-	fingerprints := []*graphql.FingerprintInput{}
-
-	// submit all file fingerprints
-	if err := scene.LoadFiles(ctx, r.Scene); err != nil {
-		return nil, err
-	}
-
-	for _, f := range scene.Files.List() {
-		duration := f.Duration
-
-		if duration != 0 {
-			if oshash := f.Fingerprints.GetString(models.FingerprintTypeOshash); oshash != "" {
-				fingerprint := graphql.FingerprintInput{
-					Hash:      oshash,
-					Algorithm: graphql.FingerprintAlgorithmOshash,
-					Duration:  int(duration),
-				}
-				fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
-			}
-
-			if checksum := f.Fingerprints.GetString(models.FingerprintTypeMD5); checksum != "" {
-				fingerprint := graphql.FingerprintInput{
-					Hash:      checksum,
-					Algorithm: graphql.FingerprintAlgorithmMd5,
-					Duration:  int(duration),
-				}
-				fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
-			}
-
-			if phash := f.Fingerprints.GetInt64(models.FingerprintTypePhash); phash != 0 {
-				fingerprint := graphql.FingerprintInput{
-					Hash:      utils.PhashToString(phash),
-					Algorithm: graphql.FingerprintAlgorithmPhash,
-					Duration:  int(duration),
-				}
-				fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
-			}
-		}
-	}
-	draft.Fingerprints = fingerprints
-
-	scenePerformers, err := pqb.FindBySceneID(ctx, scene.ID)
-	if err != nil {
-		return nil, err
-	}
-
-	performers := []*graphql.DraftEntityInput{}
-	for _, p := range scenePerformers {
-		performerDraft := graphql.DraftEntityInput{
-			Name: p.Name,
-		}
-
-		stashIDs, err := pqb.GetStashIDs(ctx, p.ID)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, stashID := range stashIDs {
-			c := stashID
-			if stashID.Endpoint == endpoint {
-				performerDraft.ID = &c.StashID
-				break
-			}
-		}
-
-		performers = append(performers, &performerDraft)
-	}
-	draft.Performers = performers
-
-	var tags []*graphql.DraftEntityInput
-	sceneTags, err := r.Tag.FindBySceneID(ctx, scene.ID)
-	if err != nil {
-		return nil, err
-	}
-	for _, tag := range sceneTags {
-		tags = append(tags, &graphql.DraftEntityInput{Name: tag.Name})
-	}
-	draft.Tags = tags
-
-	if len(cover) > 0 {
-		image = bytes.NewReader(cover)
-	}
-
-	if err := scene.LoadStashIDs(ctx, r.Scene); err != nil {
-		return nil, err
-	}
-
-	stashIDs := scene.StashIDs.List()
-	var stashID *string
-	for _, v := range stashIDs {
-		if v.Endpoint == endpoint {
-			vv := v.StashID
-			stashID = &vv
-			break
-		}
-	}
-	draft.ID = stashID
-
-	var id *string
-	var ret graphql.SubmitSceneDraft
-	err = c.submitDraft(ctx, graphql.SubmitSceneDraftDocument, draft, image, &ret)
-	id = ret.SubmitSceneDraft.ID
-
-	return id, err
-
-	// ret, err := c.client.SubmitSceneDraft(ctx, draft, uploadImage(image))
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	// id := ret.SubmitSceneDraft.ID
-	// return id, nil
-}
-
-func (c Client) SubmitStashBoxFingerprints(ctx context.Context, sceneIDs []string) (bool, error) {
-	ids, err := stringslice.StringSliceToIntSlice(sceneIDs)
-	if err != nil {
-		return false, err
-	}
-
-	endpoint := c.box.Endpoint
-
-	var fingerprints []graphql.FingerprintSubmission
-
-	r := c.repository
-	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
-		qb := r.Scene
-
-		for _, sceneID := range ids {
-			scene, err := qb.Find(ctx, sceneID)
-			if err != nil {
-				return err
-			}
-
-			if scene == nil {
-				continue
-			}
-
-			if err := scene.LoadStashIDs(ctx, qb); err != nil {
-				return err
-			}
-
-			if err := scene.LoadFiles(ctx, qb); err != nil {
-				return err
-			}
-
-			stashIDs := scene.StashIDs.List()
-			sceneStashID := ""
-			for _, stashID := range stashIDs {
-				if stashID.Endpoint == endpoint {
-					sceneStashID = stashID.StashID
-				}
-			}
-
-			if sceneStashID != "" {
-				for _, f := range scene.Files.List() {
-					duration := f.Duration
-
-					if duration != 0 {
-						if checksum := f.Fingerprints.GetString(models.FingerprintTypeMD5); checksum != "" {
-							fingerprint := graphql.FingerprintInput{
-								Hash:      checksum,
-								Algorithm: graphql.FingerprintAlgorithmMd5,
-								Duration:  int(duration),
-							}
-							fingerprints = append(fingerprints, graphql.FingerprintSubmission{
-								SceneID:     sceneStashID,
-								Fingerprint: &fingerprint,
-							})
-						}
-
-						if oshash := f.Fingerprints.GetString(models.FingerprintTypeOshash); oshash != "" {
-							fingerprint := graphql.FingerprintInput{
-								Hash:      oshash,
-								Algorithm: graphql.FingerprintAlgorithmOshash,
-								Duration:  int(duration),
-							}
-							fingerprints = append(fingerprints, graphql.FingerprintSubmission{
-								SceneID:     sceneStashID,
-								Fingerprint: &fingerprint,
-							})
-						}
-
-						if phash := f.Fingerprints.GetInt64(models.FingerprintTypePhash); phash != 0 {
-							fingerprint := graphql.FingerprintInput{
-								Hash:      utils.PhashToString(phash),
-								Algorithm: graphql.FingerprintAlgorithmPhash,
-								Duration:  int(duration),
-							}
-							fingerprints = append(fingerprints, graphql.FingerprintSubmission{
-								SceneID:     sceneStashID,
-								Fingerprint: &fingerprint,
-							})
-						}
-					}
-				}
-			}
-		}
-
-		return nil
-	}); err != nil {
-		return false, err
-	}
-
-	return c.submitStashBoxFingerprints(ctx, fingerprints)
-}
-
-func (c Client) submitStashBoxFingerprints(ctx context.Context, fingerprints []graphql.FingerprintSubmission) (bool, error) {
-	for _, fingerprint := range fingerprints {
-		_, err := c.client.SubmitFingerprint(ctx, fingerprint)
-		if err != nil {
-			return false, err
-		}
-	}
-
-	return true, nil
-}
-
-func appendFingerprintUnique(v []*graphql.FingerprintInput, toAdd *graphql.FingerprintInput) []*graphql.FingerprintInput {
-	for _, vv := range v {
-		if vv.Algorithm == toAdd.Algorithm && vv.Hash == toAdd.Hash {
-			return v
-		}
-	}
-
-	return append(v, toAdd)
-}

pkg/scraper/tag.go

@@ -32,6 +32,8 @@ func FilterTags(excludeRegexps []*regexp.Regexp, tags []*models.ScrapedTag) (new
 		return tags, nil
 	}
 
+	newTags = make([]*models.ScrapedTag, 0, len(tags))
+
 	for _, t := range tags {
 		ignore := false
 		for _, reg := range excludeRegexps {

pkg/scraper/xpath.go

@@ -151,7 +151,7 @@ func (s *xpathScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
 	return nil, ErrNotSupported
 }
 
-func (s *xpathScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
+func (s *xpathScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromScene(scene)
 	if s.scraper.QueryURLReplacements != nil {
@@ -210,7 +210,7 @@ func (s *xpathScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap
 	return scraper.scrapeScene(ctx, q)
 }
 
-func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
+func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromGallery(gallery)
 	if s.scraper.QueryURLReplacements != nil {
@@ -234,7 +234,7 @@ func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
 	return scraper.scrapeGallery(ctx, q)
 }
 
-func (s *xpathScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
+func (s *xpathScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
 	// construct the URL
 	queryURL := queryURLParametersFromImage(image)
 	if s.scraper.QueryURLReplacements != nil {

pkg/sliceutil/collections.go

@@ -167,3 +167,13 @@ func ValuesToPtrs[T any](vs []T) []*T {
 	}
 	return ret
 }
+
+// Flatten returns a single slice containing all elements of the provided
+// slice of slices.
+func Flatten[T any](vs [][]T) []T {
+	var ret []T
+	for _, v := range vs {
+		ret = append(ret, v...)
+	}
+	return ret
+}

pkg/sqlite/scene.go

@@ -493,8 +493,11 @@ func (qb *SceneStore) Find(ctx context.Context, id int) (*models.Scene, error) {
 	return ret, err
 }
 
-func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene, error) {
-	scenes := make([]*models.Scene, len(ids))
+// FindByIDs finds multiple scenes by their IDs.
+// No check is made to see if the scenes exist, and the order of the returned scenes
+// is not guaranteed to be the same as the order of the input IDs.
+func (qb *SceneStore) FindByIDs(ctx context.Context, ids []int) ([]*models.Scene, error) {
+	scenes := make([]*models.Scene, 0, len(ids))
 
 	table := qb.table()
 	if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
@@ -504,16 +507,29 @@ func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene,
 			return err
 		}
 
-		for _, s := range unsorted {
-			i := slices.Index(ids, s.ID)
-			scenes[i] = s
-		}
+		scenes = append(scenes, unsorted...)
 
 		return nil
 	}); err != nil {
 		return nil, err
 	}
 
+	return scenes, nil
+}
+
+func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene, error) {
+	scenes := make([]*models.Scene, len(ids))
+
+	unsorted, err := qb.FindByIDs(ctx, ids)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, s := range unsorted {
+		i := slices.Index(ids, s.ID)
+		scenes[i] = s
+	}
+
 	for i := range scenes {
 		if scenes[i] == nil {
 			return nil, fmt.Errorf("scene with id %d not found", ids[i])

pkg/sqlite/studio.go

@@ -586,6 +586,7 @@ var studioSortOptions = sortOptions{
 	"scenes_count",
 	"random",
 	"rating",
+	"tag_count",
 	"updated_at",
 }
 

pkg/sqlite/tag.go

@@ -562,7 +562,7 @@ func (qb *TagStore) All(ctx context.Context) ([]*models.Tag, error) {
 	table := qb.table()
 
 	return qb.getMany(ctx, qb.selectDataset().Order(
-		table.Col("name").Asc(),
+		goqu.L("COALESCE(tags.sort_name, tags.name) COLLATE NATURAL_CI").Asc(),
 		table.Col(idColumn).Asc(),
 	))
 }

pkg/stashbox/client.go

@@ -0,0 +1,101 @@
+// Package stashbox provides a client interface to a stash-box server instance.
+package stashbox
+
+import (
+	"context"
+	"net/http"
+	"regexp"
+
+	"github.com/Yamashou/gqlgenc/clientv2"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/scraper"
+	"github.com/stashapp/stash/pkg/stashbox/graphql"
+
+	"golang.org/x/time/rate"
+)
+
+// DefaultMaxRequestsPerMinute is the default maximum number of requests per minute.
+const DefaultMaxRequestsPerMinute = 240
+
+// Client represents the client interface to a stash-box server instance.
+type Client struct {
+	client *graphql.Client
+	box    models.StashBox
+
+	maxRequestsPerMinute int
+
+	// tag patterns to be excluded
+	excludeTagRE []*regexp.Regexp
+}
+
+type ClientOption func(*Client)
+
+func ExcludeTagPatterns(patterns []string) ClientOption {
+	return func(c *Client) {
+		c.excludeTagRE = scraper.CompileExclusionRegexps(patterns)
+	}
+}
+
+func MaxRequestsPerMinute(n int) ClientOption {
+	return func(c *Client) {
+		if n > 0 {
+			c.maxRequestsPerMinute = n
+		}
+	}
+}
+
+func setApiKeyHeader(apiKey string) clientv2.RequestInterceptor {
+	return func(ctx context.Context, req *http.Request, gqlInfo *clientv2.GQLRequestInfo, res interface{}, next clientv2.RequestInterceptorFunc) error {
+		req.Header.Set("ApiKey", apiKey)
+		return next(ctx, req, gqlInfo, res)
+	}
+}
+
+func rateLimit(n int) clientv2.RequestInterceptor {
+	perSec := float64(n) / 60
+	limiter := rate.NewLimiter(rate.Limit(perSec), 1)
+
+	return func(ctx context.Context, req *http.Request, gqlInfo *clientv2.GQLRequestInfo, res interface{}, next clientv2.RequestInterceptorFunc) error {
+		if err := limiter.Wait(ctx); err != nil {
+			// should only happen if the context is canceled
+			return err
+		}
+
+		return next(ctx, req, gqlInfo, res)
+	}
+}
+
+// NewClient returns a new instance of a stash-box client.
+func NewClient(box models.StashBox, options ...ClientOption) *Client {
+	ret := &Client{
+		box:                  box,
+		maxRequestsPerMinute: DefaultMaxRequestsPerMinute,
+	}
+
+	if box.MaxRequestsPerMinute > 0 {
+		ret.maxRequestsPerMinute = box.MaxRequestsPerMinute
+	}
+
+	for _, option := range options {
+		option(ret)
+	}
+
+	authHeader := setApiKeyHeader(box.APIKey)
+	limitRequests := rateLimit(ret.maxRequestsPerMinute)
+
+	client := &graphql.Client{
+		Client: clientv2.NewClient(http.DefaultClient, box.Endpoint, nil, authHeader, limitRequests),
+	}
+
+	ret.client = client
+
+	return ret
+}
+
+func (c Client) getHTTPClient() *http.Client {
+	return c.client.Client.Client
+}
+
+func (c Client) GetUser(ctx context.Context) (*graphql.Me, error) {
+	return c.client.Me(ctx)
+}

pkg/stashbox/graphql/generated_models.go

@@ -1018,6 +1018,7 @@ type StashBoxConfig struct {
 	GuidelinesURL              string `json:"guidelines_url"`
 	RequireSceneDraft          bool   `json:"require_scene_draft"`
 	EditUpdateLimit            int    `json:"edit_update_limit"`
+	RequireTagRole             bool   `json:"require_tag_role"`
 }
 
 type StringCriterionInput struct {
@@ -2143,6 +2144,8 @@ const (
 	// May grant and rescind invite tokens and resind invite keys
 	RoleEnumManageInvites RoleEnum = "MANAGE_INVITES"
 	RoleEnumBot           RoleEnum = "BOT"
+	RoleEnumReadOnly      RoleEnum = "READ_ONLY"
+	RoleEnumEditTags      RoleEnum = "EDIT_TAGS"
 )
 
 var AllRoleEnum = []RoleEnum{
@@ -2154,11 +2157,13 @@ var AllRoleEnum = []RoleEnum{
 	RoleEnumInvite,
 	RoleEnumManageInvites,
 	RoleEnumBot,
+	RoleEnumReadOnly,
+	RoleEnumEditTags,
 }
 
 func (e RoleEnum) IsValid() bool {
 	switch e {
-	case RoleEnumRead, RoleEnumVote, RoleEnumEdit, RoleEnumModify, RoleEnumAdmin, RoleEnumInvite, RoleEnumManageInvites, RoleEnumBot:
+	case RoleEnumRead, RoleEnumVote, RoleEnumEdit, RoleEnumModify, RoleEnumAdmin, RoleEnumInvite, RoleEnumManageInvites, RoleEnumBot, RoleEnumReadOnly, RoleEnumEditTags:
 		return true
 	}
 	return false

pkg/stashbox/performer.go

@@ -9,39 +9,31 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scraper"
-	"github.com/stashapp/stash/pkg/scraper/stashbox/graphql"
 	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
+	"github.com/stashapp/stash/pkg/stashbox/graphql"
 	"github.com/stashapp/stash/pkg/utils"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 )
 
-// QueryStashBoxPerformer queries stash-box for performers using a query string.
-func (c Client) QueryStashBoxPerformer(ctx context.Context, queryStr string) ([]*StashBoxPerformerQueryResult, error) {
-	performers, err := c.queryStashBoxPerformer(ctx, queryStr)
-
-	res := []*StashBoxPerformerQueryResult{
-		{
-			Query:   queryStr,
-			Results: performers,
-		},
-	}
+// QueryPerformer queries stash-box for performers using a query string.
+func (c Client) QueryPerformer(ctx context.Context, queryStr string) ([]*models.ScrapedPerformer, error) {
+	performers, err := c.queryPerformer(ctx, queryStr)
 
 	// set the deprecated image field
-	for _, p := range res[0].Results {
+	for _, p := range performers {
 		if len(p.Images) > 0 {
 			p.Image = &p.Images[0]
 		}
 	}
 
-	return res, err
+	return performers, err
 }
 
-func (c Client) queryStashBoxPerformer(ctx context.Context, queryStr string) ([]*models.ScrapedPerformer, error) {
+func (c Client) queryPerformer(ctx context.Context, queryStr string) ([]*models.ScrapedPerformer, error) {
 	performers, err := c.client.SearchPerformer(ctx, queryStr)
 	if err != nil {
 		return nil, err
@@ -67,101 +59,18 @@ func (c Client) queryStashBoxPerformer(ctx context.Context, queryStr string) ([]
 	return ret, nil
 }
 
-// FindStashBoxPerformersByNames queries stash-box for performers by name
-func (c Client) FindStashBoxPerformersByNames(ctx context.Context, performerIDs []string) ([]*StashBoxPerformerQueryResult, error) {
-	ids, err := stringslice.StringSliceToIntSlice(performerIDs)
-	if err != nil {
-		return nil, err
-	}
-
-	var performers []*models.Performer
-	r := c.repository
-	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
-		qb := r.Performer
-
-		for _, performerID := range ids {
-			performer, err := qb.Find(ctx, performerID)
-			if err != nil {
-				return err
-			}
-
-			if performer == nil {
-				return fmt.Errorf("performer with id %d not found", performerID)
-			}
-
-			if performer.Name != "" {
-				performers = append(performers, performer)
-			}
+// QueryPerformers queries stash-box for performers using a list of names.
+func (c Client) QueryPerformers(ctx context.Context, names []string) ([][]*models.ScrapedPerformer, error) {
+	ret := make([][]*models.ScrapedPerformer, len(names))
+	for i, name := range names {
+		if name != "" {
+			continue
 		}
 
-		return nil
-	}); err != nil {
-		return nil, err
-	}
-
-	return c.findStashBoxPerformersByNames(ctx, performers)
-}
-
-func (c Client) FindStashBoxPerformersByPerformerNames(ctx context.Context, performerIDs []string) ([][]*models.ScrapedPerformer, error) {
-	ids, err := stringslice.StringSliceToIntSlice(performerIDs)
-	if err != nil {
-		return nil, err
-	}
-
-	var performers []*models.Performer
-
-	r := c.repository
-	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
-		qb := r.Performer
-
-		for _, performerID := range ids {
-			performer, err := qb.Find(ctx, performerID)
-			if err != nil {
-				return err
-			}
-
-			if performer == nil {
-				return fmt.Errorf("performer with id %d not found", performerID)
-			}
-
-			if performer.Name != "" {
-				performers = append(performers, performer)
-			}
-		}
-
-		return nil
-	}); err != nil {
-		return nil, err
-	}
-
-	results, err := c.findStashBoxPerformersByNames(ctx, performers)
-	if err != nil {
-		return nil, err
-	}
-
-	var ret [][]*models.ScrapedPerformer
-	for _, r := range results {
-		ret = append(ret, r.Results)
-	}
-
-	return ret, nil
-}
-
-func (c Client) findStashBoxPerformersByNames(ctx context.Context, performers []*models.Performer) ([]*StashBoxPerformerQueryResult, error) {
-	var ret []*StashBoxPerformerQueryResult
-	for _, performer := range performers {
-		if performer.Name != "" {
-			performerResults, err := c.queryStashBoxPerformer(ctx, performer.Name)
-			if err != nil {
-				return nil, err
-			}
-
-			result := StashBoxPerformerQueryResult{
-				Query:   strconv.Itoa(performer.ID),
-				Results: performerResults,
-			}
-
-			ret = append(ret, &result)
+		var err error
+		ret[i], err = c.queryPerformer(ctx, name)
+		if err != nil {
+			return nil, err
 		}
 	}
 
@@ -388,7 +297,8 @@ func padFuzzyDate(date *string) *string {
 	return &paddedDate
 }
 
-func (c Client) FindStashBoxPerformerByID(ctx context.Context, id string) (*models.ScrapedPerformer, error) {
+// FindPerformerByID queries stash-box for a performer by ID.
+func (c Client) FindPerformerByID(ctx context.Context, id string) (*models.ScrapedPerformer, error) {
 	performer, err := c.client.FindPerformerByID(ctx, id)
 	if err != nil {
 		return nil, err
@@ -400,18 +310,12 @@ func (c Client) FindStashBoxPerformerByID(ctx context.Context, id string) (*mode
 
 	ret := performerFragmentToScrapedPerformer(*performer.FindPerformer)
 
-	r := c.repository
-	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
-		err := match.ScrapedPerformer(ctx, r.Performer, ret, &c.box.Endpoint)
-		return err
-	}); err != nil {
-		return nil, err
-	}
-
 	return ret, nil
 }
 
-func (c Client) FindStashBoxPerformerByName(ctx context.Context, name string) (*models.ScrapedPerformer, error) {
+// FindPerformerByName queries stash-box for a performer by name.
+// Unlike QueryPerformer, this function will only return a performer if the name matches exactly.
+func (c Client) FindPerformerByName(ctx context.Context, name string) (*models.ScrapedPerformer, error) {
 	performers, err := c.client.SearchPerformer(ctx, name)
 	if err != nil {
 		return nil, err
@@ -424,41 +328,17 @@ func (c Client) FindStashBoxPerformerByName(ctx context.Context, name string) (*
 		}
 	}
 
-	if ret == nil {
-		return nil, nil
-	}
-
-	r := c.repository
-	if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
-		err := match.ScrapedPerformer(ctx, r.Performer, ret, &c.box.Endpoint)
-		return err
-	}); err != nil {
-		return nil, err
-	}
-
 	return ret, nil
 }
 
-func (c Client) SubmitPerformerDraft(ctx context.Context, performer *models.Performer) (*string, error) {
+// SubmitPerformerDraft submits a performer draft to stash-box.
+// The performer parameter must have aliases, URLs and stash IDs loaded.
+func (c Client) SubmitPerformerDraft(ctx context.Context, performer *models.Performer, img []byte) (*string, error) {
 	draft := graphql.PerformerDraftInput{}
 	var image io.Reader
-	pqb := c.repository.Performer
 	endpoint := c.box.Endpoint
 
-	if err := performer.LoadAliases(ctx, pqb); err != nil {
-		return nil, err
-	}
-
-	if err := performer.LoadURLs(ctx, pqb); err != nil {
-		return nil, err
-	}
-
-	if err := performer.LoadStashIDs(ctx, pqb); err != nil {
-		return nil, err
-	}
-
-	img, _ := pqb.GetImage(ctx, performer.ID)
-	if img != nil {
+	if len(img) > 0 {
 		image = bytes.NewReader(img)
 	}
 
@@ -524,12 +404,8 @@ func (c Client) SubmitPerformerDraft(ctx context.Context, performer *models.Perf
 		draft.Urls = performer.URLs.List()
 	}
 
-	stashIDs, err := pqb.GetStashIDs(ctx, performer.ID)
-	if err != nil {
-		return nil, err
-	}
 	var stashID *string
-	for _, v := range stashIDs {
+	for _, v := range performer.StashIDs.List() {
 		c := v
 		if v.Endpoint == endpoint {
 			stashID = &c.StashID
@@ -540,7 +416,7 @@ func (c Client) SubmitPerformerDraft(ctx context.Context, performer *models.Perf
 
 	var id *string
 	var ret graphql.SubmitPerformerDraft
-	err = c.submitDraft(ctx, graphql.SubmitPerformerDraftDocument, draft, image, &ret)
+	err := c.submitDraft(ctx, graphql.SubmitPerformerDraftDocument, draft, image, &ret)
 	id = ret.SubmitPerformerDraft.ID
 
 	return id, err

pkg/stashbox/scene.go

@@ -0,0 +1,468 @@
+package stashbox
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/scraper"
+	"github.com/stashapp/stash/pkg/sliceutil"
+	"github.com/stashapp/stash/pkg/stashbox/graphql"
+	"github.com/stashapp/stash/pkg/utils"
+)
+
+// QueryScene queries stash-box for scenes using a query string.
+func (c Client) QueryScene(ctx context.Context, queryStr string) ([]*models.ScrapedScene, error) {
+	scenes, err := c.client.SearchScene(ctx, queryStr)
+	if err != nil {
+		return nil, err
+	}
+
+	sceneFragments := scenes.SearchScene
+
+	var ret []*models.ScrapedScene
+	var ignoredTags []string
+	for _, s := range sceneFragments {
+		ss, err := c.sceneFragmentToScrapedScene(ctx, s)
+		if err != nil {
+			return nil, err
+		}
+
+		var thisIgnoredTags []string
+		ss.Tags, thisIgnoredTags = scraper.FilterTags(c.excludeTagRE, ss.Tags)
+		ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
+
+		ret = append(ret, ss)
+	}
+
+	scraper.LogIgnoredTags(ignoredTags)
+
+	return ret, nil
+}
+
+// FindStashBoxScenesByFingerprints queries stash-box for a scene using the
+// scene's MD5/OSHASH checksum, or PHash.
+func (c Client) FindSceneByFingerprints(ctx context.Context, fps models.Fingerprints) ([]*models.ScrapedScene, error) {
+	res, err := c.FindScenesByFingerprints(ctx, []models.Fingerprints{fps})
+	if len(res) > 0 {
+		return res[0], err
+	}
+	return nil, err
+}
+
+// FindScenesByFingerprints queries stash-box for scenes using every
+// scene's MD5/OSHASH checksum, or PHash, and returns results in the same order
+// as the input slice.
+func (c Client) FindScenesByFingerprints(ctx context.Context, fps []models.Fingerprints) ([][]*models.ScrapedScene, error) {
+	var fingerprints [][]*graphql.FingerprintQueryInput
+
+	for _, fp := range fps {
+		fingerprints = append(fingerprints, convertFingerprints(fp))
+	}
+
+	return c.findScenesByFingerprints(ctx, fingerprints)
+}
+
+func convertFingerprints(fps models.Fingerprints) []*graphql.FingerprintQueryInput {
+	var ret []*graphql.FingerprintQueryInput
+
+	for _, f := range fps {
+		var i = &graphql.FingerprintQueryInput{}
+		switch f.Type {
+		case models.FingerprintTypeMD5:
+			i.Algorithm = graphql.FingerprintAlgorithmMd5
+			i.Hash = f.String()
+		case models.FingerprintTypeOshash:
+			i.Algorithm = graphql.FingerprintAlgorithmOshash
+			i.Hash = f.String()
+		case models.FingerprintTypePhash:
+			i.Algorithm = graphql.FingerprintAlgorithmPhash
+			i.Hash = utils.PhashToString(f.Int64())
+		default:
+			continue
+		}
+
+		if !i.Algorithm.IsValid() {
+			continue
+		}
+
+		ret = append(ret, i)
+	}
+
+	return ret
+}
+
+func (c Client) findScenesByFingerprints(ctx context.Context, scenes [][]*graphql.FingerprintQueryInput) ([][]*models.ScrapedScene, error) {
+	var results [][]*models.ScrapedScene
+
+	// filter out nils
+	var validScenes [][]*graphql.FingerprintQueryInput
+	for _, s := range scenes {
+		if len(s) > 0 {
+			validScenes = append(validScenes, s)
+		}
+	}
+
+	var ignoredTags []string
+
+	for i := 0; i < len(validScenes); i += 40 {
+		end := i + 40
+		if end > len(validScenes) {
+			end = len(validScenes)
+		}
+		scenes, err := c.client.FindScenesBySceneFingerprints(ctx, validScenes[i:end])
+
+		if err != nil {
+			return nil, err
+		}
+
+		for _, sceneFragments := range scenes.FindScenesBySceneFingerprints {
+			var sceneResults []*models.ScrapedScene
+			for _, scene := range sceneFragments {
+				ss, err := c.sceneFragmentToScrapedScene(ctx, scene)
+				if err != nil {
+					return nil, err
+				}
+
+				var thisIgnoredTags []string
+				ss.Tags, thisIgnoredTags = scraper.FilterTags(c.excludeTagRE, ss.Tags)
+				ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
+
+				sceneResults = append(sceneResults, ss)
+			}
+			results = append(results, sceneResults)
+		}
+	}
+
+	scraper.LogIgnoredTags(ignoredTags)
+
+	// repopulate the results to be the same order as the input
+	ret := make([][]*models.ScrapedScene, len(scenes))
+	upTo := 0
+
+	for i, v := range scenes {
+		if len(v) > 0 {
+			ret[i] = results[upTo]
+			upTo++
+		}
+	}
+
+	return ret, nil
+}
+
+func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.SceneFragment) (*models.ScrapedScene, error) {
+	stashID := s.ID
+
+	ss := &models.ScrapedScene{
+		Title:        s.Title,
+		Code:         s.Code,
+		Date:         s.Date,
+		Details:      s.Details,
+		Director:     s.Director,
+		URL:          findURL(s.Urls, "STUDIO"),
+		Duration:     s.Duration,
+		RemoteSiteID: &stashID,
+		Fingerprints: getFingerprints(s),
+		// Image
+		// stash_id
+	}
+
+	for _, u := range s.Urls {
+		ss.URLs = append(ss.URLs, u.URL)
+	}
+
+	if len(ss.URLs) > 0 {
+		ss.URL = &ss.URLs[0]
+	}
+
+	if len(s.Images) > 0 {
+		// TODO - #454 code sorts images by aspect ratio according to a wanted
+		// orientation. I'm just grabbing the first for now
+		ss.Image = getFirstImage(ctx, c.getHTTPClient(), s.Images)
+	}
+
+	ss.URLs = make([]string, len(s.Urls))
+	for i, u := range s.Urls {
+		ss.URLs[i] = u.URL
+	}
+
+	if s.Studio != nil {
+		var err error
+		ss.Studio, err = c.resolveStudio(ctx, s.Studio)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	for _, p := range s.Performers {
+		sp := performerFragmentToScrapedPerformer(*p.Performer)
+		ss.Performers = append(ss.Performers, sp)
+	}
+
+	for _, t := range s.Tags {
+		st := &models.ScrapedTag{
+			Name: t.Name,
+		}
+		ss.Tags = append(ss.Tags, st)
+	}
+
+	return ss, nil
+}
+
+func getFirstImage(ctx context.Context, client *http.Client, images []*graphql.ImageFragment) *string {
+	ret, err := fetchImage(ctx, client, images[0].URL)
+	if err != nil && !errors.Is(err, context.Canceled) {
+		logger.Warnf("Error fetching image %s: %s", images[0].URL, err.Error())
+	}
+
+	return ret
+}
+
+func getFingerprints(scene *graphql.SceneFragment) []*models.StashBoxFingerprint {
+	fingerprints := []*models.StashBoxFingerprint{}
+	for _, fp := range scene.Fingerprints {
+		fingerprint := models.StashBoxFingerprint{
+			Algorithm: fp.Algorithm.String(),
+			Hash:      fp.Hash,
+			Duration:  fp.Duration,
+		}
+		fingerprints = append(fingerprints, &fingerprint)
+	}
+	return fingerprints
+}
+
+type SceneDraft struct {
+	// Files, URLs, StashIDs must be loaded
+	Scene *models.Scene
+	// StashIDs must be loaded
+	Performers []*models.Performer
+	// StashIDs must be loaded
+	Studio *models.Studio
+	Tags   []*models.Tag
+	Cover  []byte
+}
+
+func (c Client) SubmitSceneDraft(ctx context.Context, d SceneDraft) (*string, error) {
+	draft := newSceneDraftInput(d, c.box.Endpoint)
+	var image io.Reader
+
+	if len(d.Cover) > 0 {
+		image = bytes.NewReader(d.Cover)
+	}
+
+	var id *string
+	var ret graphql.SubmitSceneDraft
+	err := c.submitDraft(ctx, graphql.SubmitSceneDraftDocument, draft, image, &ret)
+	id = ret.SubmitSceneDraft.ID
+
+	return id, err
+
+	// ret, err := c.client.SubmitSceneDraft(ctx, draft, uploadImage(image))
+	// if err != nil {
+	// 	return nil, err
+	// }
+
+	// id := ret.SubmitSceneDraft.ID
+	// return id, nil
+}
+
+func newSceneDraftInput(d SceneDraft, endpoint string) graphql.SceneDraftInput {
+	scene := d.Scene
+
+	draft := graphql.SceneDraftInput{}
+
+	if scene.Title != "" {
+		draft.Title = &scene.Title
+	}
+	if scene.Code != "" {
+		draft.Code = &scene.Code
+	}
+	if scene.Details != "" {
+		draft.Details = &scene.Details
+	}
+	if scene.Director != "" {
+		draft.Director = &scene.Director
+	}
+	// TODO - draft does not accept multiple URLs. Use single URL for now.
+	if len(scene.URLs.List()) > 0 {
+		url := strings.TrimSpace(scene.URLs.List()[0])
+		draft.URL = &url
+	}
+	if scene.Date != nil {
+		v := scene.Date.String()
+		draft.Date = &v
+	}
+
+	if d.Studio != nil {
+		studio := d.Studio
+
+		studioDraft := graphql.DraftEntityInput{
+			Name: studio.Name,
+		}
+
+		stashIDs := studio.StashIDs.List()
+		for _, stashID := range stashIDs {
+			c := stashID
+			if stashID.Endpoint == endpoint {
+				studioDraft.ID = &c.StashID
+				break
+			}
+		}
+		draft.Studio = &studioDraft
+	}
+
+	fingerprints := []*graphql.FingerprintInput{}
+
+	for _, f := range scene.Files.List() {
+		duration := f.Duration
+
+		if duration != 0 {
+			fingerprints = appendFingerprintsUnique(fingerprints, fileFingerprintsToInputGraphQL(f.Fingerprints, int(duration))...)
+		}
+	}
+	draft.Fingerprints = fingerprints
+
+	scenePerformers := d.Performers
+
+	inputPerformers := []*graphql.DraftEntityInput{}
+	for _, p := range scenePerformers {
+		performerDraft := graphql.DraftEntityInput{
+			Name: p.Name,
+		}
+
+		stashIDs := p.StashIDs.List()
+		for _, stashID := range stashIDs {
+			c := stashID
+			if stashID.Endpoint == endpoint {
+				performerDraft.ID = &c.StashID
+				break
+			}
+		}
+
+		inputPerformers = append(inputPerformers, &performerDraft)
+	}
+	draft.Performers = inputPerformers
+
+	var tags []*graphql.DraftEntityInput
+	sceneTags := d.Tags
+	for _, tag := range sceneTags {
+		tags = append(tags, &graphql.DraftEntityInput{Name: tag.Name})
+	}
+	draft.Tags = tags
+
+	stashIDs := scene.StashIDs.List()
+	var stashID *string
+	for _, v := range stashIDs {
+		if v.Endpoint == endpoint {
+			vv := v.StashID
+			stashID = &vv
+			break
+		}
+	}
+	draft.ID = stashID
+
+	return draft
+}
+
+func fileFingerprintsToInputGraphQL(fps models.Fingerprints, duration int) []*graphql.FingerprintInput {
+	var ret []*graphql.FingerprintInput
+
+	for _, f := range fps {
+		var i = &graphql.FingerprintInput{
+			Duration: duration,
+		}
+		switch f.Type {
+		case models.FingerprintTypeMD5:
+			i.Algorithm = graphql.FingerprintAlgorithmMd5
+			i.Hash = f.String()
+		case models.FingerprintTypeOshash:
+			i.Algorithm = graphql.FingerprintAlgorithmOshash
+			i.Hash = f.String()
+		case models.FingerprintTypePhash:
+			i.Algorithm = graphql.FingerprintAlgorithmPhash
+			i.Hash = utils.PhashToString(f.Int64())
+		default:
+			continue
+		}
+
+		if !i.Algorithm.IsValid() {
+			continue
+		}
+
+		ret = appendFingerprintUnique(ret, i)
+	}
+
+	return ret
+}
+
+func (c Client) SubmitFingerprints(ctx context.Context, scenes []*models.Scene) (bool, error) {
+	endpoint := c.box.Endpoint
+
+	var fingerprints []graphql.FingerprintSubmission
+
+	for _, scene := range scenes {
+		stashIDs := scene.StashIDs.List()
+		sceneStashID := ""
+		for _, stashID := range stashIDs {
+			if stashID.Endpoint == endpoint {
+				sceneStashID = stashID.StashID
+			}
+		}
+
+		if sceneStashID == "" {
+			continue
+		}
+
+		for _, f := range scene.Files.List() {
+			duration := f.Duration
+
+			if duration == 0 {
+				continue
+			}
+
+			fps := fileFingerprintsToInputGraphQL(f.Fingerprints, int(duration))
+			for _, fp := range fps {
+				fingerprints = append(fingerprints, graphql.FingerprintSubmission{
+					SceneID:     sceneStashID,
+					Fingerprint: fp,
+				})
+			}
+		}
+	}
+
+	return c.submitFingerprints(ctx, fingerprints)
+}
+
+func (c Client) submitFingerprints(ctx context.Context, fingerprints []graphql.FingerprintSubmission) (bool, error) {
+	for _, fingerprint := range fingerprints {
+		_, err := c.client.SubmitFingerprint(ctx, fingerprint)
+		if err != nil {
+			return false, err
+		}
+	}
+
+	return true, nil
+}
+
+func appendFingerprintUnique(v []*graphql.FingerprintInput, toAdd *graphql.FingerprintInput) []*graphql.FingerprintInput {
+	for _, vv := range v {
+		if vv.Algorithm == toAdd.Algorithm && vv.Hash == toAdd.Hash {
+			return v
+		}
+	}
+
+	return append(v, toAdd)
+}
+
+func appendFingerprintsUnique(v []*graphql.FingerprintInput, toAdd ...*graphql.FingerprintInput) []*graphql.FingerprintInput {
+	for _, a := range toAdd {
+		v = appendFingerprintUnique(v, a)
+	}
+
+	return v
+}

pkg/stashbox/studio.go

@@ -4,12 +4,33 @@ import (
 	"context"
 
 	"github.com/google/uuid"
-	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
-	"github.com/stashapp/stash/pkg/scraper/stashbox/graphql"
+	"github.com/stashapp/stash/pkg/stashbox/graphql"
 )
 
-func (c Client) FindStashBoxStudio(ctx context.Context, query string) (*models.ScrapedStudio, error) {
+func (c Client) resolveStudio(ctx context.Context, s *graphql.StudioFragment) (*models.ScrapedStudio, error) {
+	scraped := studioFragmentToScrapedStudio(*s)
+
+	if s.Parent != nil {
+		parentStudio, err := c.client.FindStudio(ctx, &s.Parent.ID, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		if parentStudio.FindStudio == nil {
+			return scraped, nil
+		}
+
+		scraped.Parent, err = c.resolveStudio(ctx, parentStudio.FindStudio)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return scraped, nil
+}
+
+func (c Client) FindStudio(ctx context.Context, query string) (*models.ScrapedStudio, error) {
 	var studio *graphql.FindStudio
 
 	_, err := uuid.Parse(query)
@@ -27,32 +48,8 @@ func (c Client) FindStashBoxStudio(ctx context.Context, query string) (*models.S
 
 	var ret *models.ScrapedStudio
 	if studio.FindStudio != nil {
-		r := c.repository
-		if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
-			ret = studioFragmentToScrapedStudio(*studio.FindStudio)
-
-			err = match.ScrapedStudio(ctx, r.Studio, ret, &c.box.Endpoint)
-			if err != nil {
-				return err
-			}
-
-			if studio.FindStudio.Parent != nil {
-				parentStudio, err := c.client.FindStudio(ctx, &studio.FindStudio.Parent.ID, nil)
-				if err != nil {
-					return err
-				}
-
-				if parentStudio.FindStudio != nil {
-					ret.Parent = studioFragmentToScrapedStudio(*parentStudio.FindStudio)
-
-					err = match.ScrapedStudio(ctx, r.Studio, ret.Parent, &c.box.Endpoint)
-					if err != nil {
-						return err
-					}
-				}
-			}
-			return nil
-		}); err != nil {
+		ret, err = c.resolveStudio(ctx, studio.FindStudio)
+		if err != nil {
 			return nil, err
 		}
 	}

pkg/tag/export.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/json"
 	"github.com/stashapp/stash/pkg/models/jsonschema"
+	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/utils"
 )
 
@@ -55,6 +56,28 @@ func ToJSON(ctx context.Context, reader FinderAliasImageGetter, tag *models.Tag)
 	return &newTagJSON, nil
 }
 
+// GetDependentTagIDs returns a slice of unique tag IDs that this tag references.
+func GetDependentTagIDs(ctx context.Context, reader FinderAliasImageGetter, tag *models.Tag) ([]int, error) {
+	var ret []int
+
+	parents, err := reader.FindByChildTagID(ctx, tag.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error getting parents: %v", err)
+	}
+
+	for _, tt := range parents {
+		toAdd, err := GetDependentTagIDs(ctx, reader, tt)
+		if err != nil {
+			return nil, fmt.Errorf("error getting dependent tag IDs: %v", err)
+		}
+
+		ret = sliceutil.AppendUniques(ret, toAdd)
+		ret = sliceutil.AppendUnique(ret, tt.ID)
+	}
+
+	return ret, nil
+}
+
 func GetIDs(tags []*models.Tag) []int {
 	var results []int
 	for _, tag := range tags {

scripts/generateLoginLocales.go

@@ -0,0 +1,90 @@
+//go:build ignore
+// +build ignore
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/stashapp/stash/pkg/fsutil"
+	"github.com/stashapp/stash/pkg/utils"
+)
+
+func main() {
+	verbose := len(os.Args) > 1 && os.Args[1] == "-v"
+
+	fmt.Printf("Generating login locales\n")
+
+	// read all json files in the locales directory
+	// and extract only the login part
+
+	// assume running from ui directory
+	dirFS := os.DirFS(filepath.Join("v2.5", "src", "locales"))
+
+	// ensure the login/locales directory exists
+	if err := fsutil.EnsureDir(filepath.Join("login", "locales")); err != nil {
+		panic(err)
+	}
+
+	fs.WalkDir(dirFS, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			panic(err)
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		if filepath.Ext(path) != ".json" {
+			return nil
+		}
+
+		// extract the login part
+		// from the json file
+		src, err := dirFS.Open(path)
+		if err != nil {
+			panic(err)
+		}
+
+		defer src.Close()
+		data, err := io.ReadAll(src)
+		if err != nil {
+			panic(err)
+		}
+
+		m := make(utils.NestedMap)
+		if err := json.Unmarshal(data, &m); err != nil {
+			panic(err)
+		}
+
+		l, found := m.Get("login")
+		if !found {
+			// nothing to do
+			return nil
+		}
+
+		// create new json file
+		// with only the login part
+		if verbose {
+			fmt.Printf("Writing %s\n", d.Name())
+		}
+
+		f, err := os.Create(filepath.Join("login", "locales", d.Name()))
+		if err != nil {
+			panic(err)
+		}
+
+		defer f.Close()
+		e := json.NewEncoder(f)
+		if err := e.Encode(l); err != nil {
+			panic(err)
+		}
+
+		return nil
+	})
+}

ui/login/login.html

@@ -9,32 +9,77 @@
     <link rel="shortcut icon" href="data:,">
     <link rel="stylesheet" href="login/login.css">
     <link rel="stylesheet" href="css">
-</head>
-<body class="login">
 
+    <!-- load locale -->
+    <script>
+        var localeStrings = {
+            username: "Username",
+            password: "Password",
+            login: "Login",
+            invalid_credentials: "Invalid credentials",
+            internal_error: "Unexpected internal error. See logs for more details"
+        };
+    </script>
+    <script src="login/locale"></script>
+</head>
+
+<script>
+    function login() {
+        var username = document.getElementById("username").value;
+        var password = document.getElementById("password").value;
+        var returnURL = document.getElementById("returnURL").value;
+
+        var xhr = new XMLHttpRequest();
+        xhr.open("POST", "login", true);
+        xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");
+        xhr.onreadystatechange = function() {
+            if (xhr.readyState == 4) {
+                if (xhr.status == 200) {
+                    window.location.replace(returnURL);
+                } else {
+                    document.getElementsByClassName("login-error")[0].innerHTML = localeStrings.invalid_credentials;
+                }
+            }
+        };
+        xhr.onerror = function() {
+            document.getElementsByClassName("login-error")[0].innerHTML = localeStrings.internal_error;
+        };
+        xhr.send("username=" + username + "&password=" + password + "&returnURL=" + returnURL);
+    }
+</script>
+
+<body class="login">
     <div class="dialog">
         <div class="card">
-            <form action="login" method="POST">
+            <form action="login" method="POST" onsubmit="event.preventDefault(); login();">
                 <div class="form-group">
-                    <label for="username"><h6>Username</h6></label>
+                    <label for="username"><h6 id="username-heading">Username</h6></label>
                     <input class="text-input form-control" id="username" name="username" type="text" placeholder="Username" />
                 </div>
                 <div class="form-group">
-                    <label for="password"><h6>Password</h6></label>
+                    <label for="password"><h6 id="password-heading">Password</h6></label>
                     <input class="text-input form-control" id="password" name="password" type="password" placeholder="Password" />
                 </div>
                 <div class="login-error">
                     {{.Error}}
                 </div>
 
-                <input type="hidden" name="returnURL" value="{{.URL}}" />
+                <input type="hidden" id="returnURL" name="returnURL" value="{{.URL}}" />
 
                 <div>
-                    <input class="btn btn-primary" type="submit" value="Login">
+                    <input id="login-button" class="btn btn-primary" type="submit" value="Login">
                 </div>
             </form>
         </div>
     </div>
 
 </body>
+
+<script>
+    document.getElementById("username-heading").innerText = localeStrings.username;
+    document.getElementById("password-heading").innerText = localeStrings.password;
+    document.getElementById("username").placeholder = localeStrings.username;
+    document.getElementById("password").placeholder = localeStrings.password;
+    document.getElementById("login-button").value = localeStrings.login;
+</script>
 </html>

ui/ui.go

@@ -1,3 +1,4 @@
+//go:generate go run -tags=dev ../scripts/generateLoginLocales.go
 package ui
 
 import (

ui/v2.5/graphql/data/config.graphql

@@ -49,6 +49,7 @@ fragment ConfigGeneralData on ConfigGeneralResult {
     name
     endpoint
     api_key
+    max_requests_per_minute
   }
   pythonPath
   transcodeInputArgs

ui/v2.5/package.json

@@ -128,7 +128,7 @@
     "terser": "^5.9.0",
     "ts-node": "^10.9.1",
     "typescript": "~4.8.4",
-    "vite": "^4.5.6",
+    "vite": "^4.5.11",
     "vite-plugin-compression": "^0.5.1",
     "vite-tsconfig-paths": "^4.0.5"
   }

ui/v2.5/src/components/Changelog/Changelog.tsx

@@ -68,7 +68,7 @@ const Changelog: React.FC = () => {
   // after new release:
   // add entry to releases, using the current* fields
   // then update the current fields.
-  const currentVersion = stashVersion || "v0.28.0";
+  const currentVersion = stashVersion || "v0.28.1";
   const currentDate = buildDate;
   const currentPage = V0280;
 

ui/v2.5/src/components/Galleries/GalleryCard.tsx

@@ -1,7 +1,7 @@
 import { Button, ButtonGroup, OverlayTrigger, Tooltip } from "react-bootstrap";
-import React, { useEffect, useState } from "react";
+import React, { useState } from "react";
 import * as GQL from "src/core/generated-graphql";
-import { GridCard, calculateCardWidth } from "../Shared/GridCard/GridCard";
+import { GridCard } from "../Shared/GridCard/GridCard";
 import { HoverPopover } from "../Shared/HoverPopover";
 import { Icon } from "../Shared/Icon";
 import { SceneLink, TagLink } from "../Shared/TagLink";
@@ -12,7 +12,6 @@ import NavUtils from "src/utils/navigation";
 import { RatingBanner } from "../Shared/RatingBanner";
 import { faBox, faPlayCircle, faTag } from "@fortawesome/free-solid-svg-icons";
 import { galleryTitle } from "src/core/galleries";
-import ScreenUtils from "src/utils/screen";
 import { StudioOverlay } from "../Shared/GridCard/StudioOverlay";
 import { GalleryPreviewScrubber } from "./GalleryPreviewScrubber";
 import cx from "classnames";
@@ -56,7 +55,7 @@ export const GalleryPreview: React.FC<IGalleryPreviewProps> = ({
 
 interface IProps {
   gallery: GQL.SlimGalleryDataFragment;
-  containerWidth?: number;
+  cardWidth?: number;
   selecting?: boolean;
   selected?: boolean | undefined;
   zoomIndex?: number;
@@ -65,37 +64,6 @@ interface IProps {
 
 export const GalleryCard: React.FC<IProps> = (props) => {
   const history = useHistory();
-  const [cardWidth, setCardWidth] = useState<number>();
-
-  useEffect(() => {
-    if (
-      !props.containerWidth ||
-      props.zoomIndex === undefined ||
-      ScreenUtils.isMobile()
-    )
-      return;
-
-    let zoomValue = props.zoomIndex;
-    let preferredCardWidth: number;
-    switch (zoomValue) {
-      case 0:
-        preferredCardWidth = 280;
-        break;
-      case 1:
-        preferredCardWidth = 340;
-        break;
-      case 2:
-        preferredCardWidth = 480;
-        break;
-      case 3:
-        preferredCardWidth = 640;
-    }
-    let fittedCardWidth = calculateCardWidth(
-      props.containerWidth,
-      preferredCardWidth!
-    );
-    setCardWidth(fittedCardWidth);
-  }, [props.containerWidth, props.zoomIndex]);
 
   function maybeRenderScenePopoverButton() {
     if (props.gallery.scenes.length === 0) return;
@@ -207,7 +175,7 @@ export const GalleryCard: React.FC<IProps> = (props) => {
     <GridCard
       className={`gallery-card zoom-${props.zoomIndex}`}
       url={`/galleries/${props.gallery.id}`}
-      width={cardWidth}
+      width={props.cardWidth}
       title={galleryTitle(props.gallery)}
       linkClassName="gallery-card-header"
       image={

ui/v2.5/src/components/Galleries/GalleryGridCard.tsx

@@ -1,7 +1,10 @@
 import React from "react";
 import * as GQL from "src/core/generated-graphql";
 import { GalleryCard } from "./GalleryCard";
-import { useContainerDimensions } from "../Shared/GridCard/GridCard";
+import {
+  useCardWidth,
+  useContainerDimensions,
+} from "../Shared/GridCard/GridCard";
 
 interface IGalleryCardGrid {
   galleries: GQL.SlimGalleryDataFragment[];
@@ -10,19 +13,23 @@ interface IGalleryCardGrid {
   onSelectChange: (id: string, selected: boolean, shiftKey: boolean) => void;
 }
 
+const zoomWidths = [280, 340, 480, 640];
+
 export const GalleryCardGrid: React.FC<IGalleryCardGrid> = ({
   galleries,
   selectedIds,
   zoomIndex,
   onSelectChange,
 }) => {
-  const [componentRef, { width }] = useContainerDimensions();
+  const [componentRef, { width: containerWidth }] = useContainerDimensions();
+  const cardWidth = useCardWidth(containerWidth, zoomIndex, zoomWidths);
+
   return (
     <div className="row justify-content-center" ref={componentRef}>
       {galleries.map((gallery) => (
         <GalleryCard
           key={gallery.id}
-          containerWidth={width}
+          cardWidth={cardWidth}
           gallery={gallery}
           zoomIndex={zoomIndex}
           selecting={selectedIds.size > 0}

ui/v2.5/src/components/Galleries/GalleryViewer.tsx

@@ -1,7 +1,7 @@
 import React, { useCallback, useMemo } from "react";
 import { useLightbox } from "src/hooks/Lightbox/hooks";
 import { LoadingIndicator } from "src/components/Shared/LoadingIndicator";
-import Gallery from "react-photo-gallery";
+import Gallery, { PhotoClickHandler } from "react-photo-gallery";
 import "flexbin/flexbin.css";
 import {
   CriterionModifier,
@@ -45,9 +45,9 @@ export const GalleryViewer: React.FC<IProps> = ({ galleryId }) => {
   }, [images]);
 
   const showLightbox = useLightbox(lightboxState);
-  const showLightboxOnClick = useCallback(
+  const showLightboxOnClick: PhotoClickHandler = useCallback(
     (event, { index }) => {
-      showLightbox(index);
+      showLightbox({ initialIndex: index });
     },
     [showLightbox]
   );

ui/v2.5/src/components/Groups/GroupCard.tsx

@@ -1,7 +1,7 @@
-import React, { useEffect, useMemo, useState } from "react";
+import React, { useMemo } from "react";
 import { Button, ButtonGroup } from "react-bootstrap";
 import * as GQL from "src/core/generated-graphql";
-import { GridCard, calculateCardWidth } from "../Shared/GridCard/GridCard";
+import { GridCard } from "../Shared/GridCard/GridCard";
 import { HoverPopover } from "../Shared/HoverPopover";
 import { Icon } from "../Shared/Icon";
 import { SceneLink, TagLink } from "../Shared/TagLink";
@@ -9,7 +9,6 @@ import { TruncatedText } from "../Shared/TruncatedText";
 import { FormattedMessage } from "react-intl";
 import { RatingBanner } from "../Shared/RatingBanner";
 import { faPlayCircle, faTag } from "@fortawesome/free-solid-svg-icons";
-import ScreenUtils from "src/utils/screen";
 import { RelatedGroupPopoverButton } from "./RelatedGroupPopover";
 
 const Description: React.FC<{
@@ -37,7 +36,7 @@ const Description: React.FC<{
 
 interface IProps {
   group: GQL.GroupDataFragment;
-  containerWidth?: number;
+  cardWidth?: number;
   sceneNumber?: number;
   selecting?: boolean;
   selected?: boolean;
@@ -50,7 +49,7 @@ interface IProps {
 export const GroupCard: React.FC<IProps> = ({
   group,
   sceneNumber,
-  containerWidth,
+  cardWidth,
   selecting,
   selected,
   zoomIndex,
@@ -58,8 +57,6 @@ export const GroupCard: React.FC<IProps> = ({
   fromGroupId,
   onMove,
 }) => {
-  const [cardWidth, setCardWidth] = useState<number>();
-
   const groupDescription = useMemo(() => {
     if (!fromGroupId) {
       return undefined;
@@ -72,32 +69,6 @@ export const GroupCard: React.FC<IProps> = ({
     return containingGroup?.description ?? undefined;
   }, [fromGroupId, group.containing_groups]);
 
-  useEffect(() => {
-    if (!containerWidth || zoomIndex === undefined || ScreenUtils.isMobile())
-      return;
-
-    let zoomValue = zoomIndex;
-    let preferredCardWidth: number;
-    switch (zoomValue) {
-      case 0:
-        preferredCardWidth = 210;
-        break;
-      case 1:
-        preferredCardWidth = 250;
-        break;
-      case 2:
-        preferredCardWidth = 300;
-        break;
-      case 3:
-        preferredCardWidth = 375;
-    }
-    let fittedCardWidth = calculateCardWidth(
-      containerWidth,
-      preferredCardWidth!
-    );
-    setCardWidth(fittedCardWidth);
-  }, [containerWidth, zoomIndex]);
-
   function maybeRenderScenesPopoverButton() {
     if (group.scenes.length === 0) return;
 

ui/v2.5/src/components/Groups/GroupCardGrid.tsx

@@ -1,7 +1,10 @@
 import React from "react";
 import * as GQL from "src/core/generated-graphql";
 import { GroupCard } from "./GroupCard";
-import { useContainerDimensions } from "../Shared/GridCard/GridCard";
+import {
+  useCardWidth,
+  useContainerDimensions,
+} from "../Shared/GridCard/GridCard";
 
 interface IGroupCardGrid {
   groups: GQL.GroupDataFragment[];
@@ -12,6 +15,8 @@ interface IGroupCardGrid {
   onMove?: (srcIds: string[], targetId: string, after: boolean) => void;
 }
 
+const zoomWidths = [210, 250, 300, 375];
+
 export const GroupCardGrid: React.FC<IGroupCardGrid> = ({
   groups,
   selectedIds,
@@ -20,13 +25,15 @@ export const GroupCardGrid: React.FC<IGroupCardGrid> = ({
   fromGroupId,
   onMove,
 }) => {
-  const [componentRef, { width }] = useContainerDimensions();
+  const [componentRef, { width: containerWidth }] = useContainerDimensions();
+  const cardWidth = useCardWidth(containerWidth, zoomIndex, zoomWidths);
+
   return (
     <div className="row justify-content-center" ref={componentRef}>
       {groups.map((p) => (
         <GroupCard
           key={p.id}
-          containerWidth={width}
+          cardWidth={cardWidth}
           group={p}
           zoomIndex={zoomIndex}
           selecting={selectedIds.size > 0}

ui/v2.5/src/components/Groups/GroupDetails/GroupScenesPanel.tsx

@@ -5,7 +5,7 @@ import {
   GroupsCriterionOption,
 } from "src/models/list-filter/criteria/groups";
 import { ListFilterModel } from "src/models/list-filter/filter";
-import { SceneList } from "src/components/Scenes/SceneList";
+import { FilteredSceneList } from "src/components/Scenes/SceneList";
 import { View } from "src/components/List/views";
 
 interface IGroupScenesPanel {
@@ -64,7 +64,7 @@ export const GroupScenesPanel: React.FC<IGroupScenesPanel> = ({
 
   if (group && group.id) {
     return (
-      <SceneList
+      <FilteredSceneList
         filterHook={filterHook}
         defaultSort="group_scene_number"
         alterQuery={active}

ui/v2.5/src/components/Images/ImageCard.tsx

@@ -1,4 +1,4 @@
-import React, { MouseEvent, useEffect, useMemo, useState } from "react";
+import React, { MouseEvent, useMemo } from "react";
 import { Button, ButtonGroup } from "react-bootstrap";
 import cx from "classnames";
 import * as GQL from "src/core/generated-graphql";
@@ -7,10 +7,7 @@ import { GalleryLink, TagLink } from "src/components/Shared/TagLink";
 import { HoverPopover } from "src/components/Shared/HoverPopover";
 import { SweatDrops } from "src/components/Shared/SweatDrops";
 import { PerformerPopoverButton } from "src/components/Shared/PerformerPopoverButton";
-import {
-  GridCard,
-  calculateCardWidth,
-} from "src/components/Shared/GridCard/GridCard";
+import { GridCard } from "src/components/Shared/GridCard/GridCard";
 import { RatingBanner } from "src/components/Shared/RatingBanner";
 import {
   faBox,
@@ -20,12 +17,11 @@ import {
 } from "@fortawesome/free-solid-svg-icons";
 import { imageTitle } from "src/core/files";
 import { TruncatedText } from "../Shared/TruncatedText";
-import ScreenUtils from "src/utils/screen";
 import { StudioOverlay } from "../Shared/GridCard/StudioOverlay";
 
 interface IImageCardProps {
   image: GQL.SlimImageDataFragment;
-  containerWidth?: number;
+  cardWidth?: number;
   selecting?: boolean;
   selected?: boolean | undefined;
   zoomIndex: number;
@@ -36,38 +32,6 @@ interface IImageCardProps {
 export const ImageCard: React.FC<IImageCardProps> = (
   props: IImageCardProps
 ) => {
-  const [cardWidth, setCardWidth] = useState<number>();
-
-  useEffect(() => {
-    if (
-      !props.containerWidth ||
-      props.zoomIndex === undefined ||
-      ScreenUtils.isMobile()
-    )
-      return;
-
-    let zoomValue = props.zoomIndex;
-    let preferredCardWidth: number;
-    switch (zoomValue) {
-      case 0:
-        preferredCardWidth = 280;
-        break;
-      case 1:
-        preferredCardWidth = 340;
-        break;
-      case 2:
-        preferredCardWidth = 480;
-        break;
-      case 3:
-        preferredCardWidth = 640;
-    }
-    let fittedCardWidth = calculateCardWidth(
-      props.containerWidth,
-      preferredCardWidth!
-    );
-    setCardWidth(fittedCardWidth);
-  }, [props.containerWidth, props.zoomIndex]);
-
   const file = useMemo(
     () =>
       props.image.visual_files.length > 0
@@ -196,7 +160,7 @@ export const ImageCard: React.FC<IImageCardProps> = (
     <GridCard
       className={`image-card zoom-${props.zoomIndex}`}
       url={`/images/${props.image.id}`}
-      width={cardWidth}
+      width={props.cardWidth}
       title={imageTitle(props.image)}
       linkClassName="image-card-link"
       image={

ui/v2.5/src/components/Images/ImageGridCard.tsx

@@ -1,7 +1,10 @@
 import React from "react";
 import * as GQL from "src/core/generated-graphql";
 import { ImageCard } from "./ImageCard";
-import { useContainerDimensions } from "../Shared/GridCard/GridCard";
+import {
+  useCardWidth,
+  useContainerDimensions,
+} from "../Shared/GridCard/GridCard";
 
 interface IImageCardGrid {
   images: GQL.SlimImageDataFragment[];
@@ -11,6 +14,8 @@ interface IImageCardGrid {
   onPreview: (index: number, ev: React.MouseEvent<Element, MouseEvent>) => void;
 }
 
+const zoomWidths = [280, 340, 480, 640];
+
 export const ImageGridCard: React.FC<IImageCardGrid> = ({
   images,
   selectedIds,
@@ -18,13 +23,15 @@ export const ImageGridCard: React.FC<IImageCardGrid> = ({
   onSelectChange,
   onPreview,
 }) => {
-  const [componentRef, { width }] = useContainerDimensions();
+  const [componentRef, { width: containerWidth }] = useContainerDimensions();
+  const cardWidth = useCardWidth(containerWidth, zoomIndex, zoomWidths);
+
   return (
     <div className="row justify-content-center" ref={componentRef}>
       {images.map((image, index) => (
         <ImageCard
           key={image.id}
-          containerWidth={width}
+          cardWidth={cardWidth}
           image={image}
           zoomIndex={zoomIndex}
           selecting={selectedIds.size > 0}

ui/v2.5/src/components/List/EditFilterDialog.tsx

@@ -488,3 +488,33 @@ export const EditFilterDialog: React.FC<IEditFilterProps> = ({
     </>
   );
 };
+
+export function useShowEditFilter(props: {
+  filter: ListFilterModel;
+  setFilter: (f: ListFilterModel) => void;
+  showModal: (content: React.ReactNode) => void;
+  closeModal: () => void;
+}) {
+  const { filter, setFilter, showModal, closeModal } = props;
+
+  const showEditFilter = useCallback(
+    (editingCriterion?: string) => {
+      function onApplyEditFilter(f: ListFilterModel) {
+        closeModal();
+        setFilter(f);
+      }
+
+      showModal(
+        <EditFilterDialog
+          filter={filter}
+          onApply={onApplyEditFilter}
+          onCancel={() => closeModal()}
+          editingCriterion={editingCriterion}
+        />
+      );
+    },
+    [filter, setFilter, showModal, closeModal]
+  );
+
+  return showEditFilter;
+}

ui/v2.5/src/components/List/FilteredListToolbar.tsx

@@ -8,11 +8,9 @@ import {
   IListFilterOperation,
   ListOperationButtons,
 } from "./ListOperationButtons";
-import { DisplayMode } from "src/models/list-filter/types";
 import { ButtonToolbar } from "react-bootstrap";
 import { View } from "./views";
-import { useListContext } from "./ListProvider";
-import { useFilter } from "./FilterProvider";
+import { IListSelect, useFilterOperations } from "./util";
 
 export interface IItemListOperation<T extends QueryResult> {
   text: string;
@@ -32,8 +30,13 @@ export interface IItemListOperation<T extends QueryResult> {
 }
 
 export interface IFilteredListToolbar {
-  showEditFilter?: (editingCriterion?: string) => void;
+  filter: ListFilterModel;
+  setFilter: (
+    value: ListFilterModel | ((prevState: ListFilterModel) => ListFilterModel)
+  ) => void;
+  showEditFilter: () => void;
   view?: View;
+  listSelect: IListSelect;
   onEdit?: () => void;
   onDelete?: () => void;
   operations?: IListFilterOperation[];
@@ -41,25 +44,22 @@ export interface IFilteredListToolbar {
 }
 
 export const FilteredListToolbar: React.FC<IFilteredListToolbar> = ({
+  filter,
+  setFilter,
   showEditFilter,
   view,
+  listSelect,
   onEdit,
   onDelete,
   operations,
   zoomable = false,
 }) => {
-  const { getSelected, onSelectAll, onSelectNone } = useListContext();
-  const { filter, setFilter } = useFilter();
-
   const filterOptions = filter.options;
-
-  function onChangeDisplayMode(displayMode: DisplayMode) {
-    setFilter(filter.setDisplayMode(displayMode));
-  }
-
-  function onChangeZoom(newZoomIndex: number) {
-    setFilter(filter.setZoom(newZoomIndex));
-  }
+  const { setDisplayMode, setZoom } = useFilterOperations({
+    filter,
+    setFilter,
+  });
+  const { selectedIds, onSelectAll, onSelectNone } = listSelect;
 
   return (
     <ButtonToolbar className="filtered-list-toolbar">
@@ -75,16 +75,16 @@ export const FilteredListToolbar: React.FC<IFilteredListToolbar> = ({
         onSelectAll={onSelectAll}
         onSelectNone={onSelectNone}
         otherOperations={operations}
-        itemsSelected={getSelected().length > 0}
+        itemsSelected={selectedIds.size > 0}
         onEdit={onEdit}
         onDelete={onDelete}
       />
       <ListViewOptions
         displayMode={filter.displayMode}
         displayModeOptions={filterOptions.displayModeOptions}
-        onSetDisplayMode={onChangeDisplayMode}
+        onSetDisplayMode={setDisplayMode}
         zoomIndex={zoomable ? filter.zoomIndex : undefined}
-        onSetZoom={zoomable ? onChangeZoom : undefined}
+        onSetZoom={zoomable ? setZoom : undefined}
       />
     </ButtonToolbar>
   );

ui/v2.5/src/components/List/ItemList.tsx

@@ -10,7 +10,10 @@ import * as GQL from "src/core/generated-graphql";
 import { QueryResult } from "@apollo/client";
 import { Criterion } from "src/models/list-filter/criteria/criterion";
 import { ListFilterModel } from "src/models/list-filter/filter";
-import { EditFilterDialog } from "src/components/List/EditFilterDialog";
+import {
+  EditFilterDialog,
+  useShowEditFilter,
+} from "src/components/List/EditFilterDialog";
 import { FilterTags } from "./FilterTags";
 import { View } from "./views";
 import { IHasID } from "src/utils/data";
@@ -23,9 +26,15 @@ import {
 import { FilterContext, SetFilterURL, useFilter } from "./FilterProvider";
 import { useModal } from "src/hooks/modal";
 import {
+  IFilterStateHook,
+  IQueryResultHook,
   useDefaultFilter,
   useEnsureValidPage,
+  useFilterOperations,
+  useFilterState,
   useListKeyboardShortcuts,
+  useListSelect,
+  useQueryResult,
   useScrollToTopOnPageChange,
 } from "./util";
 import {
@@ -36,6 +45,72 @@ import {
 import { PagedList } from "./PagedList";
 import { ConfigurationContext } from "src/hooks/Config";
 
+interface IFilteredItemList<T extends QueryResult, E extends IHasID = IHasID> {
+  filterStateProps: IFilterStateHook;
+  queryResultProps: IQueryResultHook<T, E>;
+}
+
+// Provides the common state and behaviour for filtered item list components
+export function useFilteredItemList<
+  T extends QueryResult,
+  E extends IHasID = IHasID
+>(props: IFilteredItemList<T, E>) {
+  const { configuration: config } = useContext(ConfigurationContext);
+
+  // States
+  const filterState = useFilterState({
+    config,
+    ...props.filterStateProps,
+  });
+
+  const { filter, setFilter } = filterState;
+
+  const queryResult = useQueryResult({
+    filter,
+    ...props.queryResultProps,
+  });
+  const { result, items, totalCount, pages } = queryResult;
+
+  const listSelect = useListSelect(items);
+  const { onSelectAll, onSelectNone } = listSelect;
+
+  const modalState = useModal();
+  const { showModal, closeModal } = modalState;
+
+  // Utility hooks
+  const { setPage } = useFilterOperations({ filter, setFilter });
+
+  // scroll to the top of the page when the page changes
+  useScrollToTopOnPageChange(filter.currentPage, result.loading);
+
+  // ensure that the current page is valid
+  useEnsureValidPage(filter, totalCount, setFilter);
+
+  const showEditFilter = useShowEditFilter({
+    showModal,
+    closeModal,
+    filter,
+    setFilter,
+  });
+
+  useListKeyboardShortcuts({
+    currentPage: filter.currentPage,
+    onChangePage: setPage,
+    onSelectAll,
+    onSelectNone,
+    pages,
+    showEditFilter,
+  });
+
+  return {
+    filterState,
+    queryResult,
+    listSelect,
+    modalState,
+    showEditFilter,
+  };
+}
+
 interface IItemListProps<T extends QueryResult, E extends IHasID> {
   view?: View;
   zoomable?: boolean;
@@ -83,13 +158,14 @@ export const ItemList = <T extends QueryResult, E extends IHasID>(
   const { filter, setFilter: updateFilter } = useFilter();
   const { effectiveFilter, result, cachedResult, totalCount } =
     useQueryResultContext<T, E>();
+  const listSelect = useListContext<E>();
   const {
     selectedIds,
     getSelected,
     onSelectChange,
     onSelectAll,
     onSelectNone,
-  } = useListContext<E>();
+  } = listSelect;
 
   // scroll to the top of the page when the page changes
   useScrollToTopOnPageChange(filter.currentPage, result.loading);
@@ -236,7 +312,10 @@ export const ItemList = <T extends QueryResult, E extends IHasID>(
     updateFilter(filter.clearCriteria());
   }
 
-  const filterListToolbarProps = {
+  const filterListToolbarProps: IFilteredListToolbar = {
+    filter,
+    setFilter: updateFilter,
+    listSelect,
     showEditFilter,
     view: view,
     operations: operations,

ui/v2.5/src/components/List/ListProvider.tsx

@@ -29,22 +29,12 @@ export const ListContext = <T extends IHasID = IHasID>(
 ) => {
   const { selectable = false, items, children } = props;
 
-  const {
-    selectedIds,
-    getSelected,
-    onSelectChange,
-    onSelectAll,
-    onSelectNone,
-  } = useListSelect(items);
+  const listSelect = useListSelect(items);
 
   const state: IListContextState<T> = {
     selectable,
-    selectedIds,
-    getSelected,
-    onSelectChange,
-    onSelectAll,
-    onSelectNone,
     items,
+    ...listSelect,
   };
 
   return (
@@ -74,6 +64,8 @@ const emptyState: IListContextState = {
   onSelectAll: () => {},
   onSelectNone: () => {},
   items: [],
+  hasSelection: false,
+  selectedItems: [],
 };
 
 export function useListContextOptional<T extends IHasID = IHasID>() {

ui/v2.5/src/components/List/util.ts

@@ -8,6 +8,9 @@ import { IHasID } from "src/utils/data";
 import { ConfigurationContext } from "src/hooks/Config";
 import { View } from "./views";
 import { usePrevious } from "src/hooks/state";
+import * as GQL from "src/core/generated-graphql";
+import { DisplayMode } from "src/models/list-filter/types";
+import { Criterion } from "src/models/list-filter/criteria/criterion";
 
 export function useFilterURL(
   filter: ListFilterModel,
@@ -106,6 +109,106 @@ export function useDefaultFilter(emptyFilter: ListFilterModel, view?: View) {
   return { defaultFilter: retFilter, loading };
 }
 
+function useEmptyFilter(props: {
+  filterMode: GQL.FilterMode;
+  defaultSort?: string;
+  config?: GQL.ConfigDataFragment;
+}) {
+  const { filterMode, defaultSort, config } = props;
+
+  const emptyFilter = useMemo(
+    () =>
+      new ListFilterModel(filterMode, config, {
+        defaultSortBy: defaultSort,
+      }),
+    [config, filterMode, defaultSort]
+  );
+
+  return emptyFilter;
+}
+
+export interface IFilterStateHook {
+  filterMode: GQL.FilterMode;
+  defaultSort?: string;
+  view?: View;
+  useURL?: boolean;
+}
+
+export function useFilterState(
+  props: IFilterStateHook & {
+    config?: GQL.ConfigDataFragment;
+  }
+) {
+  const { filterMode, defaultSort, config, view, useURL } = props;
+
+  const [filter, setFilterState] = useState<ListFilterModel>(
+    () =>
+      new ListFilterModel(filterMode, config, { defaultSortBy: defaultSort })
+  );
+
+  const emptyFilter = useEmptyFilter({ filterMode, defaultSort, config });
+
+  const { defaultFilter, loading } = useDefaultFilter(emptyFilter, view);
+
+  const { setFilter } = useFilterURL(filter, setFilterState, {
+    defaultFilter,
+    active: useURL,
+  });
+
+  return { loading, filter, setFilter };
+}
+
+export function useFilterOperations(props: {
+  filter: ListFilterModel;
+  setFilter: (
+    value: ListFilterModel | ((prevState: ListFilterModel) => ListFilterModel)
+  ) => void;
+}) {
+  const { setFilter } = props;
+
+  const setPage = useCallback(
+    (p: number) => {
+      setFilter((cv) => cv.changePage(p));
+    },
+    [setFilter]
+  );
+
+  const setDisplayMode = useCallback(
+    (displayMode: DisplayMode) => {
+      setFilter((cv) => cv.setDisplayMode(displayMode));
+    },
+    [setFilter]
+  );
+
+  const setZoom = useCallback(
+    (newZoomIndex: number) => {
+      setFilter((cv) => cv.setZoom(newZoomIndex));
+    },
+    [setFilter]
+  );
+
+  const removeCriterion = useCallback(
+    (removedCriterion: Criterion) => {
+      setFilter((cv) =>
+        cv.removeCriterion(removedCriterion.criterionOption.type)
+      );
+    },
+    [setFilter]
+  );
+
+  const clearAllCriteria = useCallback(() => {
+    setFilter((cv) => cv.clearCriteria());
+  }, [setFilter]);
+
+  return {
+    setPage,
+    setDisplayMode,
+    setZoom,
+    removeCriterion,
+    clearAllCriteria,
+  };
+}
+
 export function useListKeyboardShortcuts(props: {
   currentPage?: number;
   onChangePage?: (page: number) => void;
@@ -190,10 +293,11 @@ export function useListKeyboardShortcuts(props: {
   }, [onSelectAll, onSelectNone]);
 }
 
-export function useListSelect<T extends { id: string }>(items: T[]) {
+export function useListSelect<T extends IHasID = IHasID>(items: T[]) {
   const [itemsSelected, setItemsSelected] = useState<T[]>([]);
   const [lastClickedId, setLastClickedId] = useState<string>();
 
+  // TODO - this doesn't get updated when items changes
   const selectedIds = useMemo(() => {
     const newSelectedIds = new Set<string>();
     itemsSelected.forEach((item) => {
@@ -303,18 +407,26 @@ export function useListSelect<T extends { id: string }>(items: T[]) {
     setLastClickedId(undefined);
   }
 
+  // TODO - this is for backwards compatibility
   const getSelected = useCallback(() => itemsSelected, [itemsSelected]);
 
+  // convenience state
+  const hasSelection = itemsSelected.length > 0;
+
   return {
+    selectedItems: itemsSelected,
     selectedIds,
     getSelected,
     onSelectChange,
     onSelectAll,
     onSelectNone,
+    hasSelection,
   };
 }
 
-export type IListSelect<T extends IHasID> = ReturnType<typeof useListSelect<T>>;
+export type IListSelect<T extends IHasID = IHasID> = ReturnType<
+  typeof useListSelect<T>
+>;
 
 // returns true if the filter has changed in a way that impacts the total count
 function totalCountImpacted(
@@ -358,6 +470,81 @@ export function useCachedQueryResult<T extends QueryResult>(
   return cachedResult;
 }
 
+export interface IQueryResultHook<
+  T extends QueryResult,
+  E extends IHasID = IHasID
+> {
+  filterHook?: (filter: ListFilterModel) => ListFilterModel;
+  useResult: (filter: ListFilterModel) => T;
+  getCount: (data: T) => number;
+  getItems: (data: T) => E[];
+}
+
+export function useQueryResult<
+  T extends QueryResult,
+  E extends IHasID = IHasID
+>(
+  props: IQueryResultHook<T, E> & {
+    filter: ListFilterModel;
+  }
+) {
+  const { filter, filterHook, useResult, getItems, getCount } = props;
+
+  const effectiveFilter = useMemo(() => {
+    if (filterHook) {
+      return filterHook(filter.clone());
+    }
+    return filter;
+  }, [filter, filterHook]);
+
+  const result = useResult(effectiveFilter);
+
+  // use cached query result for pagination and metadata rendering
+  const cachedResult = useCachedQueryResult(effectiveFilter, result);
+
+  const items = useMemo(() => getItems(result), [getItems, result]);
+  const totalCount = useMemo(
+    () => getCount(cachedResult),
+    [getCount, cachedResult]
+  );
+
+  const pages = Math.ceil(totalCount / filter.itemsPerPage);
+
+  return {
+    effectiveFilter,
+    result,
+    cachedResult,
+    items,
+    totalCount,
+    pages,
+  };
+}
+
+// this hook collects the common logic when closing the edit/delete dialog
+// if applied is true, then the list should be refetched and selection cleared
+export function useCloseEditDelete(props: {
+  onSelectNone: () => void;
+  closeModal: () => void;
+  result: QueryResult;
+}) {
+  const { onSelectNone, closeModal, result } = props;
+
+  const onCloseEditDelete = useCallback(
+    (applied?: boolean) => {
+      closeModal();
+      if (applied) {
+        onSelectNone();
+
+        // refetch
+        result.refetch();
+      }
+    },
+    [onSelectNone, closeModal, result]
+  );
+
+  return onCloseEditDelete;
+}
+
 export function useScrollToTopOnPageChange(
   currentPage: number,
   loading: boolean

ui/v2.5/src/components/Performers/PerformerCard.tsx

@@ -1,10 +1,10 @@
-import React, { useEffect, useState } from "react";
+import React from "react";
 import { Link } from "react-router-dom";
 import { useIntl } from "react-intl";
 import * as GQL from "src/core/generated-graphql";
 import NavUtils from "src/utils/navigation";
 import TextUtils from "src/utils/text";
-import { GridCard, calculateCardWidth } from "../Shared/GridCard/GridCard";
+import { GridCard } from "../Shared/GridCard/GridCard";
 import { CountryFlag } from "../Shared/CountryFlag";
 import { SweatDrops } from "../Shared/SweatDrops";
 import { HoverPopover } from "../Shared/HoverPopover";
@@ -21,7 +21,6 @@ import { faTag } from "@fortawesome/free-solid-svg-icons";
 import { RatingBanner } from "../Shared/RatingBanner";
 import { usePerformerUpdate } from "src/core/StashService";
 import { ILabeledId } from "src/models/list-filter/types";
-import ScreenUtils from "src/utils/screen";
 import { FavoriteIcon } from "../Shared/FavoriteIcon";
 import { PatchComponent } from "src/patch";
 
@@ -35,7 +34,7 @@ export interface IPerformerCardExtraCriteria {
 
 interface IPerformerCardProps {
   performer: GQL.PerformerDataFragment;
-  containerWidth?: number;
+  cardWidth?: number;
   ageFromDate?: string;
   selecting?: boolean;
   selected?: boolean;
@@ -300,41 +299,13 @@ export const PerformerCard: React.FC<IPerformerCardProps> = PatchComponent(
   (props) => {
     const {
       performer,
-      containerWidth,
+      cardWidth,
       selecting,
       selected,
       onSelectedChanged,
       zoomIndex,
     } = props;
 
-    const [cardWidth, setCardWidth] = useState<number>();
-
-    useEffect(() => {
-      if (!containerWidth || zoomIndex === undefined || ScreenUtils.isMobile())
-        return;
-
-      let zoomValue = zoomIndex;
-      let preferredCardWidth: number;
-      switch (zoomValue) {
-        case 0:
-          preferredCardWidth = 240;
-          break;
-        case 1:
-          preferredCardWidth = 300;
-          break;
-        case 2:
-          preferredCardWidth = 375;
-          break;
-        case 3:
-          preferredCardWidth = 470;
-      }
-      let fittedCardWidth = calculateCardWidth(
-        containerWidth,
-        preferredCardWidth!
-      );
-      setCardWidth(fittedCardWidth);
-    }, [containerWidth, zoomIndex]);
-
     return (
       <GridCard
         className={`performer-card zoom-${zoomIndex}`}

ui/v2.5/src/components/Performers/PerformerCardGrid.tsx

@@ -1,7 +1,10 @@
 import React from "react";
 import * as GQL from "src/core/generated-graphql";
 import { IPerformerCardExtraCriteria, PerformerCard } from "./PerformerCard";
-import { useContainerDimensions } from "../Shared/GridCard/GridCard";
+import {
+  useCardWidth,
+  useContainerDimensions,
+} from "../Shared/GridCard/GridCard";
 
 interface IPerformerCardGrid {
   performers: GQL.PerformerDataFragment[];
@@ -11,6 +14,8 @@ interface IPerformerCardGrid {
   extraCriteria?: IPerformerCardExtraCriteria;
 }
 
+const zoomWidths = [240, 300, 375, 470];
+
 export const PerformerCardGrid: React.FC<IPerformerCardGrid> = ({
   performers,
   selectedIds,
@@ -18,13 +23,15 @@ export const PerformerCardGrid: React.FC<IPerformerCardGrid> = ({
   onSelectChange,
   extraCriteria,
 }) => {
-  const [componentRef, { width }] = useContainerDimensions();
+  const [componentRef, { width: containerWidth }] = useContainerDimensions();
+  const cardWidth = useCardWidth(containerWidth, zoomIndex, zoomWidths);
+
   return (
     <div className="row justify-content-center" ref={componentRef}>
       {performers.map((p) => (
         <PerformerCard
           key={p.id}
-          containerWidth={width}
+          cardWidth={cardWidth}
           performer={p}
           zoomIndex={zoomIndex}
           selecting={selectedIds.size > 0}

ui/v2.5/src/components/Performers/PerformerDetails/PerformerScenesPanel.tsx

@@ -1,6 +1,6 @@
 import React from "react";
 import * as GQL from "src/core/generated-graphql";
-import { SceneList } from "src/components/Scenes/SceneList";
+import { FilteredSceneList } from "src/components/Scenes/SceneList";
 import { usePerformerFilterHook } from "src/core/performers";
 import { View } from "src/components/List/views";
 import { PatchComponent } from "src/patch";
@@ -14,7 +14,7 @@ export const PerformerScenesPanel: React.FC<IPerformerDetailsProps> =
   PatchComponent("PerformerScenesPanel", ({ active, performer }) => {
     const filterHook = usePerformerFilterHook(performer);
     return (
-      <SceneList
+      <FilteredSceneList
         filterHook={filterHook}
         alterQuery={active}
         view={View.PerformerScenes}

ui/v2.5/src/components/Performers/PerformerPopover.tsx

@@ -0,0 +1,71 @@
+import React from "react";
+import { ErrorMessage } from "../Shared/ErrorMessage";
+import { LoadingIndicator } from "../Shared/LoadingIndicator";
+import { HoverPopover } from "../Shared/HoverPopover";
+import { useFindPerformer } from "../../core/StashService";
+import { PerformerCard } from "./PerformerCard";
+import { ConfigurationContext } from "../../hooks/Config";
+import { Placement } from "react-bootstrap/esm/Overlay";
+
+interface IPeromerPopoverCardProps {
+  id: string;
+}
+
+export const PerformerPopoverCard: React.FC<IPeromerPopoverCardProps> = ({
+  id,
+}) => {
+  const { data, loading, error } = useFindPerformer(id);
+
+  if (loading)
+    return (
+      <div className="tag-popover-card-placeholder">
+        <LoadingIndicator card={true} message={""} />
+      </div>
+    );
+  if (error) return <ErrorMessage error={error.message} />;
+  if (!data?.findPerformer)
+    return <ErrorMessage error={`No tag found with id ${id}.`} />;
+
+  const performer = data.findPerformer;
+
+  return (
+    <div className="tag-popover-card">
+      <PerformerCard performer={performer} zoomIndex={0} />
+    </div>
+  );
+};
+
+interface IPeroformerPopoverProps {
+  id: string;
+  hide?: boolean;
+  placement?: Placement;
+  target?: React.RefObject<HTMLElement>;
+}
+
+export const PerformerPopover: React.FC<IPeroformerPopoverProps> = ({
+  id,
+  hide,
+  children,
+  placement = "top",
+  target,
+}) => {
+  const { configuration: config } = React.useContext(ConfigurationContext);
+
+  const showPerformerCardOnHover = config?.ui.showTagCardOnHover ?? true;
+
+  if (hide || !showPerformerCardOnHover) {
+    return <>{children}</>;
+  }
+
+  return (
+    <HoverPopover
+      target={target}
+      placement={placement}
+      enterDelay={500}
+      leaveDelay={100}
+      content={<PerformerPopoverCard id={id} />}
+    >
+      {children}
+    </HoverPopover>
+  );
+};