Compare commits

..

1 Commits

Author SHA1 Message Date
DogmaDragon
a4bba6726f Correct syntax 2025-01-01 22:19:55 +02:00
298 changed files with 5587 additions and 12665 deletions

3
.gitignore vendored
View File

@@ -21,9 +21,6 @@ vendor
# GraphQL generated output
internal/api/generated_*.go
# Generated locale files
ui/login/locales/*
####
# Visual Studio
####

View File

@@ -48,6 +48,8 @@ linters-settings:
ignore-generated-header: true
severity: error
confidence: 0.8
error-code: 1
warning-code: 1
rules:
- name: blank-imports
disabled: true

View File

@@ -281,10 +281,6 @@ generate-ui:
generate-backend: touch-ui
go generate ./cmd/stash
.PHONY: generate-login-locale
generate-login-locale:
go generate ./ui
.PHONY: generate-dataloaders
generate-dataloaders:
go generate ./internal/api/loaders
@@ -355,10 +351,7 @@ ifdef STASH_SOURCEMAPS
endif
.PHONY: ui
ui: ui-only generate-login-locale
.PHONY: ui-only
ui-only: ui-env
ui: ui-env
cd ui/v2.5 && yarn build
.PHONY: zip-ui

View File

@@ -81,9 +81,9 @@ For more help you can:
# Customization
## Themes and CSS Customization
There is a [directory of community-created themes](https://docs.stashapp.cc/themes/list) on Stash-Docs.
There is a [directory of community-created themes](https://docs.stashapp.cc/user-interface-ui/themes) on Stash-Docs, along with instructions on how to install them.
You can also change the Stash interface to fit your desired style with various snippets from [Custom CSS snippets](https://docs.stashapp.cc/themes/custom-css-snippets).
You can also change the Stash interface to fit your desired style with various snippets from [Custom CSS snippets](https://docs.stashapp.cc/user-interface-ui/custom-css-snippets).
# For Developers

View File

@@ -1,7 +1,7 @@
# This dockerfile should be built with `make docker-build` from the stash root.
# Build Frontend
FROM node:20-alpine AS frontend
FROM node:alpine as frontend
RUN apk add --no-cache make git
## cache node_modules separately
COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/
@@ -13,22 +13,19 @@ RUN make pre-ui
RUN make generate-ui
ARG GITHASH
ARG STASH_VERSION
RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui-only
RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui
# Build Backend
FROM golang:1.22.8-alpine AS backend
FROM golang:1.22-alpine as backend
RUN apk add --no-cache make alpine-sdk
WORKDIR /stash
COPY ./go* ./*.go Makefile gqlgen.yml .gqlgenc.yml /stash/
COPY ./graphql /stash/graphql/
COPY ./scripts /stash/scripts/
COPY ./pkg /stash/pkg/
COPY ./cmd /stash/cmd/
COPY ./internal /stash/internal/
# needed for generate-login-locale
COPY ./ui /stash/ui/
RUN make generate-backend generate-login-locale
COPY ./cmd /stash/cmd
COPY ./internal /stash/internal
COPY --from=frontend /stash /stash/
RUN make generate-backend
ARG GITHASH
ARG STASH_VERSION
RUN make flags-release flags-pie stash

View File

@@ -1,7 +1,7 @@
# This dockerfile should be built with `make docker-cuda-build` from the stash root.
# Build Frontend
FROM node:20-alpine AS frontend
FROM node:alpine as frontend
RUN apk add --no-cache make git
## cache node_modules separately
COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/
@@ -13,22 +13,19 @@ RUN make pre-ui
RUN make generate-ui
ARG GITHASH
ARG STASH_VERSION
RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui-only
RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui
# Build Backend
FROM golang:1.22.8-bullseye AS backend
FROM golang:1.22-bullseye as backend
RUN apt update && apt install -y build-essential golang
WORKDIR /stash
COPY ./go* ./*.go Makefile gqlgen.yml .gqlgenc.yml /stash/
COPY ./graphql /stash/graphql/
COPY ./scripts /stash/scripts/
COPY ./pkg /stash/pkg/
COPY ./cmd /stash/cmd
COPY ./internal /stash/internal
# needed for generate-login-locale
COPY ./ui /stash/ui/
RUN make generate-backend generate-login-locale
COPY --from=frontend /stash /stash/
RUN make generate-backend
ARG GITHASH
ARG STASH_VERSION
RUN make flags-release flags-pie stash

View File

@@ -4,7 +4,7 @@ This dockerfile is used to build a stash docker container using the current sour
# Building the docker container
From the top-level directory (should contain `tools.go` file):
From the top-level directory (should contain `main.go` file):
```
make docker-build

View File

@@ -1,14 +1,12 @@
# Docker Installation (for most 64-bit GNU/Linux systems)
StashApp is supported on most systems that support Docker. Your OS likely ships with or makes available the necessary packages.
StashApp is supported on most systems that support Docker and docker-compose. Your OS likely ships with or makes available the necessary packages.
## Dependencies
Only `docker` is required. For the most part your understanding of the technologies can be superficial. So long as you can follow commands and are open to reading a bit, you should be fine.
Only `docker` and `docker-compose` are required. For the most part your understanding of the technologies can be superficial. So long as you can follow commands and are open to reading a bit, you should be fine.
Installation instructions are available below, and if your distributions's repository ships a current version of docker, you may use that.
Installation instructions are available below, and if your distrobution's repository ships a current version of docker, you may use that.
https://docs.docker.com/engine/install/
On some distributions, `docker compose` is shipped seperately, usually as `docker-cli-compose`. docker-compose is not recommended.
### Get the docker-compose.yml file
Now you can either navigate to the [docker-compose.yml](https://raw.githubusercontent.com/stashapp/stash/develop/docker/production/docker-compose.yml) in the repository, or if you have curl, you can make your Linux console do it for you:
@@ -21,7 +19,7 @@ curl -o docker-compose.yml https://raw.githubusercontent.com/stashapp/stash/deve
Once you have that file where you want it, modify the settings as you please, and then run:
```
docker compose up -d
docker-compose up -d
```
Installing StashApp this way will by default bind stash to port 9999. This is available in your web browser locally at http://localhost:9999 or on your network as http://YOUR-LOCAL-IP:9999
@@ -31,9 +29,9 @@ Good luck and have fun!
### Docker
Docker is effectively a cross-platform software package repository. It allows you to ship an entire environment in what's referred to as a container. Containers are intended to hold everything that is needed to run an application from one place to another, making it easy for everyone along the way to reproduce the environment.
The StashApp docker container ships with everything you need to automatically run stash, including ffmpeg.
The StashApp docker container ships with everything you need to automatically build and run stash, including ffmpeg.
### docker compose
Docker Compose lets you specify how and where to run your containers, and to manage their environment. The docker-compose.yml file in this folder gets you a fully working instance of StashApp exactly as you would need it to have a reasonable instance for testing / developing on. If you are deploying a live instance for production, a [reverse proxy](https://docs.stashapp.cc/guides/reverse-proxy/) (such as NGINX or Traefik) is recommended, but not required.
### docker-compose
Docker Compose lets you specify how and where to run your containers, and to manage their environment. The docker-compose.yml file in this folder gets you a fully working instance of StashApp exactly as you would need it to have a reasonable instance for testing / developing on. If you are deploying a live instance for production, a reverse proxy (such as NGINX or Traefik) is recommended, but not required.
The latest version is always recommended.

View File

@@ -1,5 +1,6 @@
# APPNICENAME=Stash
# APPDESCRIPTION=An organizer for your porn, written in Go
version: '3.4'
services:
stash:
image: stashapp/stash:latest
@@ -26,12 +27,10 @@ services:
- /etc/localtime:/etc/localtime:ro
## Adjust below paths (the left part) to your liking.
## E.g. you can change ./config:/root/.stash to ./stash:/root/.stash
## The left part is the path on your host, the right part is the path in the stash container.
## Keep configs, scrapers, and plugins here.
- ./config:/root/.stash
## Point this at your collection.
## The left side is where your collection is on your host, the right side is where it will be in stash.
- ./data:/data
## This is where your stash's metadata lives
- ./metadata:/metadata

18
go.mod
View File

@@ -19,9 +19,9 @@ require (
github.com/go-chi/cors v1.2.1
github.com/go-chi/httplog v0.3.1
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4
github.com/golang-jwt/jwt/v4 v4.5.2
github.com/gofrs/uuid/v5 v5.1.0
github.com/golang-jwt/jwt/v4 v4.5.1
github.com/golang-migrate/migrate/v4 v4.16.2
github.com/google/uuid v1.6.0
github.com/gorilla/securecookie v1.1.1
github.com/gorilla/sessions v1.2.1
github.com/gorilla/websocket v1.5.0
@@ -51,13 +51,12 @@ require (
github.com/vektra/mockery/v2 v2.10.0
github.com/xWTF/chardet v0.0.0-20230208095535-c780f2ac244e
github.com/zencoder/go-dash/v3 v3.0.2
golang.org/x/crypto v0.31.0
golang.org/x/crypto v0.28.0
golang.org/x/image v0.18.0
golang.org/x/net v0.33.0
golang.org/x/sys v0.28.0
golang.org/x/term v0.27.0
golang.org/x/text v0.21.0
golang.org/x/time v0.10.0
golang.org/x/net v0.30.0
golang.org/x/sys v0.26.0
golang.org/x/term v0.25.0
golang.org/x/text v0.19.0
gopkg.in/guregu/null.v4 v4.0.0
gopkg.in/yaml.v2 v2.4.0
)
@@ -79,6 +78,7 @@ require (
github.com/gobwas/ws v1.3.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
@@ -113,7 +113,7 @@ require (
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/mod v0.21.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/tools v0.26.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect

32
go.sum
View File

@@ -244,10 +244,12 @@ github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/K
github.com/gobwas/ws v1.3.0 h1:sbeU3Y4Qzlb+MOzIe6mQGf7QR4Hkv6ZD0qhGkBFL2O0=
github.com/gobwas/ws v1.3.0/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/uuid/v5 v5.1.0 h1:S5rqVKIigghZTCBKPCw0Y+bXkn26K3TB5mvQq2Ix8dk=
github.com/gofrs/uuid/v5 v5.1.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA=
github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
@@ -713,8 +715,8 @@ golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -807,8 +809,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -838,8 +840,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -930,13 +932,13 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -949,13 +951,11 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

View File

@@ -17,7 +17,7 @@ autobind:
- github.com/stashapp/stash/pkg/scraper
- github.com/stashapp/stash/internal/identify
- github.com/stashapp/stash/internal/dlna
- github.com/stashapp/stash/pkg/stashbox
- github.com/stashapp/stash/pkg/scraper/stashbox
models:
# Scalars
@@ -35,6 +35,12 @@ models:
model: github.com/stashapp/stash/internal/api.BoolMap
PluginConfigMap:
model: github.com/stashapp/stash/internal/api.PluginConfigMap
# define to force resolvers
Image:
model: github.com/stashapp/stash/pkg/models.Image
fields:
title:
resolver: true
VideoFile:
fields:
# override float fields - #1572

View File

@@ -45,7 +45,6 @@ type Query {
findSceneMarkers(
scene_marker_filter: SceneMarkerFilterType
filter: FindFilterType
ids: [ID!]
): FindSceneMarkersResultType!
findImage(id: ID, checksum: String): Image
@@ -174,12 +173,6 @@ type Query {
input: ScrapeSingleGroupInput!
): [ScrapedGroup!]!
"Scrape for a single image"
scrapeSingleImage(
source: ScraperSourceInput!
input: ScrapeSingleImageInput!
): [ScrapedImage!]!
"Scrapes content based on a URL"
scrapeURL(url: String!, ty: ScrapeContentType!): ScrapedContent
@@ -189,8 +182,6 @@ type Query {
scrapeSceneURL(url: String!): ScrapedScene
"Scrapes a complete gallery record based on a URL"
scrapeGalleryURL(url: String!): ScrapedGallery
"Scrapes a complete image record based on a URL"
scrapeImageURL(url: String!): ScrapedImage
"Scrapes a complete movie record based on a URL"
scrapeMovieURL(url: String!): ScrapedMovie
@deprecated(reason: "Use scrapeGroupURL instead")

View File

@@ -542,9 +542,6 @@ input TagFilterType {
"Filter by tag name"
name: StringCriterionInput
"Filter by tag sort_name"
sort_name: StringCriterionInput
"Filter by tag aliases"
aliases: StringCriterionInput

View File

@@ -10,7 +10,6 @@ enum ScrapeType {
"Type of the content a scraper generates"
enum ScrapeContentType {
GALLERY
IMAGE
MOVIE
GROUP
PERFORMER
@@ -23,7 +22,6 @@ union ScrapedContent =
| ScrapedTag
| ScrapedScene
| ScrapedGallery
| ScrapedImage
| ScrapedMovie
| ScrapedGroup
| ScrapedPerformer
@@ -43,8 +41,6 @@ type Scraper {
scene: ScraperSpec
"Details for gallery scraper"
gallery: ScraperSpec
"Details for image scraper"
image: ScraperSpec
"Details for movie scraper"
movie: ScraperSpec @deprecated(reason: "use group")
"Details for group scraper"
@@ -132,26 +128,6 @@ input ScrapedGalleryInput {
# no studio, tags or performers
}
type ScrapedImage {
title: String
code: String
details: String
photographer: String
urls: [String!]
date: String
studio: ScrapedStudio
tags: [ScrapedTag!]
performers: [ScrapedPerformer!]
}
input ScrapedImageInput {
title: String
code: String
details: String
urls: [String!]
date: String
}
input ScraperSourceInput {
"Index of the configured stash-box instance to use. Should be unset if scraper_id is set"
stash_box_index: Int @deprecated(reason: "use stash_box_endpoint")
@@ -214,15 +190,6 @@ input ScrapeSingleGalleryInput {
gallery_input: ScrapedGalleryInput
}
input ScrapeSingleImageInput {
"Instructs to query by string"
query: String
"Instructs to query by image id"
image_id: ID
"Instructs to query by image fragment"
image_input: ScrapedImageInput
}
input ScrapeSingleMovieInput {
"Instructs to query by string"
query: String

View File

@@ -2,15 +2,12 @@ type StashBox {
endpoint: String!
api_key: String!
name: String!
max_requests_per_minute: Int!
}
input StashBoxInput {
endpoint: String!
api_key: String!
name: String!
# defaults to 240
max_requests_per_minute: Int
}
type StashID {

View File

@@ -1,8 +1,6 @@
type Tag {
id: ID!
name: String!
"Value that does not appear in the UI but overrides name for sorting"
sort_name: String
description: String
aliases: [String!]!
ignore_auto_tag: Boolean!
@@ -27,8 +25,6 @@ type Tag {
input TagCreateInput {
name: String!
"Value that does not appear in the UI but overrides name for sorting"
sort_name: String
description: String
aliases: [String!]
ignore_auto_tag: Boolean
@@ -43,8 +39,6 @@ input TagCreateInput {
input TagUpdateInput {
id: ID!
name: String
"Value that does not appear in the UI but overrides name for sorting"
sort_name: String
description: String
aliases: [String!]
ignore_auto_tag: Boolean

View File

@@ -49,8 +49,6 @@ fragment PerformerFragment on Performer {
aliases
gender
merged_ids
deleted
merged_into_id
urls {
...URLFragment
}
@@ -58,7 +56,6 @@ fragment PerformerFragment on Performer {
...ImageFragment
}
birth_date
death_date
ethnicity
country
eye_color

View File

@@ -3,11 +3,9 @@ package api
import (
"encoding/json"
"strings"
"github.com/stashapp/stash/pkg/models"
)
// jsonNumberToNumber converts a JSON number to either a float64 or int64.
// JSONNumberToNumber converts a JSON number to either a float64 or int64.
func jsonNumberToNumber(n json.Number) interface{} {
if strings.Contains(string(n), ".") {
f, _ := n.Float64()
@@ -17,15 +15,6 @@ func jsonNumberToNumber(n json.Number) interface{} {
return ret
}
// anyJSONNumberToNumber converts a JSON number using jsonNumberToNumber, otherwise it returns the existing value
func anyJSONNumberToNumber(v any) any {
if n, ok := v.(json.Number); ok {
return jsonNumberToNumber(n)
}
return v
}
// ConvertMapJSONNumbers converts all JSON numbers in a map to either float64 or int64.
func convertMapJSONNumbers(m map[string]interface{}) (ret map[string]interface{}) {
if m == nil {
@@ -45,21 +34,3 @@ func convertMapJSONNumbers(m map[string]interface{}) (ret map[string]interface{}
return ret
}
func convertCustomFieldCriterionValues(c models.CustomFieldCriterionInput) models.CustomFieldCriterionInput {
nv := make([]any, len(c.Value))
for i, v := range c.Value {
nv[i] = anyJSONNumberToNumber(v)
}
c.Value = nv
return c
}
func convertCustomFieldCriterionInputJSONNumbers(c []models.CustomFieldCriterionInput) []models.CustomFieldCriterionInput {
ret := make([]models.CustomFieldCriterionInput, len(c))
for i, v := range c {
ret[i] = convertCustomFieldCriterionValues(v)
}
return ret
}

View File

@@ -13,6 +13,7 @@ import (
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/plugin/hook"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/scraper/stashbox"
)
var (
@@ -137,6 +138,10 @@ func (r *Resolver) withReadTxn(ctx context.Context, fn func(ctx context.Context)
return r.repository.WithReadTxn(ctx, fn)
}
func (r *Resolver) stashboxRepository() stashbox.Repository {
return stashbox.NewRepository(r.repository)
}
func (r *queryResolver) MarkerWall(ctx context.Context, q *string) (ret []*models.SceneMarker, err error) {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
ret, err = r.repository.SceneMarker.Wall(ctx, q)

View File

@@ -18,6 +18,11 @@ func (r *imageResolver) getFiles(ctx context.Context, obj *models.Image) ([]mode
return files, firstError(errs)
}
func (r *imageResolver) Title(ctx context.Context, obj *models.Image) (*string, error) {
ret := obj.GetTitle()
return &ret, nil
}
func (r *imageResolver) VisualFiles(ctx context.Context, obj *models.Image) ([]VisualFile, error) {
files, err := r.getFiles(ctx, obj)
if err != nil {

View File

@@ -28,7 +28,7 @@ func (r *mutationResolver) getImage(ctx context.Context, id int) (ret *models.Im
return ret, nil
}
func (r *mutationResolver) ImageUpdate(ctx context.Context, input models.ImageUpdateInput) (ret *models.Image, err error) {
func (r *mutationResolver) ImageUpdate(ctx context.Context, input ImageUpdateInput) (ret *models.Image, err error) {
translator := changesetTranslator{
inputMap: getUpdateInputMap(ctx),
}
@@ -46,7 +46,7 @@ func (r *mutationResolver) ImageUpdate(ctx context.Context, input models.ImageUp
return r.getImage(ctx, ret.ID)
}
func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*models.ImageUpdateInput) (ret []*models.Image, err error) {
func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*ImageUpdateInput) (ret []*models.Image, err error) {
inputMaps := getUpdateInputMaps(ctx)
// Start the transaction and save the image
@@ -89,7 +89,7 @@ func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*models.Ima
return newRet, nil
}
func (r *mutationResolver) imageUpdate(ctx context.Context, input models.ImageUpdateInput, translator changesetTranslator) (*models.Image, error) {
func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInput, translator changesetTranslator) (*models.Image, error) {
imageID, err := strconv.Atoi(input.ID)
if err != nil {
return nil, fmt.Errorf("converting id: %w", err)

View File

@@ -7,10 +7,6 @@ import (
"github.com/stashapp/stash/internal/manager"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
"github.com/stashapp/stash/pkg/stashbox"
)
func (r *mutationResolver) SubmitStashBoxFingerprints(ctx context.Context, input StashBoxFingerprintSubmissionInput) (bool, error) {
@@ -19,23 +15,8 @@ func (r *mutationResolver) SubmitStashBoxFingerprints(ctx context.Context, input
return false, err
}
ids, err := stringslice.StringSliceToIntSlice(input.SceneIds)
if err != nil {
return false, err
}
client := r.newStashBoxClient(*b)
var scenes []*models.Scene
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
scenes, err = r.sceneService.FindMany(ctx, ids, scene.LoadStashIDs, scene.LoadFiles)
return err
}); err != nil {
return false, err
}
return client.SubmitFingerprints(ctx, scenes)
return client.SubmitStashBoxFingerprints(ctx, input.SceneIds)
}
func (r *mutationResolver) StashBoxBatchPerformerTag(ctx context.Context, input manager.StashBoxBatchTagInput) (string, error) {
@@ -88,76 +69,17 @@ func (r *mutationResolver) SubmitStashBoxSceneDraft(ctx context.Context, input S
logger.Errorf("Error getting scene cover: %v", err)
}
draft, err := r.makeSceneDraft(ctx, scene, cover)
if err != nil {
return err
if err := scene.LoadURLs(ctx, r.repository.Scene); err != nil {
return fmt.Errorf("loading scene URLs: %w", err)
}
res, err = client.SubmitSceneDraft(ctx, *draft)
res, err = client.SubmitSceneDraft(ctx, scene, cover)
return err
})
return res, err
}
func (r *mutationResolver) makeSceneDraft(ctx context.Context, s *models.Scene, cover []byte) (*stashbox.SceneDraft, error) {
if err := s.LoadURLs(ctx, r.repository.Scene); err != nil {
return nil, fmt.Errorf("loading scene URLs: %w", err)
}
if err := s.LoadStashIDs(ctx, r.repository.Scene); err != nil {
return nil, err
}
draft := &stashbox.SceneDraft{
Scene: s,
}
pqb := r.repository.Performer
sqb := r.repository.Studio
if s.StudioID != nil {
var err error
draft.Studio, err = sqb.Find(ctx, *s.StudioID)
if err != nil {
return nil, err
}
if draft.Studio == nil {
return nil, fmt.Errorf("studio with id %d not found", *s.StudioID)
}
if err := draft.Studio.LoadStashIDs(ctx, r.repository.Studio); err != nil {
return nil, err
}
}
// submit all file fingerprints
if err := s.LoadFiles(ctx, r.repository.Scene); err != nil {
return nil, err
}
scenePerformers, err := pqb.FindBySceneID(ctx, s.ID)
if err != nil {
return nil, err
}
for _, p := range scenePerformers {
if err := p.LoadStashIDs(ctx, pqb); err != nil {
return nil, err
}
}
draft.Performers = scenePerformers
draft.Tags, err = r.repository.Tag.FindBySceneID(ctx, s.ID)
if err != nil {
return nil, err
}
draft.Cover = cover
return draft, nil
}
func (r *mutationResolver) SubmitStashBoxPerformerDraft(ctx context.Context, input StashBoxDraftSubmissionInput) (*string, error) {
b, err := resolveStashBox(input.StashBoxIndex, input.StashBoxEndpoint)
if err != nil {
@@ -183,22 +105,7 @@ func (r *mutationResolver) SubmitStashBoxPerformerDraft(ctx context.Context, inp
return fmt.Errorf("performer with id %d not found", id)
}
pqb := r.repository.Performer
if err := performer.LoadAliases(ctx, pqb); err != nil {
return err
}
if err := performer.LoadURLs(ctx, pqb); err != nil {
return err
}
if err := performer.LoadStashIDs(ctx, pqb); err != nil {
return err
}
img, _ := pqb.GetImage(ctx, performer.ID)
res, err = client.SubmitPerformerDraft(ctx, performer, img)
res, err = client.SubmitPerformerDraft(ctx, performer)
return err
})

View File

@@ -33,7 +33,6 @@ func (r *mutationResolver) TagCreate(ctx context.Context, input TagCreateInput)
newTag := models.NewTag()
newTag.Name = input.Name
newTag.SortName = translator.string(input.SortName)
newTag.Aliases = models.NewRelatedStrings(input.Aliases)
newTag.Favorite = translator.bool(input.Favorite)
newTag.Description = translator.string(input.Description)
@@ -103,7 +102,6 @@ func (r *mutationResolver) TagUpdate(ctx context.Context, input TagUpdateInput)
updatedTag := models.NewTagPartial()
updatedTag.Name = translator.optionalString(input.Name, "name")
updatedTag.SortName = translator.optionalString(input.SortName, "sort_name")
updatedTag.Favorite = translator.optionalBool(input.Favorite, "favorite")
updatedTag.IgnoreAutoTag = translator.optionalBool(input.IgnoreAutoTag, "ignore_auto_tag")
updatedTag.Description = translator.optionalString(input.Description, "description")

View File

@@ -9,6 +9,7 @@ import (
"github.com/stashapp/stash/internal/manager/config"
"github.com/stashapp/stash/pkg/fsutil"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scraper/stashbox"
"golang.org/x/text/collate"
)
@@ -240,7 +241,7 @@ func makeConfigUIResult() map[string]interface{} {
func (r *queryResolver) ValidateStashBoxCredentials(ctx context.Context, input config.StashBoxInput) (*StashBoxValidationResult, error) {
box := models.StashBox{Endpoint: input.Endpoint, APIKey: input.APIKey}
client := r.newStashBoxClient(box)
client := stashbox.NewClient(box, r.stashboxRepository())
user, err := client.GetUser(ctx)

View File

@@ -32,11 +32,6 @@ func (r *queryResolver) FindPerformers(ctx context.Context, performerFilter *mod
}
}
// #5682 - convert JSON numbers to float64 or int64
if performerFilter != nil {
performerFilter.CustomFields = convertCustomFieldCriterionInputJSONNumbers(performerFilter.CustomFields)
}
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
var performers []*models.Performer
var err error

View File

@@ -4,31 +4,14 @@ import (
"context"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
)
func (r *queryResolver) FindSceneMarkers(ctx context.Context, sceneMarkerFilter *models.SceneMarkerFilterType, filter *models.FindFilterType, ids []string) (ret *FindSceneMarkersResultType, err error) {
idInts, err := stringslice.StringSliceToIntSlice(ids)
if err != nil {
return nil, err
}
func (r *queryResolver) FindSceneMarkers(ctx context.Context, sceneMarkerFilter *models.SceneMarkerFilterType, filter *models.FindFilterType) (ret *FindSceneMarkersResultType, err error) {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
var sceneMarkers []*models.SceneMarker
var err error
var total int
if len(idInts) > 0 {
sceneMarkers, err = r.repository.SceneMarker.FindMany(ctx, idInts)
total = len(sceneMarkers)
} else {
sceneMarkers, total, err = r.repository.SceneMarker.Query(ctx, sceneMarkerFilter, filter)
}
sceneMarkers, total, err := r.repository.SceneMarker.Query(ctx, sceneMarkerFilter, filter)
if err != nil {
return err
}
ret = &FindSceneMarkersResultType{
Count: total,
SceneMarkers: sceneMarkers,

View File

@@ -4,11 +4,15 @@ import (
"context"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/internal/manager"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/scraper/stashbox"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
)
@@ -30,7 +34,7 @@ func (r *queryResolver) ScrapePerformerURL(ctx context.Context, url string) (*mo
return marshalScrapedPerformer(content)
}
func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string, query string) ([]*models.ScrapedScene, error) {
func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string, query string) ([]*scraper.ScrapedScene, error) {
if query == "" {
return nil, nil
}
@@ -45,10 +49,119 @@ func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string,
return nil, err
}
filterSceneTags(ret)
return ret, nil
}
func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models.ScrapedScene, error) {
func compileRegexps(patterns []string) []*regexp.Regexp {
excludePatterns := patterns
var excludeRegexps []*regexp.Regexp
for _, excludePattern := range excludePatterns {
reg, err := regexp.Compile(strings.ToLower(excludePattern))
if err != nil {
logger.Errorf("Invalid tag exclusion pattern: %v", err)
} else {
excludeRegexps = append(excludeRegexps, reg)
}
}
return excludeRegexps
}
// filterSceneTags removes tags matching excluded tag patterns from the provided scraped scenes
func filterTags(excludeRegexps []*regexp.Regexp, tags []*models.ScrapedTag) (newTags []*models.ScrapedTag, ignoredTags []string) {
if len(excludeRegexps) == 0 {
return tags, nil
}
for _, t := range tags {
ignore := false
for _, reg := range excludeRegexps {
if reg.MatchString(strings.ToLower(t.Name)) {
ignore = true
ignoredTags = sliceutil.AppendUnique(ignoredTags, t.Name)
break
}
}
if !ignore {
newTags = append(newTags, t)
}
}
return
}
// filterSceneTags removes tags matching excluded tag patterns from the provided scraped scenes
func filterSceneTags(scenes []*scraper.ScrapedScene) {
excludeRegexps := compileRegexps(manager.GetInstance().Config.GetScraperExcludeTagPatterns())
var ignoredTags []string
for _, s := range scenes {
var ignored []string
s.Tags, ignored = filterTags(excludeRegexps, s.Tags)
ignoredTags = sliceutil.AppendUniques(ignoredTags, ignored)
}
if len(ignoredTags) > 0 {
logger.Debugf("Scraping ignored tags: %s", strings.Join(ignoredTags, ", "))
}
}
// filterGalleryTags removes tags matching excluded tag patterns from the provided scraped galleries
func filterGalleryTags(g []*scraper.ScrapedGallery) {
excludeRegexps := compileRegexps(manager.GetInstance().Config.GetScraperExcludeTagPatterns())
var ignoredTags []string
for _, s := range g {
var ignored []string
s.Tags, ignored = filterTags(excludeRegexps, s.Tags)
ignoredTags = sliceutil.AppendUniques(ignoredTags, ignored)
}
if len(ignoredTags) > 0 {
logger.Debugf("Scraping ignored tags: %s", strings.Join(ignoredTags, ", "))
}
}
// filterGalleryTags removes tags matching excluded tag patterns from the provided scraped galleries
func filterPerformerTags(p []*models.ScrapedPerformer) {
excludeRegexps := compileRegexps(manager.GetInstance().Config.GetScraperExcludeTagPatterns())
var ignoredTags []string
for _, s := range p {
var ignored []string
s.Tags, ignored = filterTags(excludeRegexps, s.Tags)
ignoredTags = sliceutil.AppendUniques(ignoredTags, ignored)
}
if len(ignoredTags) > 0 {
logger.Debugf("Scraping ignored tags: %s", strings.Join(ignoredTags, ", "))
}
}
// filterGroupTags removes tags matching excluded tag patterns from the provided scraped movies
func filterGroupTags(p []*models.ScrapedMovie) {
excludeRegexps := compileRegexps(manager.GetInstance().Config.GetScraperExcludeTagPatterns())
var ignoredTags []string
for _, s := range p {
var ignored []string
s.Tags, ignored = filterTags(excludeRegexps, s.Tags)
ignoredTags = sliceutil.AppendUniques(ignoredTags, ignored)
}
if len(ignoredTags) > 0 {
logger.Debugf("Scraping ignored tags: %s", strings.Join(ignoredTags, ", "))
}
}
func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*scraper.ScrapedScene, error) {
content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeScene)
if err != nil {
return nil, err
@@ -59,10 +172,14 @@ func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models
return nil, err
}
if ret != nil {
filterSceneTags([]*scraper.ScrapedScene{ret})
}
return ret, nil
}
func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*models.ScrapedGallery, error) {
func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*scraper.ScrapedGallery, error) {
content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeGallery)
if err != nil {
return nil, err
@@ -73,16 +190,11 @@ func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*mode
return nil, err
}
return ret, nil
}
func (r *queryResolver) ScrapeImageURL(ctx context.Context, url string) (*models.ScrapedImage, error) {
content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeImage)
if err != nil {
return nil, err
if ret != nil {
filterGalleryTags([]*scraper.ScrapedGallery{ret})
}
return marshalScrapedImage(content)
return ret, nil
}
func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models.ScrapedMovie, error) {
@@ -96,6 +208,8 @@ func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models
return nil, err
}
filterGroupTags([]*models.ScrapedMovie{ret})
return ret, nil
}
@@ -110,6 +224,8 @@ func (r *queryResolver) ScrapeGroupURL(ctx context.Context, url string) (*models
return nil, err
}
filterGroupTags([]*models.ScrapedMovie{ret})
// convert to scraped group
group := &models.ScrapedGroup{
StoredID: ret.StoredID,
@@ -130,8 +246,8 @@ func (r *queryResolver) ScrapeGroupURL(ctx context.Context, url string) (*models
return group, nil
}
func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.Source, input ScrapeSingleSceneInput) ([]*models.ScrapedScene, error) {
var ret []*models.ScrapedScene
func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.Source, input ScrapeSingleSceneInput) ([]*scraper.ScrapedScene, error) {
var ret []*scraper.ScrapedScene
var sceneID int
if input.SceneID != nil {
@@ -183,14 +299,9 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.So
switch {
case input.SceneID != nil:
var fps []models.Fingerprints
fps, err = r.getScenesFingerprints(ctx, []int{sceneID})
if err != nil {
return nil, err
}
ret, err = client.FindSceneByFingerprints(ctx, fps[0])
ret, err = client.FindStashBoxSceneByFingerprints(ctx, sceneID)
case input.Query != nil:
ret, err = client.QueryScene(ctx, *input.Query)
ret, err = client.QueryStashBoxScene(ctx, *input.Query)
default:
return nil, fmt.Errorf("%w: scene_id or query must be set", ErrInput)
}
@@ -198,19 +309,16 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source scraper.So
if err != nil {
return nil, err
}
// TODO - this should happen after any scene is scraped
if err := r.matchScenesRelationships(ctx, ret, *source.StashBoxEndpoint); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("%w: scraper_id or stash_box_index must be set", ErrInput)
}
filterSceneTags(ret)
return ret, nil
}
func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source scraper.Source, input ScrapeMultiScenesInput) ([][]*models.ScrapedScene, error) {
func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source scraper.Source, input ScrapeMultiScenesInput) ([][]*scraper.ScrapedScene, error) {
if source.ScraperID != nil {
return nil, ErrNotImplemented
} else if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
@@ -226,89 +334,12 @@ func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source scraper.So
return nil, err
}
fps, err := r.getScenesFingerprints(ctx, sceneIDs)
if err != nil {
return nil, err
}
ret, err := client.FindScenesByFingerprints(ctx, fps)
if err != nil {
return nil, err
}
// match relationships - this mutates the existing scenes so we can
// just flatten the slice and pass it in
flat := sliceutil.Flatten(ret)
if err := r.matchScenesRelationships(ctx, flat, *source.StashBoxEndpoint); err != nil {
return nil, err
}
return ret, nil
return client.FindStashBoxScenesByFingerprints(ctx, sceneIDs)
}
return nil, errors.New("scraper_id or stash_box_index must be set")
}
func (r *queryResolver) getScenesFingerprints(ctx context.Context, ids []int) ([]models.Fingerprints, error) {
fingerprints := make([]models.Fingerprints, len(ids))
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
qb := r.repository.Scene
for i, sceneID := range ids {
scene, err := qb.Find(ctx, sceneID)
if err != nil {
return err
}
if scene == nil {
return fmt.Errorf("scene with id %d not found", sceneID)
}
if err := scene.LoadFiles(ctx, qb); err != nil {
return err
}
var sceneFPs models.Fingerprints
for _, f := range scene.Files.List() {
sceneFPs = append(sceneFPs, f.Fingerprints...)
}
fingerprints[i] = sceneFPs
}
return nil
}); err != nil {
return nil, err
}
return fingerprints, nil
}
// matchSceneRelationships accepts scraped scenes and attempts to match its relationships to existing stash models.
func (r *queryResolver) matchScenesRelationships(ctx context.Context, ss []*models.ScrapedScene, endpoint string) error {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
matcher := match.SceneRelationships{
PerformerFinder: r.repository.Performer,
TagFinder: r.repository.Tag,
StudioFinder: r.repository.Studio,
}
for _, s := range ss {
if err := matcher.MatchRelationships(ctx, s, endpoint); err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
return nil
}
func (r *queryResolver) ScrapeSingleStudio(ctx context.Context, source scraper.Source, input ScrapeSingleStudioInput) ([]*models.ScrapedStudio, error) {
if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
b, err := resolveStashBox(source.StashBoxIndex, source.StashBoxEndpoint)
@@ -319,7 +350,7 @@ func (r *queryResolver) ScrapeSingleStudio(ctx context.Context, source scraper.S
client := r.newStashBoxClient(*b)
var ret []*models.ScrapedStudio
out, err := client.FindStudio(ctx, *input.Query)
out, err := client.FindStashBoxStudio(ctx, *input.Query)
if err != nil {
return nil, err
@@ -328,17 +359,6 @@ func (r *queryResolver) ScrapeSingleStudio(ctx context.Context, source scraper.S
}
if len(ret) > 0 {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
for _, studio := range ret {
if err := match.ScrapedStudioHierarchy(ctx, r.repository.Studio, studio, *source.StashBoxEndpoint); err != nil {
return err
}
}
return nil
}); err != nil {
return nil, err
}
return ret, nil
}
@@ -384,33 +404,29 @@ func (r *queryResolver) ScrapeSinglePerformer(ctx context.Context, source scrape
client := r.newStashBoxClient(*b)
var query string
var res []*stashbox.StashBoxPerformerQueryResult
switch {
case input.PerformerID != nil:
names, err := r.findPerformerNames(ctx, []string{*input.PerformerID})
if err != nil {
return nil, err
}
query = names[0]
res, err = client.FindStashBoxPerformersByNames(ctx, []string{*input.PerformerID})
case input.Query != nil:
query = *input.Query
res, err = client.QueryStashBoxPerformer(ctx, *input.Query)
default:
return nil, ErrNotImplemented
}
if query == "" {
return nil, nil
}
ret, err = client.QueryPerformer(ctx, query)
if err != nil {
return nil, err
}
if len(res) > 0 {
ret = res[0].Results
}
default:
return nil, errors.New("scraper_id or stash_box_index must be set")
}
filterPerformerTags(ret)
return ret, nil
}
@@ -418,11 +434,6 @@ func (r *queryResolver) ScrapeMultiPerformers(ctx context.Context, source scrape
if source.ScraperID != nil {
return nil, ErrNotImplemented
} else if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
names, err := r.findPerformerNames(ctx, input.PerformerIds)
if err != nil {
return nil, err
}
b, err := resolveStashBox(source.StashBoxIndex, source.StashBoxEndpoint)
if err != nil {
return nil, err
@@ -430,40 +441,14 @@ func (r *queryResolver) ScrapeMultiPerformers(ctx context.Context, source scrape
client := r.newStashBoxClient(*b)
return client.QueryPerformers(ctx, names)
return client.FindStashBoxPerformersByPerformerNames(ctx, input.PerformerIds)
}
return nil, errors.New("scraper_id or stash_box_index must be set")
}
func (r *queryResolver) findPerformerNames(ctx context.Context, performerIDs []string) ([]string, error) {
ids, err := stringslice.StringSliceToIntSlice(performerIDs)
if err != nil {
return nil, err
}
names := make([]string, len(ids))
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
p, err := r.repository.Performer.FindMany(ctx, ids)
if err != nil {
return err
}
for i, pp := range p {
names[i] = pp.Name
}
return nil
}); err != nil {
return nil, err
}
return names, nil
}
func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.Source, input ScrapeSingleGalleryInput) ([]*models.ScrapedGallery, error) {
var ret []*models.ScrapedGallery
func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.Source, input ScrapeSingleGalleryInput) ([]*scraper.ScrapedGallery, error) {
var ret []*scraper.ScrapedGallery
if source.StashBoxIndex != nil || source.StashBoxEndpoint != nil {
return nil, ErrNotSupported
@@ -502,42 +487,10 @@ func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.
return nil, ErrNotImplemented
}
filterGalleryTags(ret)
return ret, nil
}
func (r *queryResolver) ScrapeSingleImage(ctx context.Context, source scraper.Source, input ScrapeSingleImageInput) ([]*models.ScrapedImage, error) {
if source.StashBoxIndex != nil {
return nil, ErrNotSupported
}
if source.ScraperID == nil {
return nil, fmt.Errorf("%w: scraper_id must be set", ErrInput)
}
var c scraper.ScrapedContent
switch {
case input.ImageID != nil:
imageID, err := strconv.Atoi(*input.ImageID)
if err != nil {
return nil, fmt.Errorf("%w: image id is not an integer: '%s'", ErrInput, *input.ImageID)
}
c, err = r.scraperCache().ScrapeID(ctx, *source.ScraperID, imageID, scraper.ScrapeContentTypeImage)
if err != nil {
return nil, err
}
return marshalScrapedImages([]scraper.ScrapedContent{c})
case input.ImageInput != nil:
c, err := r.scraperCache().ScrapeFragment(ctx, *source.ScraperID, scraper.Input{Image: input.ImageInput})
if err != nil {
return nil, err
}
return marshalScrapedImages([]scraper.ScrapedContent{c})
default:
return nil, ErrNotImplemented
}
}
func (r *queryResolver) ScrapeSingleMovie(ctx context.Context, source scraper.Source, input ScrapeSingleMovieInput) ([]*models.ScrapedMovie, error) {
return nil, ErrNotSupported
}

View File

@@ -9,8 +9,8 @@ import (
// marshalScrapedScenes converts ScrapedContent into ScrapedScene. If conversion fails, an
// error is returned to the caller.
func marshalScrapedScenes(content []scraper.ScrapedContent) ([]*models.ScrapedScene, error) {
var ret []*models.ScrapedScene
func marshalScrapedScenes(content []scraper.ScrapedContent) ([]*scraper.ScrapedScene, error) {
var ret []*scraper.ScrapedScene
for _, c := range content {
if c == nil {
// graphql schema requires scenes to be non-nil
@@ -18,9 +18,9 @@ func marshalScrapedScenes(content []scraper.ScrapedContent) ([]*models.ScrapedSc
}
switch s := c.(type) {
case *models.ScrapedScene:
case *scraper.ScrapedScene:
ret = append(ret, s)
case models.ScrapedScene:
case scraper.ScrapedScene:
ret = append(ret, &s)
default:
return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedScene", models.ErrConversion)
@@ -55,8 +55,8 @@ func marshalScrapedPerformers(content []scraper.ScrapedContent) ([]*models.Scrap
// marshalScrapedGalleries converts ScrapedContent into ScrapedGallery. If
// conversion fails, an error is returned.
func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*models.ScrapedGallery, error) {
var ret []*models.ScrapedGallery
func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*scraper.ScrapedGallery, error) {
var ret []*scraper.ScrapedGallery
for _, c := range content {
if c == nil {
// graphql schema requires galleries to be non-nil
@@ -64,9 +64,9 @@ func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*models.Scrape
}
switch g := c.(type) {
case *models.ScrapedGallery:
case *scraper.ScrapedGallery:
ret = append(ret, g)
case models.ScrapedGallery:
case scraper.ScrapedGallery:
ret = append(ret, &g)
default:
return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedGallery", models.ErrConversion)
@@ -76,27 +76,6 @@ func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*models.Scrape
return ret, nil
}
func marshalScrapedImages(content []scraper.ScrapedContent) ([]*models.ScrapedImage, error) {
var ret []*models.ScrapedImage
for _, c := range content {
if c == nil {
// graphql schema requires images to be non-nil
continue
}
switch g := c.(type) {
case *models.ScrapedImage:
ret = append(ret, g)
case models.ScrapedImage:
ret = append(ret, &g)
default:
return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedImage", models.ErrConversion)
}
}
return ret, nil
}
// marshalScrapedMovies converts ScrapedContent into ScrapedMovie. If conversion
// fails, an error is returned.
func marshalScrapedMovies(content []scraper.ScrapedContent) ([]*models.ScrapedMovie, error) {
@@ -131,7 +110,7 @@ func marshalScrapedPerformer(content scraper.ScrapedContent) (*models.ScrapedPer
}
// marshalScrapedScene will marshal a single scraped scene
func marshalScrapedScene(content scraper.ScrapedContent) (*models.ScrapedScene, error) {
func marshalScrapedScene(content scraper.ScrapedContent) (*scraper.ScrapedScene, error) {
s, err := marshalScrapedScenes([]scraper.ScrapedContent{content})
if err != nil {
return nil, err
@@ -141,7 +120,7 @@ func marshalScrapedScene(content scraper.ScrapedContent) (*models.ScrapedScene,
}
// marshalScrapedGallery will marshal a single scraped gallery
func marshalScrapedGallery(content scraper.ScrapedContent) (*models.ScrapedGallery, error) {
func marshalScrapedGallery(content scraper.ScrapedContent) (*scraper.ScrapedGallery, error) {
g, err := marshalScrapedGalleries([]scraper.ScrapedContent{content})
if err != nil {
return nil, err
@@ -150,16 +129,6 @@ func marshalScrapedGallery(content scraper.ScrapedContent) (*models.ScrapedGalle
return g[0], nil
}
// marshalScrapedImage will marshal a single scraped image
func marshalScrapedImage(content scraper.ScrapedContent) (*models.ScrapedImage, error) {
g, err := marshalScrapedImages([]scraper.ScrapedContent{content})
if err != nil {
return nil, err
}
return g[0], nil
}
// marshalScrapedMovie will marshal a single scraped movie
func marshalScrapedMovie(content scraper.ScrapedContent) (*models.ScrapedMovie, error) {
m, err := marshalScrapedMovies([]scraper.ScrapedContent{content})

View File

@@ -41,11 +41,10 @@ import (
)
const (
loginEndpoint = "/login"
loginLocaleEndpoint = loginEndpoint + "/locale"
logoutEndpoint = "/logout"
gqlEndpoint = "/graphql"
playgroundEndpoint = "/playground"
loginEndpoint = "/login"
logoutEndpoint = "/logout"
gqlEndpoint = "/graphql"
playgroundEndpoint = "/playground"
)
type Server struct {
@@ -229,7 +228,6 @@ func Initialize() (*Server, error) {
r.Get(loginEndpoint, handleLogin())
r.Post(loginEndpoint, handleLoginPost())
r.Get(logoutEndpoint, handleLogout())
r.Get(loginLocaleEndpoint, handleLoginLocale(cfg))
r.HandleFunc(loginEndpoint+"/*", func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.TrimPrefix(r.URL.Path, loginEndpoint)
w.Header().Set("Cache-Control", "no-cache")

View File

@@ -17,11 +17,7 @@ import (
"github.com/stashapp/stash/ui"
)
const (
returnURLParam = "returnURL"
defaultLocale = "en-GB"
)
const returnURLParam = "returnURL"
func getLoginPage() []byte {
data, err := fs.ReadFile(ui.LoginUIBox, "login.html")
@@ -62,47 +58,6 @@ func serveLoginPage(w http.ResponseWriter, r *http.Request, returnURL string, lo
utils.ServeStaticContent(w, r, buffer.Bytes())
}
func handleLoginLocale(cfg *config.Config) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
// get the locale from the config
lang := cfg.GetLanguage()
if lang == "" {
lang = defaultLocale
}
data, err := getLoginLocale(lang)
if err != nil {
logger.Debugf("Failed to load login locale file for language %s: %v", lang, err)
// try again with the default language
if lang != defaultLocale {
data, err = getLoginLocale(defaultLocale)
if err != nil {
logger.Errorf("Failed to load login locale file for default language %s: %v", defaultLocale, err)
}
}
// if there's still an error, response with an internal server error
if err != nil {
http.Error(w, "Failed to load login locale file", http.StatusInternalServerError)
return
}
}
// write a script to set the locale string map as a global variable
localeScript := fmt.Sprintf("var localeStrings = %s;", data)
w.Header().Set("Content-Type", "application/javascript")
_, _ = w.Write([]byte(localeScript))
}
}
func getLoginLocale(lang string) ([]byte, error) {
data, err := fs.ReadFile(ui.LoginUIBox, "locales/"+lang+".json")
if err != nil {
return nil, err
}
return data, nil
}
func handleLogin() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
returnURL := r.URL.Query().Get(returnURLParam)
@@ -123,26 +78,31 @@ func handleLogin() http.HandlerFunc {
func handleLoginPost() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
url := r.FormValue(returnURLParam)
if url == "" {
url = getProxyPrefix(r) + "/"
}
err := manager.GetInstance().SessionStore.Login(w, r)
if err != nil {
// always log the error
logger.Errorf("Error logging in: %v from IP: %s", err, r.RemoteAddr)
logger.Errorf("Error logging in: %v", err)
}
var invalidCredentialsError *session.InvalidCredentialsError
if errors.As(err, &invalidCredentialsError) {
http.Error(w, "Username or password is invalid", http.StatusUnauthorized)
// serve login page with an error
serveLoginPage(w, r, url, "Username or password is invalid")
return
}
if err != nil {
// don't expose the error to the user
http.Error(w, "An unexpected error occurred. See logs", http.StatusInternalServerError)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
http.Redirect(w, r, url, http.StatusFound)
}
}

View File

@@ -4,14 +4,13 @@ import (
"fmt"
"strings"
"github.com/stashapp/stash/internal/manager"
"github.com/stashapp/stash/internal/manager/config"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/stashbox"
"github.com/stashapp/stash/pkg/scraper/stashbox"
)
func (r *Resolver) newStashBoxClient(box models.StashBox) *stashbox.Client {
return stashbox.NewClient(box, stashbox.ExcludeTagPatterns(manager.GetInstance().Config.GetScraperExcludeTagPatterns()))
return stashbox.NewClient(box, r.stashboxRepository())
}
func resolveStashBoxFn(indexField, endpointField string) func(index *int, endpoint *string) (*models.StashBox, error) {

View File

@@ -13,6 +13,7 @@ import (
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/txn"
"github.com/stashapp/stash/pkg/utils"
@@ -31,7 +32,7 @@ func (e *MultipleMatchesFoundError) Error() string {
}
type SceneScraper interface {
ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error)
ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error)
}
type SceneUpdatePostHookExecutor interface {
@@ -95,7 +96,7 @@ func (t *SceneIdentifier) Identify(ctx context.Context, scene *models.Scene) err
}
type scrapeResult struct {
result *models.ScrapedScene
result *scraper.ScrapedScene
source ScraperSource
}
@@ -373,7 +374,7 @@ func getFieldOptions(options []MetadataOptions) map[string]*FieldOptions {
return ret
}
func getScenePartial(scene *models.Scene, scraped *models.ScrapedScene, fieldOptions map[string]*FieldOptions, setOrganized bool) models.ScenePartial {
func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOptions map[string]*FieldOptions, setOrganized bool) models.ScenePartial {
partial := models.ScenePartial{}
if scraped.Title != nil && (scene.Title != *scraped.Title) {

View File

@@ -10,6 +10,7 @@ import (
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/models/mocks"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
@@ -18,10 +19,10 @@ var testCtx = context.Background()
type mockSceneScraper struct {
errIDs []int
results map[int][]*models.ScrapedScene
results map[int][]*scraper.ScrapedScene
}
func (s mockSceneScraper) ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
func (s mockSceneScraper) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
if slices.Contains(s.errIDs, sceneID) {
return nil, errors.New("scrape scene error")
}
@@ -69,7 +70,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
{
Scraper: mockSceneScraper{
errIDs: []int{errID1},
results: map[int][]*models.ScrapedScene{
results: map[int][]*scraper.ScrapedScene{
found1ID: {{
Title: &scrapedTitle,
}},
@@ -79,7 +80,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
{
Scraper: mockSceneScraper{
errIDs: []int{errID2},
results: map[int][]*models.ScrapedScene{
results: map[int][]*scraper.ScrapedScene{
found2ID: {{
Title: &scrapedTitle,
}},
@@ -249,7 +250,7 @@ func TestSceneIdentifier_modifyScene(t *testing.T) {
StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
},
&scrapeResult{
result: &models.ScrapedScene{},
result: &scraper.ScrapedScene{},
source: ScraperSource{
Options: defaultOptions,
},
@@ -385,14 +386,14 @@ func Test_getScenePartial(t *testing.T) {
Mode: models.RelationshipUpdateModeSet,
}
scrapedScene := &models.ScrapedScene{
scrapedScene := &scraper.ScrapedScene{
Title: &scrapedTitle,
Date: &scrapedDate,
Details: &scrapedDetails,
URLs: []string{scrapedURL},
}
scrapedUnchangedScene := &models.ScrapedScene{
scrapedUnchangedScene := &scraper.ScrapedScene{
Title: &originalTitle,
Date: &originalDate,
Details: &originalDetails,
@@ -422,7 +423,7 @@ func Test_getScenePartial(t *testing.T) {
type args struct {
scene *models.Scene
scraped *models.ScrapedScene
scraped *scraper.ScrapedScene
fieldOptions map[string]*FieldOptions
setOrganized bool
}

View File

@@ -9,6 +9,7 @@ import (
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/models/mocks"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/utils"
"github.com/stretchr/testify/mock"
)
@@ -124,7 +125,7 @@ func Test_sceneRelationships_studio(t *testing.T) {
source: ScraperSource{
RemoteSite: "endpoint",
},
result: &models.ScrapedScene{
result: &scraper.ScrapedScene{
Studio: tt.result,
},
}
@@ -314,7 +315,7 @@ func Test_sceneRelationships_performers(t *testing.T) {
tr.scene = tt.scene
tr.fieldOptions["performers"] = tt.fieldOptions
tr.result = &scrapeResult{
result: &models.ScrapedScene{
result: &scraper.ScrapedScene{
Performers: tt.scraped,
},
}
@@ -506,7 +507,7 @@ func Test_sceneRelationships_tags(t *testing.T) {
tr.scene = tt.scene
tr.fieldOptions["tags"] = tt.fieldOptions
tr.result = &scrapeResult{
result: &models.ScrapedScene{
result: &scraper.ScrapedScene{
Tags: tt.scraped,
},
}
@@ -726,7 +727,7 @@ func Test_sceneRelationships_stashIDs(t *testing.T) {
source: ScraperSource{
RemoteSite: tt.endpoint,
},
result: &models.ScrapedScene{
result: &scraper.ScrapedScene{
RemoteSiteID: tt.remoteSiteID,
},
}
@@ -826,7 +827,7 @@ func Test_sceneRelationships_cover(t *testing.T) {
ID: tt.sceneID,
}
tr.result = &scrapeResult{
result: &models.ScrapedScene{
result: &scraper.ScrapedScene{
Image: tt.image,
},
}

View File

@@ -284,7 +284,7 @@ const (
// slice default values
var (
defaultVideoExtensions = []string{"m4v", "mp4", "mov", "wmv", "avi", "mpg", "mpeg", "rmvb", "rm", "flv", "asf", "mkv", "webm", "f4v"}
defaultVideoExtensions = []string{"m4v", "mp4", "mov", "wmv", "avi", "mpg", "mpeg", "rmvb", "rm", "flv", "asf", "mkv", "webm"}
defaultImageExtensions = []string{"png", "jpg", "jpeg", "gif", "webp"}
defaultGalleryExtensions = []string{"zip", "cbz"}
defaultMenuItems = []string{"scenes", "images", "movies", "markers", "galleries", "performers", "studios", "tags"}
@@ -1105,10 +1105,9 @@ func stashBoxValidate(str string) bool {
}
type StashBoxInput struct {
Endpoint string `json:"endpoint"`
APIKey string `json:"api_key"`
Name string `json:"name"`
MaxRequestsPerMinute int `json:"max_requests_per_minute"`
Endpoint string `json:"endpoint"`
APIKey string `json:"api_key"`
Name string `json:"name"`
}
func (i *Config) ValidateStashBoxes(boxes []*StashBoxInput) error {

View File

@@ -14,9 +14,6 @@ type SceneService interface {
AssignFile(ctx context.Context, sceneID int, fileID models.FileID) error
Merge(ctx context.Context, sourceIDs []int, destinationID int, fileDeleter *scene.FileDeleter, options scene.MergeOptions) error
Destroy(ctx context.Context, scene *models.Scene, fileDeleter *scene.FileDeleter, deleteGenerated, deleteFile bool) error
FindMany(ctx context.Context, ids []int, load ...scene.LoadRelationshipOption) ([]*models.Scene, error)
sceneFingerprintGetter
}
type ImageService interface {

View File

@@ -42,8 +42,8 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error
logger.Infof("Migrating database from %d to %d", schemaInfo.CurrentSchemaVersion, schemaInfo.RequiredSchemaVersion)
// set the number of tasks = backup + required steps + optimise
progress.SetTotal(int(schemaInfo.StepsRequired + 2))
// set the number of tasks = required steps + optimise
progress.SetTotal(int(schemaInfo.StepsRequired + 1))
database := s.Database
@@ -61,20 +61,12 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error
}
}
progress.ExecuteTask("Backing up database", func() {
defer progress.Increment()
// perform database backup
err = database.Backup(backupPath)
})
if err != nil {
// perform database backup
if err := database.Backup(backupPath); err != nil {
return fmt.Errorf("error backing up database: %s", err)
}
err = s.runMigrations(ctx, progress)
if err != nil {
if err := s.runMigrations(ctx, progress); err != nil {
errStr := fmt.Sprintf("error performing migration: %s", err)
// roll back to the backed up version
@@ -95,11 +87,6 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error
}
}
// reinitialise the database
if err := database.ReInitialise(); err != nil {
return fmt.Errorf("error reinitialising database: %s", err)
}
logger.Infof("Database migration complete")
return nil
@@ -137,8 +124,6 @@ func (s *MigrateJob) runMigrations(ctx context.Context, progress *job.Progress)
defer m.Close()
logger.Info("Running migrations")
for {
currentSchemaVersion := m.CurrentSchemaVersion()
targetSchemaVersion := m.RequiredSchemaVersion()
@@ -159,15 +144,21 @@ func (s *MigrateJob) runMigrations(ctx context.Context, progress *job.Progress)
progress.Increment()
}
// perform post-migrate analyze using the migrator connection
// reinitialise the database
if err := database.ReInitialise(); err != nil {
return fmt.Errorf("error reinitialising database: %s", err)
}
// optimise the database
progress.ExecuteTask("Optimising database", func() {
err = m.PostMigrate(ctx)
progress.Increment()
err = database.Optimise(ctx)
})
if err != nil {
return fmt.Errorf("error optimising database: %s", err)
}
progress.Increment()
return nil
}

View File

@@ -105,11 +105,11 @@ func (t *GenerateMarkersTask) generateSceneMarkers(ctx context.Context) {
func (t *GenerateMarkersTask) generateMarker(videoFile *models.VideoFile, scene *models.Scene, sceneMarker *models.SceneMarker) {
sceneHash := scene.GetHash(t.fileNamingAlgorithm)
seconds := float64(sceneMarker.Seconds)
seconds := int(sceneMarker.Seconds)
g := t.generator
if err := g.MarkerPreviewVideo(context.TODO(), videoFile.Path, sceneHash, seconds, sceneMarker.EndSeconds, instance.Config.GetPreviewAudio()); err != nil {
if err := g.MarkerPreviewVideo(context.TODO(), videoFile.Path, sceneHash, seconds, instance.Config.GetPreviewAudio()); err != nil {
logger.Errorf("[generator] failed to generate marker video: %v", err)
logErrorOutput(err)
}

View File

@@ -9,13 +9,11 @@ import (
"github.com/stashapp/stash/internal/identify"
"github.com/stashapp/stash/pkg/job"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/scraper/stashbox"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
"github.com/stashapp/stash/pkg/stashbox"
"github.com/stashapp/stash/pkg/txn"
)
var ErrInput = errors.New("invalid request input")
@@ -171,20 +169,12 @@ func (j *IdentifyJob) getSources() ([]identify.ScraperSource, error) {
var src identify.ScraperSource
if stashBox != nil {
matcher := match.SceneRelationships{
PerformerFinder: instance.Repository.Performer,
TagFinder: instance.Repository.Tag,
StudioFinder: instance.Repository.Studio,
}
stashboxRepository := stashbox.NewRepository(instance.Repository)
src = identify.ScraperSource{
Name: "stash-box: " + stashBox.Endpoint,
Scraper: stashboxSource{
Client: stashbox.NewClient(*stashBox, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns())),
endpoint: stashBox.Endpoint,
txnManager: instance.Repository.TxnManager,
sceneFingerprintGetter: instance.SceneService,
matcher: matcher,
stashbox.NewClient(*stashBox, stashboxRepository),
stashBox.Endpoint,
},
RemoteSite: stashBox.Endpoint,
}
@@ -257,42 +247,14 @@ func resolveStashBox(sb []*models.StashBox, source scraper.Source) (*models.Stas
type stashboxSource struct {
*stashbox.Client
endpoint string
txnManager models.TxnManager
sceneFingerprintGetter sceneFingerprintGetter
matcher match.SceneRelationships
}
type sceneFingerprintGetter interface {
GetScenesFingerprints(ctx context.Context, ids []int) ([]models.Fingerprints, error)
}
func (s stashboxSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
var fps []models.Fingerprints
if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
var err error
fps, err = s.sceneFingerprintGetter.GetScenesFingerprints(ctx, []int{sceneID})
return err
}); err != nil {
return nil, fmt.Errorf("error getting scene fingerprints: %w", err)
}
results, err := s.FindSceneByFingerprints(ctx, fps[0])
func (s stashboxSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
results, err := s.FindStashBoxSceneByFingerprints(ctx, sceneID)
if err != nil {
return nil, fmt.Errorf("error querying stash-box using scene ID %d: %w", sceneID, err)
}
if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
for _, ret := range results {
if err := s.matcher.MatchRelationships(ctx, ret, s.endpoint); err != nil {
return err
}
}
return nil
}); err != nil {
return nil, fmt.Errorf("error matching scene relationships: %w", err)
}
if len(results) > 0 {
return results, nil
}
@@ -309,7 +271,7 @@ type scraperSource struct {
scraperID string
}
func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
content, err := s.cache.ScrapeID(ctx, s.scraperID, sceneID, scraper.ScrapeContentTypeScene)
if err != nil {
return nil, err
@@ -320,8 +282,8 @@ func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*models
return nil, nil
}
if scene, ok := content.(models.ScrapedScene); ok {
return []*models.ScrapedScene{&scene}, nil
if scene, ok := content.(scraper.ScrapedScene); ok {
return []*scraper.ScrapedScene{&scene}, nil
}
return nil, errors.New("could not convert content to scene")

View File

@@ -709,11 +709,6 @@ func (t *ImportTask) ImportScenes(ctx context.Context) {
return err
}
// skip importing markers if the scene was not created
if sceneImporter.ID == 0 {
return nil
}
// import the scene markers
for _, m := range sceneJSON.Markers {
markerImporter := &scene.MarkerImporter{

View File

@@ -6,11 +6,9 @@ import (
"strconv"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/performer"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/stashbox"
"github.com/stashapp/stash/pkg/scraper/stashbox"
"github.com/stashapp/stash/pkg/studio"
)
@@ -96,7 +94,8 @@ func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*mode
r := instance.Repository
client := stashbox.NewClient(*t.box, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns()))
stashboxRepository := stashbox.NewRepository(r)
client := stashbox.NewClient(*t.box, stashboxRepository)
if t.refresh {
var remoteID string
@@ -119,19 +118,7 @@ func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*mode
return nil, err
}
if remoteID != "" {
performer, err = client.FindPerformerByID(ctx, remoteID)
if performer != nil && performer.RemoteMergedIntoId != nil {
mergedPerformer, err := t.handleMergedPerformer(ctx, performer, client)
if err != nil {
return nil, err
}
if mergedPerformer != nil {
logger.Infof("Performer id %s merged into %s, updating local performer", remoteID, *performer.RemoteMergedIntoId)
performer = mergedPerformer
}
}
performer, err = client.FindStashBoxPerformerByID(ctx, remoteID)
}
} else {
var name string
@@ -140,35 +127,12 @@ func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*mode
} else {
name = t.performer.Name
}
performer, err = client.FindPerformerByName(ctx, name)
}
if performer != nil {
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
return match.ScrapedPerformer(ctx, r.Performer, performer, t.box.Endpoint)
}); err != nil {
return nil, err
}
performer, err = client.FindStashBoxPerformerByName(ctx, name)
}
return performer, err
}
func (t *StashBoxBatchTagTask) handleMergedPerformer(ctx context.Context, performer *models.ScrapedPerformer, client *stashbox.Client) (mergedPerformer *models.ScrapedPerformer, err error) {
mergedPerformer, err = client.FindPerformerByID(ctx, *performer.RemoteMergedIntoId)
if err != nil {
return nil, fmt.Errorf("loading merged performer %s from stashbox", *performer.RemoteMergedIntoId)
}
if mergedPerformer.StoredID != nil && *mergedPerformer.StoredID != *performer.StoredID {
logger.Warnf("Performer %s merged into %s, but both exist locally, not merging", *performer.StoredID, *mergedPerformer.StoredID)
return nil, nil
}
mergedPerformer.StoredID = performer.StoredID
return mergedPerformer, nil
}
func (t *StashBoxBatchTagTask) processMatchedPerformer(ctx context.Context, p *models.ScrapedPerformer, excluded map[string]bool) {
// Refreshing an existing performer
if t.performer != nil {
@@ -192,19 +156,6 @@ func (t *StashBoxBatchTagTask) processMatchedPerformer(ctx context.Context, p *m
partial := p.ToPartial(t.box.Endpoint, excluded, existingStashIDs)
// if we're setting the performer's aliases, and not the name, then filter out the name
// from the aliases to avoid duplicates
// add the name to the aliases if it's not already there
if partial.Aliases != nil && !partial.Name.Set {
partial.Aliases.Values = sliceutil.Filter(partial.Aliases.Values, func(s string) bool {
return s != t.performer.Name
})
if p.Name != nil && t.performer.Name != *p.Name {
partial.Aliases.Values = sliceutil.AppendUnique(partial.Aliases.Values, *p.Name)
}
}
if err := performer.ValidateUpdate(ctx, t.performer.ID, partial, qb); err != nil {
return err
}
@@ -295,7 +246,8 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
r := instance.Repository
client := stashbox.NewClient(*t.box, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns()))
stashboxRepository := stashbox.NewRepository(r)
client := stashbox.NewClient(*t.box, stashboxRepository)
if t.refresh {
var remoteID string
@@ -316,7 +268,7 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
return nil, err
}
if remoteID != "" {
studio, err = client.FindStudio(ctx, remoteID)
studio, err = client.FindStashBoxStudio(ctx, remoteID)
}
} else {
var name string
@@ -325,19 +277,7 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
} else {
name = t.studio.Name
}
studio, err = client.FindStudio(ctx, name)
}
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
if studio != nil {
if err := match.ScrapedStudioHierarchy(ctx, r.Studio, studio, t.box.Endpoint); err != nil {
return err
}
}
return nil
}); err != nil {
return nil, err
studio, err = client.FindStashBoxStudio(ctx, name)
}
return studio, err

View File

@@ -20,52 +20,18 @@ type GroupNamesFinder interface {
FindByNames(ctx context.Context, names []string, nocase bool) ([]*models.Group, error)
}
type SceneRelationships struct {
PerformerFinder PerformerFinder
TagFinder models.TagQueryer
StudioFinder StudioFinder
}
// MatchRelationships accepts a scraped scene and attempts to match its relationships to existing stash models.
func (r SceneRelationships) MatchRelationships(ctx context.Context, s *models.ScrapedScene, endpoint string) error {
thisStudio := s.Studio
for thisStudio != nil {
if err := ScrapedStudio(ctx, r.StudioFinder, s.Studio, endpoint); err != nil {
return err
}
thisStudio = thisStudio.Parent
}
for _, p := range s.Performers {
err := ScrapedPerformer(ctx, r.PerformerFinder, p, endpoint)
if err != nil {
return err
}
}
for _, t := range s.Tags {
err := ScrapedTag(ctx, r.TagFinder, t)
if err != nil {
return err
}
}
return nil
}
// ScrapedPerformer matches the provided performer with the
// performers in the database and sets the ID field if one is found.
func ScrapedPerformer(ctx context.Context, qb PerformerFinder, p *models.ScrapedPerformer, stashBoxEndpoint string) error {
func ScrapedPerformer(ctx context.Context, qb PerformerFinder, p *models.ScrapedPerformer, stashBoxEndpoint *string) error {
if p.StoredID != nil || p.Name == nil {
return nil
}
// Check if a performer with the StashID already exists
if stashBoxEndpoint != "" && p.RemoteSiteID != nil {
if stashBoxEndpoint != nil && p.RemoteSiteID != nil {
performers, err := qb.FindByStashID(ctx, models.StashID{
StashID: *p.RemoteSiteID,
Endpoint: stashBoxEndpoint,
Endpoint: *stashBoxEndpoint,
})
if err != nil {
return err
@@ -107,16 +73,16 @@ type StudioFinder interface {
// ScrapedStudio matches the provided studio with the studios
// in the database and sets the ID field if one is found.
func ScrapedStudio(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio, stashBoxEndpoint string) error {
func ScrapedStudio(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio, stashBoxEndpoint *string) error {
if s.StoredID != nil {
return nil
}
// Check if a studio with the StashID already exists
if stashBoxEndpoint != "" && s.RemoteSiteID != nil {
if stashBoxEndpoint != nil && s.RemoteSiteID != nil {
studios, err := qb.FindByStashID(ctx, models.StashID{
StashID: *s.RemoteSiteID,
Endpoint: stashBoxEndpoint,
Endpoint: *stashBoxEndpoint,
})
if err != nil {
return err
@@ -152,19 +118,6 @@ func ScrapedStudio(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio
return nil
}
// ScrapedStudioHierarchy executes ScrapedStudio for the provided studio and its parents recursively.
func ScrapedStudioHierarchy(ctx context.Context, qb StudioFinder, s *models.ScrapedStudio, stashBoxEndpoint string) error {
if err := ScrapedStudio(ctx, qb, s, stashBoxEndpoint); err != nil {
return err
}
if s.Parent == nil {
return nil
}
return ScrapedStudioHierarchy(ctx, qb, s.Parent, stashBoxEndpoint)
}
// ScrapedGroup matches the provided movie with the movies
// in the database and returns the ID field if one is found.
func ScrapedGroup(ctx context.Context, qb GroupNamesFinder, storedID *string, name *string) (matchedID *string, err error) {

View File

@@ -26,20 +26,6 @@ func (f *Fingerprint) Value() string {
}
}
// String returns the string representation of the Fingerprint.
// It will return an empty string if the Fingerprint is not a string.
func (f Fingerprint) String() string {
s, _ := f.Fingerprint.(string)
return s
}
// Int64 returns the int64 representation of the Fingerprint.
// It will return 0 if the Fingerprint is not an int64.
func (f Fingerprint) Int64() int64 {
v, _ := f.Fingerprint.(int64)
return v
}
type Fingerprints []Fingerprint
func (f Fingerprints) Remove(type_ string) Fingerprints {
@@ -116,27 +102,33 @@ func (f Fingerprints) For(type_ string) *Fingerprint {
}
func (f Fingerprints) Get(type_ string) interface{} {
fp := f.For(type_)
if fp == nil {
return nil
for _, fp := range f {
if fp.Type == type_ {
return fp.Fingerprint
}
}
return fp.Fingerprint
return nil
}
func (f Fingerprints) GetString(type_ string) string {
fp := f.For(type_)
if fp == nil {
return ""
fp := f.Get(type_)
if fp != nil {
s, _ := fp.(string)
return s
}
return fp.String()
return ""
}
func (f Fingerprints) GetInt64(type_ string) int64 {
fp := f.For(type_)
fp := f.Get(type_)
if fp != nil {
return 0
v, _ := fp.(int64)
return v
}
return fp.Int64()
return 0
}
// AppendUnique appends a fingerprint to the list if a Fingerprint of the same type does not already exist in the list. If one does, then it is updated with o's Fingerprint value.

View File

@@ -63,28 +63,6 @@ type ImageFilterType struct {
UpdatedAt *TimestampCriterionInput `json:"updated_at"`
}
type ImageUpdateInput struct {
ClientMutationID *string `json:"clientMutationId"`
ID string `json:"id"`
Title *string `json:"title"`
Code *string `json:"code"`
Urls []string `json:"urls"`
Date *string `json:"date"`
Details *string `json:"details"`
Photographer *string `json:"photographer"`
Rating100 *int `json:"rating100"`
Organized *bool `json:"organized"`
SceneIds []string `json:"scene_ids"`
StudioID *string `json:"studio_id"`
TagIds []string `json:"tag_ids"`
PerformerIds []string `json:"performer_ids"`
GalleryIds []string `json:"gallery_ids"`
PrimaryFileID *string `json:"primary_file_id"`
// deprecated
URL *string `json:"url"`
}
type ImageDestroyInput struct {
ID string `json:"id"`
DeleteFile *bool `json:"delete_file"`

View File

@@ -14,7 +14,6 @@ import (
type SceneMarker struct {
Title string `json:"title,omitempty"`
Seconds string `json:"seconds,omitempty"`
EndSeconds string `json:"end_seconds,omitempty"`
PrimaryTag string `json:"primary_tag,omitempty"`
Tags []string `json:"tags,omitempty"`
CreatedAt json.JSONTime `json:"created_at,omitempty"`

View File

@@ -11,7 +11,6 @@ import (
type Tag struct {
Name string `json:"name,omitempty"`
SortName string `json:"sort_name,omitempty"`
Description string `json:"description,omitempty"`
Favorite bool `json:"favorite,omitempty"`
Aliases []string `json:"aliases,omitempty"`

View File

@@ -128,15 +128,13 @@ type ScrapedPerformer struct {
Aliases *string `json:"aliases"`
Tags []*ScrapedTag `json:"tags"`
// This should be a base64 encoded data URL
Image *string `json:"image"` // deprecated: use Images
Images []string `json:"images"`
Details *string `json:"details"`
DeathDate *string `json:"death_date"`
HairColor *string `json:"hair_color"`
Weight *string `json:"weight"`
RemoteSiteID *string `json:"remote_site_id"`
RemoteDeleted bool `json:"remote_deleted"`
RemoteMergedIntoId *string `json:"remote_merged_into_id"`
Image *string `json:"image"` // deprecated: use Images
Images []string `json:"images"`
Details *string `json:"details"`
DeathDate *string `json:"death_date"`
HairColor *string `json:"hair_color"`
Weight *string `json:"weight"`
RemoteSiteID *string `json:"remote_site_id"`
}
func (ScrapedPerformer) IsScrapedContent() {}
@@ -492,88 +490,3 @@ func (g ScrapedGroup) ScrapedMovie() ScrapedMovie {
return ret
}
type ScrapedScene struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
Director *string `json:"director"`
URL *string `json:"url"`
URLs []string `json:"urls"`
Date *string `json:"date"`
// This should be a base64 encoded data URL
Image *string `json:"image"`
File *SceneFileType `json:"file"`
Studio *ScrapedStudio `json:"studio"`
Tags []*ScrapedTag `json:"tags"`
Performers []*ScrapedPerformer `json:"performers"`
Groups []*ScrapedGroup `json:"groups"`
Movies []*ScrapedMovie `json:"movies"`
RemoteSiteID *string `json:"remote_site_id"`
Duration *int `json:"duration"`
Fingerprints []*StashBoxFingerprint `json:"fingerprints"`
}
func (ScrapedScene) IsScrapedContent() {}
type ScrapedSceneInput struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
Director *string `json:"director"`
URL *string `json:"url"`
URLs []string `json:"urls"`
Date *string `json:"date"`
RemoteSiteID *string `json:"remote_site_id"`
}
type ScrapedImage struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
Photographer *string `json:"photographer"`
URLs []string `json:"urls"`
Date *string `json:"date"`
Studio *ScrapedStudio `json:"studio"`
Tags []*ScrapedTag `json:"tags"`
Performers []*ScrapedPerformer `json:"performers"`
}
func (ScrapedImage) IsScrapedContent() {}
type ScrapedImageInput struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
URLs []string `json:"urls"`
Date *string `json:"date"`
}
type ScrapedGallery struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
Photographer *string `json:"photographer"`
URLs []string `json:"urls"`
Date *string `json:"date"`
Studio *ScrapedStudio `json:"studio"`
Tags []*ScrapedTag `json:"tags"`
Performers []*ScrapedPerformer `json:"performers"`
// deprecated
URL *string `json:"url"`
}
func (ScrapedGallery) IsScrapedContent() {}
type ScrapedGalleryInput struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
Photographer *string `json:"photographer"`
URLs []string `json:"urls"`
Date *string `json:"date"`
// deprecated
URL *string `json:"url"`
}

View File

@@ -8,7 +8,6 @@ import (
type Tag struct {
ID int `json:"id"`
Name string `json:"name"`
SortName string `json:"sort_name"`
Favorite bool `json:"favorite"`
Description string `json:"description"`
IgnoreAutoTag bool `json:"ignore_auto_tag"`
@@ -48,7 +47,6 @@ func (s *Tag) LoadChildIDs(ctx context.Context, l TagRelationLoader) error {
type TagPartial struct {
Name OptionalString
SortName OptionalString
Description OptionalString
Favorite OptionalBool
IgnoreAutoTag OptionalBool

View File

@@ -7,8 +7,7 @@ type StashBoxFingerprint struct {
}
type StashBox struct {
Endpoint string `json:"endpoint"`
APIKey string `json:"api_key"`
Name string `json:"name"`
MaxRequestsPerMinute int `json:"max_requests_per_minute" koanf:"max_requests_per_minute"`
Endpoint string `json:"endpoint"`
APIKey string `json:"api_key"`
Name string `json:"name"`
}

View File

@@ -4,8 +4,6 @@ type TagFilterType struct {
OperatorFilter[TagFilterType]
// Filter by tag name
Name *StringCriterionInput `json:"name"`
// Filter by tag sort_name
SortName *StringCriterionInput `json:"sort_name"`
// Filter by tag aliases
Aliases *StringCriterionInput `json:"aliases"`
// Filter by tag favorites

View File

@@ -14,11 +14,7 @@ interface IPluginApi {
Button: React.FC<any>;
Nav: React.FC<any> & {
Link: React.FC<any>;
Item: React.FC<any>;
};
Tab: React.FC<any> & {
Pane: React.FC<any>;
}
},
FontAwesomeSolid: {
faEthernet: any;
@@ -49,7 +45,7 @@ interface IPluginApi {
const React = PluginApi.React;
const GQL = PluginApi.GQL;
const { Button, Nav, Tab } = PluginApi.libraries.Bootstrap;
const { Button } = PluginApi.libraries.Bootstrap;
const { faEthernet } = PluginApi.libraries.FontAwesomeSolid;
const {
Link,
@@ -148,10 +144,6 @@ interface IPluginApi {
return <><Overlays />{original({...props})}</>;
});
PluginApi.patch.instead("FrontPage", function (props: any, _: any, original: (props: any) => any) {
return <><p>Hello from Test React!</p>{original({...props})}</>;
});
const TestPage: React.FC = () => {
const componentsToLoad = [
PluginApi.loadableComponents.SceneCard,
@@ -245,37 +237,5 @@ interface IPluginApi {
)
}
]
});
PluginApi.patch.before("ScenePage.Tabs", function (props: any) {
return [
{
children: (
<>
{props.children}
<Nav.Item>
<Nav.Link eventKey="test-react-tab">
Test React tab
</Nav.Link>
</Nav.Item>
</>
),
},
];
});
PluginApi.patch.before("ScenePage.TabContent", function (props: any) {
return [
{
children: (
<>
{props.children}
<Tab.Pane eventKey="test-react-tab">
Test React tab content {props.scene.id}
</Tab.Pane>
</>
),
},
];
});
})
})();

View File

@@ -235,10 +235,6 @@ func GetSceneMarkersJSON(ctx context.Context, markerReader models.SceneMarkerFin
UpdatedAt: json.JSONTime{Time: sceneMarker.UpdatedAt},
}
if sceneMarker.EndSeconds != nil {
sceneMarkerJSON.EndSeconds = getDecimalString(*sceneMarker.EndSeconds)
}
results = append(results, sceneMarkerJSON)
}

View File

@@ -1,66 +0,0 @@
package scene
import (
"context"
"fmt"
"github.com/stashapp/stash/pkg/models"
)
type LoadRelationshipOption func(context.Context, *models.Scene, models.SceneReader) error
func LoadURLs(ctx context.Context, scene *models.Scene, r models.SceneReader) error {
if err := scene.LoadURLs(ctx, r); err != nil {
return fmt.Errorf("loading scene URLs: %w", err)
}
return nil
}
func LoadStashIDs(ctx context.Context, scene *models.Scene, r models.SceneReader) error {
if err := scene.LoadStashIDs(ctx, r); err != nil {
return fmt.Errorf("failed to load stash IDs for scene %d: %w", scene.ID, err)
}
return nil
}
func LoadFiles(ctx context.Context, scene *models.Scene, r models.SceneReader) error {
if err := scene.LoadFiles(ctx, r); err != nil {
return fmt.Errorf("failed to load files for scene %d: %w", scene.ID, err)
}
return nil
}
// FindMany retrieves multiple scenes by their IDs.
// This method will load the specified relationships for each scene.
func (s *Service) FindMany(ctx context.Context, ids []int, load ...LoadRelationshipOption) ([]*models.Scene, error) {
var scenes []*models.Scene
qb := s.Repository
var err error
scenes, err = qb.FindMany(ctx, ids)
if err != nil {
return nil, err
}
// TODO - we should bulk load these relationships
for _, scene := range scenes {
if err := s.LoadRelationships(ctx, scene, load...); err != nil {
return nil, err
}
}
return scenes, nil
}
func (s *Service) LoadRelationships(ctx context.Context, scene *models.Scene, load ...LoadRelationshipOption) error {
for _, l := range load {
if err := l(ctx, scene, s.Repository); err != nil {
return err
}
}
return nil
}

View File

@@ -1,40 +0,0 @@
package scene
import (
"context"
"fmt"
"github.com/stashapp/stash/pkg/models"
)
// GetFingerprints returns the fingerprints for the given scene ids.
func (s *Service) GetScenesFingerprints(ctx context.Context, ids []int) ([]models.Fingerprints, error) {
fingerprints := make([]models.Fingerprints, len(ids))
qb := s.Repository
for i, sceneID := range ids {
scene, err := qb.Find(ctx, sceneID)
if err != nil {
return nil, err
}
if scene == nil {
return nil, fmt.Errorf("scene with id %d not found", sceneID)
}
if err := scene.LoadFiles(ctx, qb); err != nil {
return nil, err
}
var sceneFPs models.Fingerprints
for _, f := range scene.Files.List() {
sceneFPs = append(sceneFPs, f.Fingerprints...)
}
fingerprints[i] = sceneFPs
}
return fingerprints, nil
}

View File

@@ -11,7 +11,7 @@ import (
const (
markerPreviewWidth = 640
maxMarkerPreviewDuration = 20
markerPreviewDuration = 20
markerPreviewAudioBitrate = "64k"
markerImageDuration = 5
@@ -20,28 +20,20 @@ const (
markerScreenshotQuality = 2
)
func (g Generator) MarkerPreviewVideo(ctx context.Context, input string, hash string, seconds float64, endSeconds *float64, includeAudio bool) error {
func (g Generator) MarkerPreviewVideo(ctx context.Context, input string, hash string, seconds int, includeAudio bool) error {
lockCtx := g.LockManager.ReadLock(ctx, input)
defer lockCtx.Cancel()
output := g.MarkerPaths.GetVideoPreviewPath(hash, int(seconds))
output := g.MarkerPaths.GetVideoPreviewPath(hash, seconds)
if !g.Overwrite {
if exists, _ := fsutil.FileExists(output); exists {
return nil
}
}
duration := float64(maxMarkerPreviewDuration)
// don't allow preview to exceed max duration
if endSeconds != nil && *endSeconds-seconds < maxMarkerPreviewDuration {
duration = float64(*endSeconds) - seconds
}
if err := g.generateFile(lockCtx, g.MarkerPaths, mp4Pattern, output, g.markerPreviewVideo(input, sceneMarkerOptions{
Seconds: seconds,
Duration: duration,
Audio: includeAudio,
Seconds: seconds,
Audio: includeAudio,
})); err != nil {
return err
}
@@ -52,9 +44,8 @@ func (g Generator) MarkerPreviewVideo(ctx context.Context, input string, hash st
}
type sceneMarkerOptions struct {
Seconds float64
Duration float64
Audio bool
Seconds int
Audio bool
}
func (g Generator) markerPreviewVideo(input string, options sceneMarkerOptions) generateFn {
@@ -78,8 +69,8 @@ func (g Generator) markerPreviewVideo(input string, options sceneMarkerOptions)
)
trimOptions := transcoder.TranscodeOptions{
Duration: options.Duration,
StartTime: options.Seconds,
Duration: markerPreviewDuration,
StartTime: float64(options.Seconds),
OutputPath: tmpFn,
VideoCodec: ffmpeg.VideoCodecLibX264,
VideoArgs: videoArgs,
@@ -99,11 +90,11 @@ func (g Generator) markerPreviewVideo(input string, options sceneMarkerOptions)
}
}
func (g Generator) SceneMarkerWebp(ctx context.Context, input string, hash string, seconds float64) error {
func (g Generator) SceneMarkerWebp(ctx context.Context, input string, hash string, seconds int) error {
lockCtx := g.LockManager.ReadLock(ctx, input)
defer lockCtx.Cancel()
output := g.MarkerPaths.GetWebpPreviewPath(hash, int(seconds))
output := g.MarkerPaths.GetWebpPreviewPath(hash, seconds)
if !g.Overwrite {
if exists, _ := fsutil.FileExists(output); exists {
return nil
@@ -152,11 +143,11 @@ func (g Generator) sceneMarkerWebp(input string, options sceneMarkerOptions) gen
}
}
func (g Generator) SceneMarkerScreenshot(ctx context.Context, input string, hash string, seconds float64, width int) error {
func (g Generator) SceneMarkerScreenshot(ctx context.Context, input string, hash string, seconds int, width int) error {
lockCtx := g.LockManager.ReadLock(ctx, input)
defer lockCtx.Cancel()
output := g.MarkerPaths.GetScreenshotPath(hash, int(seconds))
output := g.MarkerPaths.GetScreenshotPath(hash, seconds)
if !g.Overwrite {
if exists, _ := fsutil.FileExists(output); exists {
return nil
@@ -176,7 +167,7 @@ func (g Generator) SceneMarkerScreenshot(ctx context.Context, input string, hash
}
type SceneMarkerScreenshotOptions struct {
Seconds float64
Seconds int
Width int
}
@@ -189,7 +180,7 @@ func (g Generator) sceneMarkerScreenshot(input string, options SceneMarkerScreen
Width: options.Width,
}
args := transcoder.ScreenshotTime(input, options.Seconds, ssOptions)
args := transcoder.ScreenshotTime(input, float64(options.Seconds), ssOptions)
return g.generate(lockCtx, args)
}

View File

@@ -27,20 +27,12 @@ type MarkerImporter struct {
func (i *MarkerImporter) PreImport(ctx context.Context) error {
seconds, _ := strconv.ParseFloat(i.Input.Seconds, 64)
var endSeconds *float64
if i.Input.EndSeconds != "" {
parsedEndSeconds, _ := strconv.ParseFloat(i.Input.EndSeconds, 64)
endSeconds = &parsedEndSeconds
}
i.marker = models.SceneMarker{
Title: i.Input.Title,
Seconds: seconds,
EndSeconds: endSeconds,
SceneID: i.SceneID,
CreatedAt: i.Input.CreatedAt.GetTime(),
UpdatedAt: i.Input.UpdatedAt.GetTime(),
Title: i.Input.Title,
Seconds: seconds,
SceneID: i.SceneID,
CreatedAt: i.Input.CreatedAt.GetTime(),
UpdatedAt: i.Input.UpdatedAt.GetTime(),
}
if err := i.populateTags(ctx); err != nil {

View File

@@ -29,9 +29,8 @@ type scraperActionImpl interface {
scrapeByName(ctx context.Context, name string, ty ScrapeContentType) ([]ScrapedContent, error)
scrapeByFragment(ctx context.Context, input Input) (ScrapedContent, error)
scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error)
scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error)
scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error)
scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error)
scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error)
}
func (c config) getScraper(scraper scraperTypeConfig, client *http.Client, globalConfig GlobalConfig) scraperActionImpl {

View File

@@ -89,8 +89,8 @@ func autotagMatchTags(ctx context.Context, path string, tagReader models.TagAuto
return ret, nil
}
func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scene *models.Scene) (*models.ScrapedScene, error) {
var ret *models.ScrapedScene
func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scene *models.Scene) (*ScrapedScene, error) {
var ret *ScrapedScene
const trimExt = false
// populate performers, studio and tags based on scene path
@@ -115,7 +115,7 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
}
if len(performers) > 0 || studio != nil || len(tags) > 0 {
ret = &models.ScrapedScene{
ret = &ScrapedScene{
Performers: performers,
Studio: studio,
Tags: tags,
@@ -130,7 +130,7 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
return ret, nil
}
func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error) {
func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error) {
path := gallery.Path
if path == "" {
// not valid for non-path-based galleries
@@ -140,7 +140,7 @@ func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, ga
// only trim extension if gallery is file-based
trimExt := gallery.PrimaryFileID != nil
var ret *models.ScrapedGallery
var ret *ScrapedGallery
// populate performers, studio and tags based on scene path
if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
@@ -160,7 +160,7 @@ func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, ga
}
if len(performers) > 0 || studio != nil || len(tags) > 0 {
ret = &models.ScrapedGallery{
ret = &ScrapedGallery{
Performers: performers,
Studio: studio,
Tags: tags,

View File

@@ -7,7 +7,6 @@ import (
"net/http"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
@@ -16,7 +15,6 @@ import (
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/txn"
)
@@ -42,7 +40,6 @@ type GlobalConfig interface {
GetScraperCertCheck() bool
GetPythonPath() string
GetProxy() string
GetScraperExcludeTagPatterns() []string
}
func isCDPPathHTTP(c GlobalConfig) bool {
@@ -80,18 +77,11 @@ type GalleryFinder interface {
models.URLLoader
}
type ImageFinder interface {
models.ImageGetter
models.FileLoader
models.URLLoader
}
type Repository struct {
TxnManager models.TxnManager
SceneFinder SceneFinder
GalleryFinder GalleryFinder
ImageFinder ImageFinder
TagFinder TagFinder
PerformerFinder PerformerFinder
GroupFinder match.GroupNamesFinder
@@ -103,7 +93,6 @@ func NewRepository(repo models.Repository) Repository {
TxnManager: repo.TxnManager,
SceneFinder: repo.Scene,
GalleryFinder: repo.Gallery,
ImageFinder: repo.Image,
TagFinder: repo.Tag,
PerformerFinder: repo.Performer,
GroupFinder: repo.Group,
@@ -238,10 +227,6 @@ func (c Cache) findScraper(scraperID string) scraper {
return nil
}
func (c Cache) compileExcludeTagPatterns() []*regexp.Regexp {
return CompileExclusionRegexps(c.globalConfig.GetScraperExcludeTagPatterns())
}
func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeContentType) ([]ScrapedContent, error) {
// find scraper with the provided id
s := c.findScraper(id)
@@ -262,20 +247,13 @@ func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeConten
return nil, fmt.Errorf("error while name scraping with scraper %s: %w", id, err)
}
ignoredRegex := c.compileExcludeTagPatterns()
var ignoredTags []string
for i, cc := range content {
var thisIgnoredTags []string
content[i], thisIgnoredTags, err = c.postScrape(ctx, cc, ignoredRegex)
content[i], err = c.postScrape(ctx, cc)
if err != nil {
return nil, fmt.Errorf("error while post-scraping with scraper %s: %w", id, err)
}
ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
}
LogIgnoredTags(ignoredTags)
return content, nil
}
@@ -299,7 +277,7 @@ func (c Cache) ScrapeFragment(ctx context.Context, id string, input Input) (Scra
return nil, fmt.Errorf("error while fragment scraping with scraper %s: %w", id, err)
}
return c.postScrapeSingle(ctx, content)
return c.postScrape(ctx, content)
}
// ScrapeURL scrapes a given url for the given content. Searches the scraper cache
@@ -321,7 +299,7 @@ func (c Cache) ScrapeURL(ctx context.Context, url string, ty ScrapeContentType)
return ret, nil
}
return c.postScrapeSingle(ctx, ret)
return c.postScrape(ctx, ret)
}
}
@@ -379,34 +357,12 @@ func (c Cache) ScrapeID(ctx context.Context, scraperID string, id int, ty Scrape
return nil, fmt.Errorf("scraper %s: %w", scraperID, err)
}
if scraped != nil {
ret = scraped
}
case ScrapeContentTypeImage:
is, ok := s.(imageScraper)
if !ok {
return nil, fmt.Errorf("%w: cannot use scraper %s as a image scraper", ErrNotSupported, scraperID)
}
scene, err := c.getImage(ctx, id)
if err != nil {
return nil, fmt.Errorf("scraper %s: unable to load image id %v: %w", scraperID, id, err)
}
// don't assign nil concrete pointer to ret interface, otherwise nil
// detection is harder
scraped, err := is.viaImage(ctx, c.client, scene)
if err != nil {
return nil, fmt.Errorf("scraper %s: %w", scraperID, err)
}
if scraped != nil {
ret = scraped
}
}
return c.postScrapeSingle(ctx, ret)
return c.postScrape(ctx, ret)
}
func (c Cache) getScene(ctx context.Context, sceneID int) (*models.Scene, error) {
@@ -470,31 +426,3 @@ func (c Cache) getGallery(ctx context.Context, galleryID int) (*models.Gallery,
}
return ret, nil
}
func (c Cache) getImage(ctx context.Context, imageID int) (*models.Image, error) {
var ret *models.Image
r := c.repository
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
qb := r.ImageFinder
var err error
ret, err = qb.Find(ctx, imageID)
if err != nil {
return err
}
if ret == nil {
return fmt.Errorf("image with id %d not found", imageID)
}
err = ret.LoadFiles(ctx, qb)
if err != nil {
return err
}
return ret.LoadURLs(ctx, qb)
}); err != nil {
return nil, err
}
return ret, nil
}

View File

@@ -45,16 +45,8 @@ type config struct {
// Configuration for querying a gallery by a URL
GalleryByURL []*scrapeByURLConfig `yaml:"galleryByURL"`
// Configuration for querying an image by a URL
ImageByURL []*scrapeByURLConfig `yaml:"imageByURL"`
// Configuration for querying image by an Image fragment
ImageByFragment *scraperTypeConfig `yaml:"imageByFragment"`
// Configuration for querying a movie by a URL - deprecated, use GroupByURL
MovieByURL []*scrapeByURLConfig `yaml:"movieByURL"`
// Configuration for querying a group by a URL
GroupByURL []*scrapeByURLConfig `yaml:"groupByURL"`
// Scraper debugging options
@@ -303,21 +295,6 @@ func (c config) spec() Scraper {
ret.Gallery = &gallery
}
image := ScraperSpec{}
if c.ImageByFragment != nil {
image.SupportedScrapes = append(image.SupportedScrapes, ScrapeTypeFragment)
}
if len(c.ImageByURL) > 0 {
image.SupportedScrapes = append(image.SupportedScrapes, ScrapeTypeURL)
for _, v := range c.ImageByURL {
image.Urls = append(image.Urls, v.URL...)
}
}
if len(image.SupportedScrapes) > 0 {
ret.Image = &image
}
group := ScraperSpec{}
if len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0 {
group.SupportedScrapes = append(group.SupportedScrapes, ScrapeTypeURL)
@@ -342,8 +319,6 @@ func (c config) supports(ty ScrapeContentType) bool {
return (c.SceneByName != nil && c.SceneByQueryFragment != nil) || c.SceneByFragment != nil || len(c.SceneByURL) > 0
case ScrapeContentTypeGallery:
return c.GalleryByFragment != nil || len(c.GalleryByURL) > 0
case ScrapeContentTypeImage:
return c.ImageByFragment != nil || len(c.ImageByURL) > 0
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
return len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0
}
@@ -371,12 +346,6 @@ func (c config) matchesURL(url string, ty ScrapeContentType) bool {
return true
}
}
case ScrapeContentTypeImage:
for _, scraper := range c.ImageByURL {
if scraper.matchesURL(url) {
return true
}
}
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
for _, scraper := range c.MovieByURL {
if scraper.matchesURL(url) {

32
pkg/scraper/gallery.go Normal file
View File

@@ -0,0 +1,32 @@
package scraper
import "github.com/stashapp/stash/pkg/models"
type ScrapedGallery struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
Photographer *string `json:"photographer"`
URLs []string `json:"urls"`
Date *string `json:"date"`
Studio *models.ScrapedStudio `json:"studio"`
Tags []*models.ScrapedTag `json:"tags"`
Performers []*models.ScrapedPerformer `json:"performers"`
// deprecated
URL *string `json:"url"`
}
func (ScrapedGallery) IsScrapedContent() {}
type ScrapedGalleryInput struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
Photographer *string `json:"photographer"`
URLs []string `json:"urls"`
Date *string `json:"date"`
// deprecated
URL *string `json:"url"`
}

View File

@@ -33,9 +33,6 @@ func (g group) fragmentScraper(input Input) *scraperTypeConfig {
case input.Gallery != nil:
// TODO - this should be galleryByQueryFragment
return g.config.GalleryByFragment
case input.Image != nil:
// TODO - this should be imageByImageFragment
return g.config.ImageByFragment
case input.Scene != nil:
return g.config.SceneByQueryFragment
}
@@ -60,7 +57,7 @@ func (g group) viaFragment(ctx context.Context, client *http.Client, input Input
return s.scrapeByFragment(ctx, input)
}
func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*models.ScrapedScene, error) {
func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error) {
if g.config.SceneByFragment == nil {
return nil, ErrNotSupported
}
@@ -69,7 +66,7 @@ func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.
return s.scrapeSceneByScene(ctx, scene)
}
func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error) {
func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error) {
if g.config.GalleryByFragment == nil {
return nil, ErrNotSupported
}
@@ -78,15 +75,6 @@ func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *mod
return s.scrapeGalleryByGallery(ctx, gallery)
}
func (g group) viaImage(ctx context.Context, client *http.Client, gallery *models.Image) (*models.ScrapedImage, error) {
if g.config.ImageByFragment == nil {
return nil, ErrNotSupported
}
s := g.config.getScraper(*g.config.ImageByFragment, client, g.globalConf)
return s.scrapeImageByImage(ctx, gallery)
}
func loadUrlCandidates(c config, ty ScrapeContentType) []*scrapeByURLConfig {
switch ty {
case ScrapeContentTypePerformer:
@@ -97,8 +85,6 @@ func loadUrlCandidates(c config, ty ScrapeContentType) []*scrapeByURLConfig {
return append(c.MovieByURL, c.GroupByURL...)
case ScrapeContentTypeGallery:
return c.GalleryByURL
case ScrapeContentTypeImage:
return c.ImageByURL
}
panic("loadUrlCandidates: unreachable")

View File

@@ -37,7 +37,7 @@ func setPerformerImage(ctx context.Context, client *http.Client, p *models.Scrap
return nil
}
func setSceneImage(ctx context.Context, client *http.Client, s *models.ScrapedScene, globalConfig GlobalConfig) error {
func setSceneImage(ctx context.Context, client *http.Client, s *ScrapedScene, globalConfig GlobalConfig) error {
// don't try to get the image if it doesn't appear to be a URL
if s.Image == nil || !strings.HasPrefix(*s.Image, "http") {
// nothing to do

View File

@@ -102,12 +102,6 @@ func (s *jsonScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCont
return nil, err
}
return ret, nil
case ScrapeContentTypeImage:
ret, err := scraper.scrapeImage(ctx, q)
if err != nil || ret == nil {
return nil, err
}
return ret, nil
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
ret, err := scraper.scrapeGroup(ctx, q)
if err != nil || ret == nil {
@@ -172,7 +166,7 @@ func (s *jsonScraper) scrapeByName(ctx context.Context, name string, ty ScrapeCo
return nil, ErrNotSupported
}
func (s *jsonScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
func (s *jsonScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
// construct the URL
queryURL := queryURLParametersFromScene(scene)
if s.scraper.QueryURLReplacements != nil {
@@ -231,31 +225,7 @@ func (s *jsonScraper) scrapeByFragment(ctx context.Context, input Input) (Scrape
return scraper.scrapeScene(ctx, q)
}
func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
// construct the URL
queryURL := queryURLParametersFromImage(image)
if s.scraper.QueryURLReplacements != nil {
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
}
url := queryURL.constructURL(s.scraper.QueryURL)
scraper := s.getJsonScraper()
if scraper == nil {
return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config")
}
doc, err := s.loadURL(ctx, url)
if err != nil {
return nil, err
}
q := s.getJsonQuery(doc)
return scraper.scrapeImage(ctx, q)
}
func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
// construct the URL
queryURL := queryURLParametersFromGallery(gallery)
if s.scraper.QueryURLReplacements != nil {

View File

@@ -43,9 +43,7 @@ func (s mappedConfig) applyCommon(c commonMappedConfig, src string) string {
return ret
}
type isMultiFunc func(key string) bool
func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonMappedConfig, isMulti isMultiFunc) mappedResults {
func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonMappedConfig) mappedResults {
var ret mappedResults
for k, attrConfig := range s {
@@ -53,7 +51,7 @@ func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonM
if attrConfig.Fixed != "" {
// TODO - not sure if this needs to set _all_ indexes for the key
const i = 0
ret = ret.setSingleValue(i, k, attrConfig.Fixed)
ret = ret.setKey(i, k, attrConfig.Fixed)
} else {
selector := attrConfig.Selector
selector = s.applyCommon(common, selector)
@@ -65,15 +63,8 @@ func (s mappedConfig) process(ctx context.Context, q mappedQuery, common commonM
if len(found) > 0 {
result := s.postProcess(ctx, q, attrConfig, found)
// HACK - if the key is URLs, then we need to set the value as a multi-value
isMulti := isMulti != nil && isMulti(k)
if isMulti {
ret = ret.setMultiValue(0, k, result)
} else {
for i, text := range result {
ret = ret.setSingleValue(i, k, text)
}
for i, text := range result {
ret = ret.setKey(i, k, text)
}
}
}
@@ -190,7 +181,6 @@ type mappedGalleryScraperConfig struct {
Performers mappedConfig `yaml:"Performers"`
Studio mappedConfig `yaml:"Studio"`
}
type _mappedGalleryScraperConfig mappedGalleryScraperConfig
func (s *mappedGalleryScraperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -238,60 +228,6 @@ func (s *mappedGalleryScraperConfig) UnmarshalYAML(unmarshal func(interface{}) e
return nil
}
type mappedImageScraperConfig struct {
mappedConfig
Tags mappedConfig `yaml:"Tags"`
Performers mappedConfig `yaml:"Performers"`
Studio mappedConfig `yaml:"Studio"`
}
type _mappedImageScraperConfig mappedImageScraperConfig
func (s *mappedImageScraperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
// HACK - unmarshal to map first, then remove known scene sub-fields, then
// remarshal to yaml and pass that down to the base map
parentMap := make(map[string]interface{})
if err := unmarshal(parentMap); err != nil {
return err
}
// move the known sub-fields to a separate map
thisMap := make(map[string]interface{})
thisMap[mappedScraperConfigSceneTags] = parentMap[mappedScraperConfigSceneTags]
thisMap[mappedScraperConfigScenePerformers] = parentMap[mappedScraperConfigScenePerformers]
thisMap[mappedScraperConfigSceneStudio] = parentMap[mappedScraperConfigSceneStudio]
delete(parentMap, mappedScraperConfigSceneTags)
delete(parentMap, mappedScraperConfigScenePerformers)
delete(parentMap, mappedScraperConfigSceneStudio)
// re-unmarshal the sub-fields
yml, err := yaml.Marshal(thisMap)
if err != nil {
return err
}
// needs to be a different type to prevent infinite recursion
c := _mappedImageScraperConfig{}
if err := yaml.Unmarshal(yml, &c); err != nil {
return err
}
*s = mappedImageScraperConfig(c)
yml, err = yaml.Marshal(parentMap)
if err != nil {
return err
}
if err := yaml.Unmarshal(yml, &s.mappedConfig); err != nil {
return err
}
return nil
}
type mappedPerformerScraperConfig struct {
mappedConfig
@@ -849,77 +785,41 @@ type mappedScraper struct {
Common commonMappedConfig `yaml:"common"`
Scene *mappedSceneScraperConfig `yaml:"scene"`
Gallery *mappedGalleryScraperConfig `yaml:"gallery"`
Image *mappedImageScraperConfig `yaml:"image"`
Performer *mappedPerformerScraperConfig `yaml:"performer"`
Movie *mappedMovieScraperConfig `yaml:"movie"`
}
type mappedResult map[string]interface{}
type mappedResult map[string]string
type mappedResults []mappedResult
func (r mappedResult) apply(dest interface{}) {
destVal := reflect.ValueOf(dest).Elem()
destVal := reflect.ValueOf(dest)
// dest should be a pointer
destVal = destVal.Elem()
// all fields are either string pointers or string slices
for key, value := range r {
if err := mapFieldValue(destVal, key, value); err != nil {
logger.Errorf("Error mapping field %s in %T: %v", key, dest, err)
}
}
}
field := destVal.FieldByName(key)
func mapFieldValue(destVal reflect.Value, key string, value interface{}) error {
field := destVal.FieldByName(key)
fieldType := field.Type()
if field.IsValid() && field.CanSet() {
switch v := value.(type) {
case string:
// if the field is a pointer to a string, then we need to convert the string to a pointer
// if the field is a string slice, then we need to convert the string to a slice
switch {
case fieldType.Kind() == reflect.String:
field.SetString(v)
case fieldType.Kind() == reflect.Ptr && fieldType.Elem().Kind() == reflect.String:
ptr := reflect.New(fieldType.Elem())
ptr.Elem().SetString(v)
field.Set(ptr)
case fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.String:
field.Set(reflect.ValueOf([]string{v}))
default:
return fmt.Errorf("cannot convert %T to %s", value, fieldType)
}
case []string:
// expect the field to be a string slice
if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.String {
field.Set(reflect.ValueOf(v))
if field.IsValid() {
var reflectValue reflect.Value
if field.Kind() == reflect.Ptr {
// need to copy the value, otherwise everything is set to the
// same pointer
localValue := value
reflectValue = reflect.ValueOf(&localValue)
} else {
return fmt.Errorf("cannot convert %T to %s", value, fieldType)
reflectValue = reflect.ValueOf(value)
}
default:
// fallback to reflection
reflectValue := reflect.ValueOf(value)
reflectValueType := reflectValue.Type()
switch {
case reflectValueType.ConvertibleTo(fieldType):
field.Set(reflectValue.Convert(fieldType))
case fieldType.Kind() == reflect.Pointer && reflectValueType.ConvertibleTo(fieldType.Elem()):
ptr := reflect.New(fieldType.Elem())
ptr.Elem().Set(reflectValue.Convert(fieldType.Elem()))
field.Set(ptr)
default:
return fmt.Errorf("cannot convert %T to %s", value, fieldType)
}
field.Set(reflectValue)
} else {
logger.Errorf("Field %s does not exist in %T", key, dest)
}
} else {
return fmt.Errorf("field does not exist or cannot be set")
}
return nil
}
func (r mappedResults) setSingleValue(index int, key string, value string) mappedResults {
func (r mappedResults) setKey(index int, key string, value string) mappedResults {
if index >= len(r) {
r = append(r, make(mappedResult))
}
@@ -929,20 +829,6 @@ func (r mappedResults) setSingleValue(index int, key string, value string) mappe
return r
}
func (r mappedResults) setMultiValue(index int, key string, value []string) mappedResults {
if index >= len(r) {
r = append(r, make(mappedResult))
}
logger.Debugf(`[%d][%s] = %s`, index, key, value)
r[index][key] = value
return r
}
func urlsIsMulti(key string) bool {
return key == "URLs"
}
func (s mappedScraper) scrapePerformer(ctx context.Context, q mappedQuery) (*models.ScrapedPerformer, error) {
var ret models.ScrapedPerformer
@@ -953,12 +839,12 @@ func (s mappedScraper) scrapePerformer(ctx context.Context, q mappedQuery) (*mod
performerTagsMap := performerMap.Tags
results := performerMap.process(ctx, q, s.Common, urlsIsMulti)
results := performerMap.process(ctx, q, s.Common)
// now apply the tags
if performerTagsMap != nil {
logger.Debug(`Processing performer tags:`)
tagResults := performerTagsMap.process(ctx, q, s.Common, nil)
tagResults := performerTagsMap.process(ctx, q, s.Common)
for _, p := range tagResults {
tag := &models.ScrapedTag{}
@@ -986,8 +872,7 @@ func (s mappedScraper) scrapePerformers(ctx context.Context, q mappedQuery) ([]*
return nil, nil
}
// isMulti is nil because it will behave incorrect when scraping multiple performers
results := performerMap.process(ctx, q, s.Common, nil)
results := performerMap.process(ctx, q, s.Common)
for _, r := range results {
var p models.ScrapedPerformer
r.apply(&p)
@@ -997,8 +882,8 @@ func (s mappedScraper) scrapePerformers(ctx context.Context, q mappedQuery) ([]*
return ret, nil
}
// processSceneRelationships sets the relationships on the models.ScrapedScene. It returns true if any relationships were set.
func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQuery, resultIndex int, ret *models.ScrapedScene) bool {
// processSceneRelationships sets the relationships on the ScrapedScene. It returns true if any relationships were set.
func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQuery, resultIndex int, ret *ScrapedScene) bool {
sceneScraperConfig := s.Scene
scenePerformersMap := sceneScraperConfig.Performers
@@ -1016,7 +901,7 @@ func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQu
if sceneStudioMap != nil {
logger.Debug(`Processing scene studio:`)
studioResults := sceneStudioMap.process(ctx, q, s.Common, nil)
studioResults := sceneStudioMap.process(ctx, q, s.Common)
if len(studioResults) > 0 && resultIndex < len(studioResults) {
studio := &models.ScrapedStudio{}
@@ -1040,15 +925,14 @@ func (s mappedScraper) processPerformers(ctx context.Context, performersMap mapp
// now apply the performers and tags
if performersMap.mappedConfig != nil {
logger.Debug(`Processing performers:`)
// isMulti is nil because it will behave incorrect when scraping multiple performers
performerResults := performersMap.process(ctx, q, s.Common, nil)
performerResults := performersMap.process(ctx, q, s.Common)
scenePerformerTagsMap := performersMap.Tags
// process performer tags once
var performerTagResults mappedResults
if scenePerformerTagsMap != nil {
performerTagResults = scenePerformerTagsMap.process(ctx, q, s.Common, nil)
performerTagResults = scenePerformerTagsMap.process(ctx, q, s.Common)
}
for _, p := range performerResults {
@@ -1071,7 +955,7 @@ func (s mappedScraper) processPerformers(ctx context.Context, performersMap mapp
func processRelationships[T any](ctx context.Context, s mappedScraper, relationshipMap mappedConfig, q mappedQuery) []*T {
var ret []*T
results := relationshipMap.process(ctx, q, s.Common, nil)
results := relationshipMap.process(ctx, q, s.Common)
for _, p := range results {
var value T
@@ -1082,8 +966,8 @@ func processRelationships[T any](ctx context.Context, s mappedScraper, relations
return ret
}
func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*models.ScrapedScene, error) {
var ret []*models.ScrapedScene
func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*ScrapedScene, error) {
var ret []*ScrapedScene
sceneScraperConfig := s.Scene
sceneMap := sceneScraperConfig.mappedConfig
@@ -1092,12 +976,11 @@ func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*mode
}
logger.Debug(`Processing scenes:`)
// urlsIsMulti is nil because it will behave incorrect when scraping multiple scenes
results := sceneMap.process(ctx, q, s.Common, nil)
results := sceneMap.process(ctx, q, s.Common)
for i, r := range results {
logger.Debug(`Processing scene:`)
var thisScene models.ScrapedScene
var thisScene ScrapedScene
r.apply(&thisScene)
s.processSceneRelationships(ctx, q, i, &thisScene)
ret = append(ret, &thisScene)
@@ -1106,7 +989,7 @@ func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*mode
return ret, nil
}
func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*models.ScrapedScene, error) {
func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*ScrapedScene, error) {
sceneScraperConfig := s.Scene
if sceneScraperConfig == nil {
return nil, nil
@@ -1115,9 +998,9 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*models.
sceneMap := sceneScraperConfig.mappedConfig
logger.Debug(`Processing scene:`)
results := sceneMap.process(ctx, q, s.Common, urlsIsMulti)
results := sceneMap.process(ctx, q, s.Common)
var ret models.ScrapedScene
var ret ScrapedScene
if len(results) > 0 {
results[0].apply(&ret)
}
@@ -1133,59 +1016,8 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*models.
return nil, nil
}
func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*models.ScrapedImage, error) {
var ret models.ScrapedImage
imageScraperConfig := s.Image
if imageScraperConfig == nil {
return nil, nil
}
imageMap := imageScraperConfig.mappedConfig
imagePerformersMap := imageScraperConfig.Performers
imageTagsMap := imageScraperConfig.Tags
imageStudioMap := imageScraperConfig.Studio
logger.Debug(`Processing image:`)
results := imageMap.process(ctx, q, s.Common, urlsIsMulti)
// now apply the performers and tags
if imagePerformersMap != nil {
logger.Debug(`Processing image performers:`)
ret.Performers = processRelationships[models.ScrapedPerformer](ctx, s, imagePerformersMap, q)
}
if imageTagsMap != nil {
logger.Debug(`Processing image tags:`)
ret.Tags = processRelationships[models.ScrapedTag](ctx, s, imageTagsMap, q)
}
if imageStudioMap != nil {
logger.Debug(`Processing image studio:`)
studioResults := imageStudioMap.process(ctx, q, s.Common, nil)
if len(studioResults) > 0 {
studio := &models.ScrapedStudio{}
studioResults[0].apply(studio)
ret.Studio = studio
}
}
// if no basic fields are populated, and no relationships, then return nil
if len(results) == 0 && len(ret.Performers) == 0 && len(ret.Tags) == 0 && ret.Studio == nil {
return nil, nil
}
if len(results) > 0 {
results[0].apply(&ret)
}
return &ret, nil
}
func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*models.ScrapedGallery, error) {
var ret models.ScrapedGallery
func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*ScrapedGallery, error) {
var ret ScrapedGallery
galleryScraperConfig := s.Gallery
if galleryScraperConfig == nil {
@@ -1199,12 +1031,12 @@ func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*model
galleryStudioMap := galleryScraperConfig.Studio
logger.Debug(`Processing gallery:`)
results := galleryMap.process(ctx, q, s.Common, urlsIsMulti)
results := galleryMap.process(ctx, q, s.Common)
// now apply the performers and tags
if galleryPerformersMap != nil {
logger.Debug(`Processing gallery performers:`)
performerResults := galleryPerformersMap.process(ctx, q, s.Common, urlsIsMulti)
performerResults := galleryPerformersMap.process(ctx, q, s.Common)
for _, p := range performerResults {
performer := &models.ScrapedPerformer{}
@@ -1215,7 +1047,7 @@ func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*model
if galleryTagsMap != nil {
logger.Debug(`Processing gallery tags:`)
tagResults := galleryTagsMap.process(ctx, q, s.Common, nil)
tagResults := galleryTagsMap.process(ctx, q, s.Common)
for _, p := range tagResults {
tag := &models.ScrapedTag{}
@@ -1226,7 +1058,7 @@ func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*model
if galleryStudioMap != nil {
logger.Debug(`Processing gallery studio:`)
studioResults := galleryStudioMap.process(ctx, q, s.Common, nil)
studioResults := galleryStudioMap.process(ctx, q, s.Common)
if len(studioResults) > 0 {
studio := &models.ScrapedStudio{}
@@ -1260,11 +1092,11 @@ func (s mappedScraper) scrapeGroup(ctx context.Context, q mappedQuery) (*models.
movieStudioMap := movieScraperConfig.Studio
movieTagsMap := movieScraperConfig.Tags
results := movieMap.process(ctx, q, s.Common, urlsIsMulti)
results := movieMap.process(ctx, q, s.Common)
if movieStudioMap != nil {
logger.Debug(`Processing movie studio:`)
studioResults := movieStudioMap.process(ctx, q, s.Common, nil)
studioResults := movieStudioMap.process(ctx, q, s.Common)
if len(studioResults) > 0 {
studio := &models.ScrapedStudio{}
@@ -1276,7 +1108,7 @@ func (s mappedScraper) scrapeGroup(ctx context.Context, q mappedQuery) (*models.
// now apply the tags
if movieTagsMap != nil {
logger.Debug(`Processing movie tags:`)
tagResults := movieTagsMap.process(ctx, q, s.Common, nil)
tagResults := movieTagsMap.process(ctx, q, s.Common)
for _, p := range tagResults {
tag := &models.ScrapedTag{}

View File

@@ -2,76 +2,56 @@ package scraper
import (
"context"
"regexp"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/utils"
)
// postScrape handles post-processing of scraped content. If the content
// requires post-processing, this function fans out to the given content
// type and post-processes it.
func (c Cache) postScrape(ctx context.Context, content ScrapedContent, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
func (c Cache) postScrape(ctx context.Context, content ScrapedContent) (ScrapedContent, error) {
// Analyze the concrete type, call the right post-processing function
switch v := content.(type) {
case *models.ScrapedPerformer:
if v != nil {
return c.postScrapePerformer(ctx, *v, excludeTagRE)
return c.postScrapePerformer(ctx, *v)
}
case models.ScrapedPerformer:
return c.postScrapePerformer(ctx, v, excludeTagRE)
case *models.ScrapedScene:
return c.postScrapePerformer(ctx, v)
case *ScrapedScene:
if v != nil {
return c.postScrapeScene(ctx, *v, excludeTagRE)
return c.postScrapeScene(ctx, *v)
}
case models.ScrapedScene:
return c.postScrapeScene(ctx, v, excludeTagRE)
case *models.ScrapedGallery:
case ScrapedScene:
return c.postScrapeScene(ctx, v)
case *ScrapedGallery:
if v != nil {
return c.postScrapeGallery(ctx, *v, excludeTagRE)
return c.postScrapeGallery(ctx, *v)
}
case models.ScrapedGallery:
return c.postScrapeGallery(ctx, v, excludeTagRE)
case *models.ScrapedImage:
if v != nil {
return c.postScrapeImage(ctx, *v, excludeTagRE)
}
case models.ScrapedImage:
return c.postScrapeImage(ctx, v, excludeTagRE)
case ScrapedGallery:
return c.postScrapeGallery(ctx, v)
case *models.ScrapedMovie:
if v != nil {
return c.postScrapeMovie(ctx, *v, excludeTagRE)
return c.postScrapeMovie(ctx, *v)
}
case models.ScrapedMovie:
return c.postScrapeMovie(ctx, v, excludeTagRE)
return c.postScrapeMovie(ctx, v)
case *models.ScrapedGroup:
if v != nil {
return c.postScrapeGroup(ctx, *v, excludeTagRE)
return c.postScrapeGroup(ctx, *v)
}
case models.ScrapedGroup:
return c.postScrapeGroup(ctx, v, excludeTagRE)
return c.postScrapeGroup(ctx, v)
}
// If nothing matches, pass the content through
return content, nil, nil
return content, nil
}
// postScrapeSingle handles post-processing of a single scraped content item.
// This is a convenience function that includes logging the ignored tags, as opposed to logging them in the caller.
func (c Cache) postScrapeSingle(ctx context.Context, content ScrapedContent) (ScrapedContent, error) {
ret, ignoredTags, err := c.postScrape(ctx, content, c.compileExcludeTagPatterns())
if err != nil {
return nil, err
}
LogIgnoredTags(ignoredTags)
return ret, nil
}
func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerformer, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerformer) (ScrapedContent, error) {
r := c.repository
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
tqb := r.TagFinder
@@ -80,11 +60,11 @@ func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerforme
if err != nil {
return err
}
p.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
p.Tags = tags
return nil
}); err != nil {
return nil, nil, err
return nil, err
}
// post-process - set the image if applicable
@@ -119,10 +99,10 @@ func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerforme
}
}
return p, ignoredTags, nil
return p, nil
}
func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie) (ScrapedContent, error) {
r := c.repository
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
tqb := r.TagFinder
@@ -130,17 +110,17 @@ func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie, exclu
if err != nil {
return err
}
m.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
m.Tags = tags
if m.Studio != nil {
if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, ""); err != nil {
if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, nil); err != nil {
return err
}
}
return nil
}); err != nil {
return nil, nil, err
return nil, err
}
// post-process - set the image if applicable
@@ -151,10 +131,10 @@ func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie, exclu
logger.Warnf("could not set back image using URL %s: %v", *m.BackImage, err)
}
return m, ignoredTags, nil
return m, nil
}
func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup) (ScrapedContent, error) {
r := c.repository
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
tqb := r.TagFinder
@@ -162,17 +142,17 @@ func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup, exclu
if err != nil {
return err
}
m.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
m.Tags = tags
if m.Studio != nil {
if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, ""); err != nil {
if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, nil); err != nil {
return err
}
}
return nil
}); err != nil {
return nil, nil, err
return nil, err
}
// post-process - set the image if applicable
@@ -183,25 +163,24 @@ func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup, exclu
logger.Warnf("could not set back image using URL %s: %v", *m.BackImage, err)
}
return m, ignoredTags, nil
return m, nil
}
func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPerformer, excludeTagRE []*regexp.Regexp) (ignoredTags []string, err error) {
func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPerformer) error {
tqb := c.repository.TagFinder
tags, err := postProcessTags(ctx, tqb, p.Tags)
if err != nil {
return nil, err
return err
}
p.Tags = tags
p.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
p.Country = resolveCountryName(p.Country)
return ignoredTags, nil
return nil
}
func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene) (ScrapedContent, error) {
// set the URL/URLs field
if scene.URL == nil && len(scene.URLs) > 0 {
scene.URL = &scene.URLs[0]
@@ -222,16 +201,13 @@ func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene, e
continue
}
thisIgnoredTags, err := c.postScrapeScenePerformer(ctx, *p, excludeTagRE)
if err != nil {
if err := c.postScrapeScenePerformer(ctx, *p); err != nil {
return err
}
if err := match.ScrapedPerformer(ctx, pqb, p, ""); err != nil {
if err := match.ScrapedPerformer(ctx, pqb, p, nil); err != nil {
return err
}
ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
}
for _, p := range scene.Movies {
@@ -274,10 +250,10 @@ func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene, e
if err != nil {
return err
}
scene.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
scene.Tags = tags
if scene.Studio != nil {
err := match.ScrapedStudio(ctx, sqb, scene.Studio, "")
err := match.ScrapedStudio(ctx, sqb, scene.Studio, nil)
if err != nil {
return err
}
@@ -285,7 +261,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene, e
return nil
}); err != nil {
return nil, nil, err
return nil, err
}
// post-process - set the image if applicable
@@ -293,10 +269,10 @@ func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene, e
logger.Warnf("Could not set image using URL %s: %v", *scene.Image, err)
}
return scene, ignoredTags, nil
return scene, nil
}
func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery) (ScrapedContent, error) {
// set the URL/URLs field
if g.URL == nil && len(g.URLs) > 0 {
g.URL = &g.URLs[0]
@@ -312,7 +288,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery, e
sqb := r.StudioFinder
for _, p := range g.Performers {
err := match.ScrapedPerformer(ctx, pqb, p, "")
err := match.ScrapedPerformer(ctx, pqb, p, nil)
if err != nil {
return err
}
@@ -322,10 +298,10 @@ func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery, e
if err != nil {
return err
}
g.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
g.Tags = tags
if g.Studio != nil {
err := match.ScrapedStudio(ctx, sqb, g.Studio, "")
err := match.ScrapedStudio(ctx, sqb, g.Studio, nil)
if err != nil {
return err
}
@@ -333,43 +309,22 @@ func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery, e
return nil
}); err != nil {
return nil, nil, err
return nil, err
}
return g, ignoredTags, nil
return g, nil
}
func (c Cache) postScrapeImage(ctx context.Context, image models.ScrapedImage, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
r := c.repository
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
pqb := r.PerformerFinder
tqb := r.TagFinder
sqb := r.StudioFinder
func postProcessTags(ctx context.Context, tqb models.TagQueryer, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) {
var ret []*models.ScrapedTag
for _, p := range image.Performers {
if err := match.ScrapedPerformer(ctx, pqb, p, ""); err != nil {
return err
}
}
tags, err := postProcessTags(ctx, tqb, image.Tags)
for _, t := range scrapedTags {
err := match.ScrapedTag(ctx, tqb, t)
if err != nil {
return err
return nil, err
}
image.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
if image.Studio != nil {
err := match.ScrapedStudio(ctx, sqb, image.Studio, "")
if err != nil {
return err
}
}
return nil
}); err != nil {
return nil, nil, err
ret = append(ret, t)
}
return image, ignoredTags, nil
return ret, nil
}

View File

@@ -26,7 +26,7 @@ func queryURLParametersFromScene(scene *models.Scene) queryURLParameters {
return ret
}
func queryURLParametersFromScrapedScene(scene models.ScrapedSceneInput) queryURLParameters {
func queryURLParametersFromScrapedScene(scene ScrapedSceneInput) queryURLParameters {
ret := make(queryURLParameters)
setField := func(field string, value *string) {
@@ -73,24 +73,6 @@ func queryURLParametersFromGallery(gallery *models.Gallery) queryURLParameters {
return ret
}
func queryURLParametersFromImage(image *models.Image) queryURLParameters {
ret := make(queryURLParameters)
ret["checksum"] = image.Checksum
if image.Path != "" {
ret["filename"] = filepath.Base(image.Path)
}
if image.Title != "" {
ret["title"] = image.Title
}
if len(image.URLs.List()) > 0 {
ret["url"] = image.URLs.List()[0]
}
return ret
}
func (p queryURLParameters) applyReplacements(r queryURLReplacements) {
for k, v := range p {
rpl, found := r[k]

39
pkg/scraper/scene.go Normal file
View File

@@ -0,0 +1,39 @@
package scraper
import (
"github.com/stashapp/stash/pkg/models"
)
type ScrapedScene struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
Director *string `json:"director"`
URL *string `json:"url"`
URLs []string `json:"urls"`
Date *string `json:"date"`
// This should be a base64 encoded data URL
Image *string `json:"image"`
File *models.SceneFileType `json:"file"`
Studio *models.ScrapedStudio `json:"studio"`
Tags []*models.ScrapedTag `json:"tags"`
Performers []*models.ScrapedPerformer `json:"performers"`
Groups []*models.ScrapedGroup `json:"groups"`
Movies []*models.ScrapedMovie `json:"movies"`
RemoteSiteID *string `json:"remote_site_id"`
Duration *int `json:"duration"`
Fingerprints []*models.StashBoxFingerprint `json:"fingerprints"`
}
func (ScrapedScene) IsScrapedContent() {}
type ScrapedSceneInput struct {
Title *string `json:"title"`
Code *string `json:"code"`
Details *string `json:"details"`
Director *string `json:"director"`
URL *string `json:"url"`
URLs []string `json:"urls"`
Date *string `json:"date"`
RemoteSiteID *string `json:"remote_site_id"`
}

View File

@@ -36,7 +36,6 @@ const (
ScrapeContentTypeGroup ScrapeContentType = "GROUP"
ScrapeContentTypePerformer ScrapeContentType = "PERFORMER"
ScrapeContentTypeScene ScrapeContentType = "SCENE"
ScrapeContentTypeImage ScrapeContentType = "IMAGE"
)
var AllScrapeContentType = []ScrapeContentType{
@@ -45,12 +44,11 @@ var AllScrapeContentType = []ScrapeContentType{
ScrapeContentTypeGroup,
ScrapeContentTypePerformer,
ScrapeContentTypeScene,
ScrapeContentTypeImage,
}
func (e ScrapeContentType) IsValid() bool {
switch e {
case ScrapeContentTypeGallery, ScrapeContentTypeMovie, ScrapeContentTypeGroup, ScrapeContentTypePerformer, ScrapeContentTypeScene, ScrapeContentTypeImage:
case ScrapeContentTypeGallery, ScrapeContentTypeMovie, ScrapeContentTypeGroup, ScrapeContentTypePerformer, ScrapeContentTypeScene:
return true
}
return false
@@ -86,8 +84,6 @@ type Scraper struct {
Scene *ScraperSpec `json:"scene"`
// Details for gallery scraper
Gallery *ScraperSpec `json:"gallery"`
// Details for image scraper
Image *ScraperSpec `json:"image"`
// Details for movie scraper
Group *ScraperSpec `json:"group"`
// Details for movie scraper
@@ -163,9 +159,8 @@ var (
// set to nil.
type Input struct {
Performer *ScrapedPerformerInput
Scene *models.ScrapedSceneInput
Gallery *models.ScrapedGalleryInput
Image *models.ScrapedImageInput
Scene *ScrapedSceneInput
Gallery *ScrapedGalleryInput
}
// populateURL populates the URL field of the input based on the
@@ -227,15 +222,7 @@ type fragmentScraper interface {
type sceneScraper interface {
scraper
viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*models.ScrapedScene, error)
}
// imageScraper is a scraper which supports image scrapes with
// image data as the input.
type imageScraper interface {
scraper
viaImage(ctx context.Context, client *http.Client, image *models.Image) (*models.ScrapedImage, error)
viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error)
}
// galleryScraper is a scraper which supports gallery scrapes with
@@ -243,5 +230,5 @@ type imageScraper interface {
type galleryScraper interface {
scraper
viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error)
viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error)
}

View File

@@ -328,7 +328,7 @@ func (s *scriptScraper) scrapeByName(ctx context.Context, name string, ty Scrape
}
}
case ScrapeContentTypeScene:
var scenes []models.ScrapedScene
var scenes []ScrapedScene
err = s.runScraperScript(ctx, input, &scenes)
if err == nil {
for _, s := range scenes {
@@ -377,62 +377,44 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
err := s.runScraperScript(ctx, input, &performer)
return performer, err
case ScrapeContentTypeGallery:
var gallery *models.ScrapedGallery
var gallery *ScrapedGallery
err := s.runScraperScript(ctx, input, &gallery)
return gallery, err
case ScrapeContentTypeScene:
var scene *models.ScrapedScene
var scene *ScrapedScene
err := s.runScraperScript(ctx, input, &scene)
return scene, err
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
var movie *models.ScrapedMovie
err := s.runScraperScript(ctx, input, &movie)
return movie, err
case ScrapeContentTypeImage:
var image *models.ScrapedImage
err := s.runScraperScript(ctx, input, &image)
return image, err
}
return nil, ErrNotSupported
}
func (s *scriptScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
func (s *scriptScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
inString, err := json.Marshal(sceneInputFromScene(scene))
if err != nil {
return nil, err
}
var ret *models.ScrapedScene
var ret *ScrapedScene
err = s.runScraperScript(ctx, string(inString), &ret)
return ret, err
}
func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
inString, err := json.Marshal(galleryInputFromGallery(gallery))
if err != nil {
return nil, err
}
var ret *models.ScrapedGallery
err = s.runScraperScript(ctx, string(inString), &ret)
return ret, err
}
func (s *scriptScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
inString, err := json.Marshal(imageToUpdateInput(image))
if err != nil {
return nil, err
}
var ret *models.ScrapedImage
var ret *ScrapedGallery
err = s.runScraperScript(ctx, string(inString), &ret)

View File

@@ -54,7 +54,7 @@ type stashFindPerformerNamePerformer struct {
func (p stashFindPerformerNamePerformer) toPerformer() *models.ScrapedPerformer {
return &models.ScrapedPerformer{
Name: &p.Name,
// HACK - put id into the URL field
// put id into the URL field
URL: &p.ID,
}
}
@@ -107,18 +107,16 @@ func (s *stashScraper) imageGetter() imageGetter {
}
func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (ScrapedContent, error) {
if input.Performer != nil {
return s.scrapeByPerformerFragment(ctx, *input.Performer)
if input.Gallery != nil || input.Scene != nil {
return nil, fmt.Errorf("%w: using stash scraper as a fragment scraper", ErrNotSupported)
}
if input.Scene != nil {
return s.scrapeBySceneFragment(ctx, *input.Scene)
if input.Performer == nil {
return nil, fmt.Errorf("%w: the given performer is nil", ErrNotSupported)
}
return nil, fmt.Errorf("%w: using stash scraper as a fragment scraper", ErrNotSupported)
}
scrapedPerformer := input.Performer
func (s *stashScraper) scrapeByPerformerFragment(ctx context.Context, scrapedPerformer ScrapedPerformerInput) (ScrapedContent, error) {
client := s.getStashClient()
var q struct {
@@ -170,45 +168,6 @@ func (s *stashScraper) scrapeByPerformerFragment(ctx context.Context, scrapedPer
return &ret, nil
}
func (s *stashScraper) scrapeBySceneFragment(ctx context.Context, scrapedScene models.ScrapedSceneInput) (ScrapedContent, error) {
client := s.getStashClient()
var q struct {
FindScene *scrapedSceneStash `graphql:"findScene(id: $f)"`
}
sceneID := scrapedScene.URLs[0]
// get the id from the URL field
vars := map[string]interface{}{
"f": graphql.ID(sceneID),
}
err := client.Query(ctx, &q, vars)
if err != nil {
return nil, convertGraphqlError(err)
}
if q.FindScene == nil {
return nil, nil
}
// need to copy back to a scraped scene
ret, err := s.scrapedStashSceneToScrapedScene(ctx, q.FindScene)
if err != nil {
return nil, err
}
// get the scene image directly
ig := s.imageGetter()
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, ig)
if err != nil {
return nil, err
}
return ret, nil
}
type scrapedStudioStash struct {
Name string `graphql:"name" json:"name"`
URL *string `graphql:"url" json:"url"`
@@ -219,8 +178,8 @@ type stashFindSceneNamesResultType struct {
Scenes []*scrapedSceneStash `graphql:"scenes"`
}
func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scene *scrapedSceneStash) (*models.ScrapedScene, error) {
ret := models.ScrapedScene{}
func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scene *scrapedSceneStash) (*ScrapedScene, error) {
ret := ScrapedScene{}
err := copier.Copy(&ret, scene)
if err != nil {
return nil, err
@@ -273,10 +232,6 @@ func (s *stashScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
if err != nil {
return nil, err
}
// HACK - put id into the URL field
// put id into the URL field
converted.URLs = []string{scene.ID}
ret = append(ret, converted)
}
@@ -341,7 +296,7 @@ type scrapedSceneStash struct {
Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
}
func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
// query by MD5
var q struct {
FindScene *scrapedSceneStash `graphql:"findSceneByHash(input: $c)"`
@@ -379,7 +334,7 @@ func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sce
return nil, err
}
// get the scene image directly
// get the performer image directly
ig := s.imageGetter()
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, ig)
if err != nil {
@@ -401,7 +356,7 @@ type scrapedGalleryStash struct {
Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
}
func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
var q struct {
FindGallery *scrapedGalleryStash `graphql:"findGalleryByHash(input: $c)"`
}
@@ -425,7 +380,7 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
}
// need to copy back to a scraped scene
ret := models.ScrapedGallery{}
ret := ScrapedGallery{}
if err := copier.Copy(&ret, q.FindGallery); err != nil {
return nil, err
}
@@ -433,33 +388,6 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
return &ret, nil
}
func (s *stashScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
return nil, ErrNotSupported
}
func (s *stashScraper) scrapeByURL(_ context.Context, _ string, _ ScrapeContentType) (ScrapedContent, error) {
return nil, ErrNotSupported
}
func imageToUpdateInput(gallery *models.Image) models.ImageUpdateInput {
dateToStringPtr := func(s *models.Date) *string {
if s != nil {
v := s.String()
return &v
}
return nil
}
// fallback to file basename if title is empty
title := gallery.GetTitle()
urls := gallery.URLs.List()
return models.ImageUpdateInput{
ID: strconv.Itoa(gallery.ID),
Title: &title,
Details: &gallery.Details,
Urls: urls,
Date: dateToStringPtr(gallery.Date),
}
}

View File

@@ -196,12 +196,9 @@ type PerformerFragment struct {
Aliases []string "json:\"aliases\" graphql:\"aliases\""
Gender *GenderEnum "json:\"gender,omitempty\" graphql:\"gender\""
MergedIds []string "json:\"merged_ids\" graphql:\"merged_ids\""
Deleted bool "json:\"deleted\" graphql:\"deleted\""
MergedIntoID *string "json:\"merged_into_id,omitempty\" graphql:\"merged_into_id\""
Urls []*URLFragment "json:\"urls\" graphql:\"urls\""
Images []*ImageFragment "json:\"images\" graphql:\"images\""
BirthDate *string "json:\"birth_date,omitempty\" graphql:\"birth_date\""
DeathDate *string "json:\"death_date,omitempty\" graphql:\"death_date\""
Ethnicity *EthnicityEnum "json:\"ethnicity,omitempty\" graphql:\"ethnicity\""
Country *string "json:\"country,omitempty\" graphql:\"country\""
EyeColor *EyeColorEnum "json:\"eye_color,omitempty\" graphql:\"eye_color\""
@@ -251,18 +248,6 @@ func (t *PerformerFragment) GetMergedIds() []string {
}
return t.MergedIds
}
func (t *PerformerFragment) GetDeleted() bool {
if t == nil {
t = &PerformerFragment{}
}
return t.Deleted
}
func (t *PerformerFragment) GetMergedIntoID() *string {
if t == nil {
t = &PerformerFragment{}
}
return t.MergedIntoID
}
func (t *PerformerFragment) GetUrls() []*URLFragment {
if t == nil {
t = &PerformerFragment{}
@@ -281,12 +266,6 @@ func (t *PerformerFragment) GetBirthDate() *string {
}
return t.BirthDate
}
func (t *PerformerFragment) GetDeathDate() *string {
if t == nil {
t = &PerformerFragment{}
}
return t.DeathDate
}
func (t *PerformerFragment) GetEthnicity() *EthnicityEnum {
if t == nil {
t = &PerformerFragment{}
@@ -874,8 +853,6 @@ fragment PerformerFragment on Performer {
aliases
gender
merged_ids
deleted
merged_into_id
urls {
... URLFragment
}
@@ -883,7 +860,6 @@ fragment PerformerFragment on Performer {
... ImageFragment
}
birth_date
death_date
ethnicity
country
eye_color
@@ -1009,8 +985,6 @@ fragment PerformerFragment on Performer {
aliases
gender
merged_ids
deleted
merged_into_id
urls {
... URLFragment
}
@@ -1018,7 +992,6 @@ fragment PerformerFragment on Performer {
... ImageFragment
}
birth_date
death_date
ethnicity
country
eye_color
@@ -1144,8 +1117,6 @@ fragment PerformerFragment on Performer {
aliases
gender
merged_ids
deleted
merged_into_id
urls {
... URLFragment
}
@@ -1153,7 +1124,6 @@ fragment PerformerFragment on Performer {
... ImageFragment
}
birth_date
death_date
ethnicity
country
eye_color
@@ -1279,8 +1249,6 @@ fragment PerformerFragment on Performer {
aliases
gender
merged_ids
deleted
merged_into_id
urls {
... URLFragment
}
@@ -1288,7 +1256,6 @@ fragment PerformerFragment on Performer {
... ImageFragment
}
birth_date
death_date
ethnicity
country
eye_color
@@ -1353,8 +1320,6 @@ fragment PerformerFragment on Performer {
aliases
gender
merged_ids
deleted
merged_into_id
urls {
... URLFragment
}
@@ -1362,7 +1327,6 @@ fragment PerformerFragment on Performer {
... ImageFragment
}
birth_date
death_date
ethnicity
country
eye_color
@@ -1432,8 +1396,6 @@ fragment PerformerFragment on Performer {
aliases
gender
merged_ids
deleted
merged_into_id
urls {
... URLFragment
}
@@ -1441,7 +1403,6 @@ fragment PerformerFragment on Performer {
... ImageFragment
}
birth_date
death_date
ethnicity
country
eye_color
@@ -1572,8 +1533,6 @@ fragment PerformerFragment on Performer {
aliases
gender
merged_ids
deleted
merged_into_id
urls {
... URLFragment
}
@@ -1581,7 +1540,6 @@ fragment PerformerFragment on Performer {
... ImageFragment
}
birth_date
death_date
ethnicity
country
eye_color

View File

@@ -23,10 +23,6 @@ type EditTarget interface {
IsEditTarget()
}
type NotificationData interface {
IsNotificationData()
}
type SceneDraftPerformer interface {
IsSceneDraftPerformer()
}
@@ -41,6 +37,7 @@ type SceneDraftTag interface {
type ActivateNewUserInput struct {
Name string `json:"name"`
Email string `json:"email"`
ActivationKey string `json:"activation_key"`
Password string `json:"password"`
}
@@ -74,35 +71,11 @@ type CancelEditInput struct {
ID string `json:"id"`
}
type CommentCommentedEdit struct {
Comment *EditComment `json:"comment"`
}
func (CommentCommentedEdit) IsNotificationData() {}
type CommentOwnEdit struct {
Comment *EditComment `json:"comment"`
}
func (CommentOwnEdit) IsNotificationData() {}
type CommentVotedEdit struct {
Comment *EditComment `json:"comment"`
}
func (CommentVotedEdit) IsNotificationData() {}
type DateCriterionInput struct {
Value string `json:"value"`
Modifier CriterionModifier `json:"modifier"`
}
type DownvoteOwnEdit struct {
Edit *Edit `json:"edit"`
}
func (DownvoteOwnEdit) IsNotificationData() {}
type Draft struct {
ID string `json:"id"`
Created time.Time `json:"created"`
@@ -159,8 +132,6 @@ type Edit struct {
Destructive bool `json:"destructive"`
Status VoteStatusEnum `json:"status"`
Applied bool `json:"applied"`
UpdateCount int `json:"update_count"`
Updatable bool `json:"updatable"`
Created time.Time `json:"created"`
Updated *time.Time `json:"updated,omitempty"`
Closed *time.Time `json:"closed,omitempty"`
@@ -172,7 +143,6 @@ type EditComment struct {
User *User `json:"user,omitempty"`
Date time.Time `json:"date"`
Comment string `json:"comment"`
Edit *Edit `json:"edit"`
}
type EditCommentInput struct {
@@ -236,50 +206,14 @@ type EyeColorCriterionInput struct {
Modifier CriterionModifier `json:"modifier"`
}
type FailedOwnEdit struct {
Edit *Edit `json:"edit"`
}
func (FailedOwnEdit) IsNotificationData() {}
type FavoritePerformerEdit struct {
Edit *Edit `json:"edit"`
}
func (FavoritePerformerEdit) IsNotificationData() {}
type FavoritePerformerScene struct {
Scene *Scene `json:"scene"`
}
func (FavoritePerformerScene) IsNotificationData() {}
type FavoriteStudioEdit struct {
Edit *Edit `json:"edit"`
}
func (FavoriteStudioEdit) IsNotificationData() {}
type FavoriteStudioScene struct {
Scene *Scene `json:"scene"`
}
func (FavoriteStudioScene) IsNotificationData() {}
type Fingerprint struct {
Hash string `json:"hash"`
Algorithm FingerprintAlgorithm `json:"algorithm"`
Duration int `json:"duration"`
// number of times this fingerprint has been submitted (excluding reports)
Submissions int `json:"submissions"`
// number of times this fingerprint has been reported
Reports int `json:"reports"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
// true if the current user submitted this fingerprint
UserSubmitted bool `json:"user_submitted"`
// true if the current user reported this fingerprint
UserReported bool `json:"user_reported"`
Hash string `json:"hash"`
Algorithm FingerprintAlgorithm `json:"algorithm"`
Duration int `json:"duration"`
Submissions int `json:"submissions"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
UserSubmitted bool `json:"user_submitted"`
}
type FingerprintEditInput struct {
@@ -306,18 +240,11 @@ type FingerprintQueryInput struct {
}
type FingerprintSubmission struct {
SceneID string `json:"scene_id"`
Fingerprint *FingerprintInput `json:"fingerprint"`
Unmatch *bool `json:"unmatch,omitempty"`
Vote *FingerprintSubmissionType `json:"vote,omitempty"`
SceneID string `json:"scene_id"`
Fingerprint *FingerprintInput `json:"fingerprint"`
Unmatch *bool `json:"unmatch,omitempty"`
}
type FingerprintedSceneEdit struct {
Edit *Edit `json:"edit"`
}
func (FingerprintedSceneEdit) IsNotificationData() {}
type FuzzyDate struct {
Date string `json:"date"`
Accuracy DateAccuracyEnum `json:"accuracy"`
@@ -376,11 +303,6 @@ type InviteKey struct {
Expires *time.Time `json:"expires,omitempty"`
}
type MarkNotificationReadInput struct {
Type NotificationEnum `json:"type"`
ID string `json:"id"`
}
type Measurements struct {
CupSize *string `json:"cup_size,omitempty"`
BandSize *int `json:"band_size,omitempty"`
@@ -406,12 +328,6 @@ type NewUserInput struct {
InviteKey *string `json:"invite_key,omitempty"`
}
type Notification struct {
Created time.Time `json:"created"`
Read bool `json:"read"`
Data NotificationData `json:"data"`
}
type Performer struct {
ID string `json:"id"`
Name string `json:"name"`
@@ -421,7 +337,6 @@ type Performer struct {
Urls []*URL `json:"urls"`
Birthdate *FuzzyDate `json:"birthdate,omitempty"`
BirthDate *string `json:"birth_date,omitempty"`
DeathDate *string `json:"death_date,omitempty"`
Age *int `json:"age,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
@@ -444,14 +359,11 @@ type Performer struct {
Edits []*Edit `json:"edits"`
SceneCount int `json:"scene_count"`
Scenes []*Scene `json:"scenes"`
// IDs of performers that were merged into this one
MergedIds []string `json:"merged_ids"`
// ID of performer that replaces this one
MergedIntoID *string `json:"merged_into_id,omitempty"`
Studios []*PerformerStudio `json:"studios"`
IsFavorite bool `json:"is_favorite"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
MergedIds []string `json:"merged_ids"`
Studios []*PerformerStudio `json:"studios"`
IsFavorite bool `json:"is_favorite"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
}
func (Performer) IsEditTarget() {}
@@ -477,7 +389,6 @@ type PerformerCreateInput struct {
Gender *GenderEnum `json:"gender,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
Deathdate *string `json:"deathdate,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
EyeColor *EyeColorEnum `json:"eye_color,omitempty"`
@@ -507,7 +418,6 @@ type PerformerDraft struct {
Aliases *string `json:"aliases,omitempty"`
Gender *string `json:"gender,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
Deathdate *string `json:"deathdate,omitempty"`
Urls []string `json:"urls,omitempty"`
Ethnicity *string `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
@@ -532,7 +442,6 @@ type PerformerDraftInput struct {
Aliases *string `json:"aliases,omitempty"`
Gender *string `json:"gender,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
Deathdate *string `json:"deathdate,omitempty"`
Urls []string `json:"urls,omitempty"`
Ethnicity *string `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
@@ -557,7 +466,6 @@ type PerformerEdit struct {
AddedUrls []*URL `json:"added_urls,omitempty"`
RemovedUrls []*URL `json:"removed_urls,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
Deathdate *string `json:"deathdate,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
EyeColor *EyeColorEnum `json:"eye_color,omitempty"`
@@ -594,7 +502,6 @@ type PerformerEditDetailsInput struct {
Gender *GenderEnum `json:"gender,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
Deathdate *string `json:"deathdate,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
EyeColor *EyeColorEnum `json:"eye_color,omitempty"`
@@ -647,7 +554,6 @@ type PerformerQueryInput struct {
// Filter to search urls - assumes like query unless quoted
URL *string `json:"url,omitempty"`
Birthdate *DateCriterionInput `json:"birthdate,omitempty"`
Deathdate *DateCriterionInput `json:"deathdate,omitempty"`
BirthYear *IntCriterionInput `json:"birth_year,omitempty"`
Age *IntCriterionInput `json:"age,omitempty"`
Ethnicity *EthnicityFilterEnum `json:"ethnicity,omitempty"`
@@ -698,7 +604,6 @@ type PerformerUpdateInput struct {
Gender *GenderEnum `json:"gender,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
Deathdate *string `json:"deathdate,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
EyeColor *EyeColorEnum `json:"eye_color,omitempty"`
@@ -725,17 +630,6 @@ type QueryEditsResultType struct {
Edits []*Edit `json:"edits"`
}
type QueryExistingPerformerInput struct {
Name *string `json:"name,omitempty"`
Disambiguation *string `json:"disambiguation,omitempty"`
Urls []string `json:"urls"`
}
type QueryExistingPerformerResult struct {
Edits []*Edit `json:"edits"`
Performers []*Performer `json:"performers"`
}
type QueryExistingSceneInput struct {
Title *string `json:"title,omitempty"`
StudioID *string `json:"studio_id,omitempty"`
@@ -747,18 +641,6 @@ type QueryExistingSceneResult struct {
Scenes []*Scene `json:"scenes"`
}
type QueryNotificationsInput struct {
Page int `json:"page"`
PerPage int `json:"per_page"`
Type *NotificationEnum `json:"type,omitempty"`
UnreadOnly *bool `json:"unread_only,omitempty"`
}
type QueryNotificationsResult struct {
Count int `json:"count"`
Notifications []*Notification `json:"notifications"`
}
type QueryPerformersResultType struct {
Count int `json:"count"`
Performers []*Performer `json:"performers"`
@@ -809,43 +691,41 @@ type RoleCriterionInput struct {
}
type Scene struct {
ID string `json:"id"`
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Date *string `json:"date,omitempty"`
ReleaseDate *string `json:"release_date,omitempty"`
ProductionDate *string `json:"production_date,omitempty"`
Urls []*URL `json:"urls"`
Studio *Studio `json:"studio,omitempty"`
Tags []*Tag `json:"tags"`
Images []*Image `json:"images"`
Performers []*PerformerAppearance `json:"performers"`
Fingerprints []*Fingerprint `json:"fingerprints"`
Duration *int `json:"duration,omitempty"`
Director *string `json:"director,omitempty"`
Code *string `json:"code,omitempty"`
Deleted bool `json:"deleted"`
Edits []*Edit `json:"edits"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
ID string `json:"id"`
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Date *string `json:"date,omitempty"`
ReleaseDate *string `json:"release_date,omitempty"`
Urls []*URL `json:"urls"`
Studio *Studio `json:"studio,omitempty"`
Tags []*Tag `json:"tags"`
Images []*Image `json:"images"`
Performers []*PerformerAppearance `json:"performers"`
Fingerprints []*Fingerprint `json:"fingerprints"`
Duration *int `json:"duration,omitempty"`
Director *string `json:"director,omitempty"`
Code *string `json:"code,omitempty"`
Deleted bool `json:"deleted"`
Edits []*Edit `json:"edits"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
}
func (Scene) IsEditTarget() {}
type SceneCreateInput struct {
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Date string `json:"date"`
ProductionDate *string `json:"production_date,omitempty"`
StudioID *string `json:"studio_id,omitempty"`
Performers []*PerformerAppearanceInput `json:"performers,omitempty"`
TagIds []string `json:"tag_ids,omitempty"`
ImageIds []string `json:"image_ids,omitempty"`
Fingerprints []*FingerprintEditInput `json:"fingerprints"`
Duration *int `json:"duration,omitempty"`
Director *string `json:"director,omitempty"`
Code *string `json:"code,omitempty"`
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Date string `json:"date"`
StudioID *string `json:"studio_id,omitempty"`
Performers []*PerformerAppearanceInput `json:"performers,omitempty"`
TagIds []string `json:"tag_ids,omitempty"`
ImageIds []string `json:"image_ids,omitempty"`
Fingerprints []*FingerprintEditInput `json:"fingerprints"`
Duration *int `json:"duration,omitempty"`
Director *string `json:"director,omitempty"`
Code *string `json:"code,omitempty"`
}
type SceneDestroyInput struct {
@@ -853,31 +733,29 @@ type SceneDestroyInput struct {
}
type SceneDraft struct {
ID *string `json:"id,omitempty"`
Title *string `json:"title,omitempty"`
Code *string `json:"code,omitempty"`
Details *string `json:"details,omitempty"`
Director *string `json:"director,omitempty"`
Urls []string `json:"urls,omitempty"`
Date *string `json:"date,omitempty"`
ProductionDate *string `json:"production_date,omitempty"`
Studio SceneDraftStudio `json:"studio,omitempty"`
Performers []SceneDraftPerformer `json:"performers"`
Tags []SceneDraftTag `json:"tags,omitempty"`
Image *Image `json:"image,omitempty"`
Fingerprints []*DraftFingerprint `json:"fingerprints"`
ID *string `json:"id,omitempty"`
Title *string `json:"title,omitempty"`
Code *string `json:"code,omitempty"`
Details *string `json:"details,omitempty"`
Director *string `json:"director,omitempty"`
URL *URL `json:"url,omitempty"`
Date *string `json:"date,omitempty"`
Studio SceneDraftStudio `json:"studio,omitempty"`
Performers []SceneDraftPerformer `json:"performers"`
Tags []SceneDraftTag `json:"tags,omitempty"`
Image *Image `json:"image,omitempty"`
Fingerprints []*DraftFingerprint `json:"fingerprints"`
}
func (SceneDraft) IsDraftData() {}
type SceneEdit struct {
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
AddedUrls []*URL `json:"added_urls,omitempty"`
RemovedUrls []*URL `json:"removed_urls,omitempty"`
Date *string `json:"date,omitempty"`
ProductionDate *string `json:"production_date,omitempty"`
Studio *Studio `json:"studio,omitempty"`
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
AddedUrls []*URL `json:"added_urls,omitempty"`
RemovedUrls []*URL `json:"removed_urls,omitempty"`
Date *string `json:"date,omitempty"`
Studio *Studio `json:"studio,omitempty"`
// Added or modified performer appearance entries
AddedPerformers []*PerformerAppearance `json:"added_performers,omitempty"`
RemovedPerformers []*PerformerAppearance `json:"removed_performers,omitempty"`
@@ -901,20 +779,19 @@ type SceneEdit struct {
func (SceneEdit) IsEditDetails() {}
type SceneEditDetailsInput struct {
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Date *string `json:"date,omitempty"`
ProductionDate *string `json:"production_date,omitempty"`
StudioID *string `json:"studio_id,omitempty"`
Performers []*PerformerAppearanceInput `json:"performers,omitempty"`
TagIds []string `json:"tag_ids,omitempty"`
ImageIds []string `json:"image_ids,omitempty"`
Duration *int `json:"duration,omitempty"`
Director *string `json:"director,omitempty"`
Code *string `json:"code,omitempty"`
Fingerprints []*FingerprintInput `json:"fingerprints,omitempty"`
DraftID *string `json:"draft_id,omitempty"`
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Date *string `json:"date,omitempty"`
StudioID *string `json:"studio_id,omitempty"`
Performers []*PerformerAppearanceInput `json:"performers,omitempty"`
TagIds []string `json:"tag_ids,omitempty"`
ImageIds []string `json:"image_ids,omitempty"`
Duration *int `json:"duration,omitempty"`
Director *string `json:"director,omitempty"`
Code *string `json:"code,omitempty"`
Fingerprints []*FingerprintInput `json:"fingerprints,omitempty"`
DraftID *string `json:"draft_id,omitempty"`
}
type SceneEditInput struct {
@@ -932,8 +809,6 @@ type SceneQueryInput struct {
URL *string `json:"url,omitempty"`
// Filter by date
Date *DateCriterionInput `json:"date,omitempty"`
// Filter by production date
ProductionDate *DateCriterionInput `json:"production_date,omitempty"`
// Filter to only include scenes with this studio
Studios *MultiIDCriterionInput `json:"studios,omitempty"`
// Filter to only include scenes with this studio as primary or parent
@@ -957,20 +832,19 @@ type SceneQueryInput struct {
}
type SceneUpdateInput struct {
ID string `json:"id"`
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Date *string `json:"date,omitempty"`
ProductionDate *string `json:"production_date,omitempty"`
StudioID *string `json:"studio_id,omitempty"`
Performers []*PerformerAppearanceInput `json:"performers,omitempty"`
TagIds []string `json:"tag_ids,omitempty"`
ImageIds []string `json:"image_ids,omitempty"`
Fingerprints []*FingerprintEditInput `json:"fingerprints,omitempty"`
Duration *int `json:"duration,omitempty"`
Director *string `json:"director,omitempty"`
Code *string `json:"code,omitempty"`
ID string `json:"id"`
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Date *string `json:"date,omitempty"`
StudioID *string `json:"studio_id,omitempty"`
Performers []*PerformerAppearanceInput `json:"performers,omitempty"`
TagIds []string `json:"tag_ids,omitempty"`
ImageIds []string `json:"image_ids,omitempty"`
Fingerprints []*FingerprintEditInput `json:"fingerprints,omitempty"`
Duration *int `json:"duration,omitempty"`
Director *string `json:"director,omitempty"`
Code *string `json:"code,omitempty"`
}
type Site struct {
@@ -1016,8 +890,6 @@ type StashBoxConfig struct {
MinDestructiveVotingPeriod int `json:"min_destructive_voting_period"`
VoteCronInterval string `json:"vote_cron_interval"`
GuidelinesURL string `json:"guidelines_url"`
RequireSceneDraft bool `json:"require_scene_draft"`
EditUpdateLimit int `json:"edit_update_limit"`
}
type StringCriterionInput struct {
@@ -1028,7 +900,6 @@ type StringCriterionInput struct {
type Studio struct {
ID string `json:"id"`
Name string `json:"name"`
Aliases []string `json:"aliases"`
Urls []*URL `json:"urls"`
Parent *Studio `json:"parent,omitempty"`
ChildStudios []*Studio `json:"child_studios"`
@@ -1046,7 +917,6 @@ func (Studio) IsSceneDraftStudio() {}
type StudioCreateInput struct {
Name string `json:"name"`
Aliases []string `json:"aliases,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
ParentID *string `json:"parent_id,omitempty"`
ImageIds []string `json:"image_ids,omitempty"`
@@ -1059,22 +929,19 @@ type StudioDestroyInput struct {
type StudioEdit struct {
Name *string `json:"name,omitempty"`
// Added and modified URLs
AddedUrls []*URL `json:"added_urls,omitempty"`
RemovedUrls []*URL `json:"removed_urls,omitempty"`
Parent *Studio `json:"parent,omitempty"`
AddedImages []*Image `json:"added_images,omitempty"`
RemovedImages []*Image `json:"removed_images,omitempty"`
AddedAliases []string `json:"added_aliases,omitempty"`
RemovedAliases []string `json:"removed_aliases,omitempty"`
Images []*Image `json:"images"`
Urls []*URL `json:"urls"`
AddedUrls []*URL `json:"added_urls,omitempty"`
RemovedUrls []*URL `json:"removed_urls,omitempty"`
Parent *Studio `json:"parent,omitempty"`
AddedImages []*Image `json:"added_images,omitempty"`
RemovedImages []*Image `json:"removed_images,omitempty"`
Images []*Image `json:"images"`
Urls []*URL `json:"urls"`
}
func (StudioEdit) IsEditDetails() {}
type StudioEditDetailsInput struct {
Name *string `json:"name,omitempty"`
Aliases []string `json:"aliases,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
ParentID *string `json:"parent_id,omitempty"`
ImageIds []string `json:"image_ids,omitempty"`
@@ -1089,7 +956,7 @@ type StudioEditInput struct {
type StudioQueryInput struct {
// Filter to search name - assumes like query unless quoted
Name *string `json:"name,omitempty"`
// Filter to search studio name, aliases and parent studio name - assumes like query unless quoted
// Filter to search studio and parent studio name - assumes like query unless quoted
Names *string `json:"names,omitempty"`
// Filter to search url - assumes like query unless quoted
URL *string `json:"url,omitempty"`
@@ -1106,7 +973,6 @@ type StudioQueryInput struct {
type StudioUpdateInput struct {
ID string `json:"id"`
Name *string `json:"name,omitempty"`
Aliases []string `json:"aliases,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
ParentID *string `json:"parent_id,omitempty"`
ImageIds []string `json:"image_ids,omitempty"`
@@ -1221,12 +1087,6 @@ type URLInput struct {
SiteID string `json:"site_id"`
}
type UpdatedEdit struct {
Edit *Edit `json:"edit"`
}
func (UpdatedEdit) IsNotificationData() {}
type User struct {
ID string `json:"id"`
Name string `json:"name"`
@@ -1235,8 +1095,7 @@ type User struct {
// Should not be visible to other users
Email *string `json:"email,omitempty"`
// Should not be visible to other users
APIKey *string `json:"api_key,omitempty"`
NotificationSubscriptions []NotificationEnum `json:"notification_subscriptions"`
APIKey *string `json:"api_key,omitempty"`
// Vote counts by type
VoteCount *UserVoteCount `json:"vote_count"`
// Edit counts by status
@@ -1249,12 +1108,6 @@ type User struct {
InviteCodes []*InviteKey `json:"invite_codes,omitempty"`
}
type UserChangeEmailInput struct {
ExistingEmailToken *string `json:"existing_email_token,omitempty"`
NewEmailToken *string `json:"new_email_token,omitempty"`
NewEmail *string `json:"new_email,omitempty"`
}
type UserChangePasswordInput struct {
// Password in plain text
ExistingPassword *string `json:"existing_password,omitempty"`
@@ -1768,52 +1621,6 @@ func (e FingerprintAlgorithm) MarshalGQL(w io.Writer) {
fmt.Fprint(w, strconv.Quote(e.String()))
}
type FingerprintSubmissionType string
const (
// Positive vote
FingerprintSubmissionTypeValid FingerprintSubmissionType = "VALID"
// Report as invalid
FingerprintSubmissionTypeInvalid FingerprintSubmissionType = "INVALID"
// Remove vote
FingerprintSubmissionTypeRemove FingerprintSubmissionType = "REMOVE"
)
var AllFingerprintSubmissionType = []FingerprintSubmissionType{
FingerprintSubmissionTypeValid,
FingerprintSubmissionTypeInvalid,
FingerprintSubmissionTypeRemove,
}
func (e FingerprintSubmissionType) IsValid() bool {
switch e {
case FingerprintSubmissionTypeValid, FingerprintSubmissionTypeInvalid, FingerprintSubmissionTypeRemove:
return true
}
return false
}
func (e FingerprintSubmissionType) String() string {
return string(e)
}
func (e *FingerprintSubmissionType) UnmarshalGQL(v interface{}) error {
str, ok := v.(string)
if !ok {
return fmt.Errorf("enums must be strings")
}
*e = FingerprintSubmissionType(str)
if !e.IsValid() {
return fmt.Errorf("%s is not a valid FingerprintSubmissionType", str)
}
return nil
}
func (e FingerprintSubmissionType) MarshalGQL(w io.Writer) {
fmt.Fprint(w, strconv.Quote(e.String()))
}
type GenderEnum string
const (
@@ -1925,7 +1732,6 @@ const (
HairColorEnumGrey HairColorEnum = "GREY"
HairColorEnumBald HairColorEnum = "BALD"
HairColorEnumVarious HairColorEnum = "VARIOUS"
HairColorEnumWhite HairColorEnum = "WHITE"
HairColorEnumOther HairColorEnum = "OTHER"
)
@@ -1938,13 +1744,12 @@ var AllHairColorEnum = []HairColorEnum{
HairColorEnumGrey,
HairColorEnumBald,
HairColorEnumVarious,
HairColorEnumWhite,
HairColorEnumOther,
}
func (e HairColorEnum) IsValid() bool {
switch e {
case HairColorEnumBlonde, HairColorEnumBrunette, HairColorEnumBlack, HairColorEnumRed, HairColorEnumAuburn, HairColorEnumGrey, HairColorEnumBald, HairColorEnumVarious, HairColorEnumWhite, HairColorEnumOther:
case HairColorEnumBlonde, HairColorEnumBrunette, HairColorEnumBlack, HairColorEnumRed, HairColorEnumAuburn, HairColorEnumGrey, HairColorEnumBald, HairColorEnumVarious, HairColorEnumOther:
return true
}
return false
@@ -1971,65 +1776,6 @@ func (e HairColorEnum) MarshalGQL(w io.Writer) {
fmt.Fprint(w, strconv.Quote(e.String()))
}
type NotificationEnum string
const (
NotificationEnumFavoritePerformerScene NotificationEnum = "FAVORITE_PERFORMER_SCENE"
NotificationEnumFavoritePerformerEdit NotificationEnum = "FAVORITE_PERFORMER_EDIT"
NotificationEnumFavoriteStudioScene NotificationEnum = "FAVORITE_STUDIO_SCENE"
NotificationEnumFavoriteStudioEdit NotificationEnum = "FAVORITE_STUDIO_EDIT"
NotificationEnumCommentOwnEdit NotificationEnum = "COMMENT_OWN_EDIT"
NotificationEnumDownvoteOwnEdit NotificationEnum = "DOWNVOTE_OWN_EDIT"
NotificationEnumFailedOwnEdit NotificationEnum = "FAILED_OWN_EDIT"
NotificationEnumCommentCommentedEdit NotificationEnum = "COMMENT_COMMENTED_EDIT"
NotificationEnumCommentVotedEdit NotificationEnum = "COMMENT_VOTED_EDIT"
NotificationEnumUpdatedEdit NotificationEnum = "UPDATED_EDIT"
NotificationEnumFingerprintedSceneEdit NotificationEnum = "FINGERPRINTED_SCENE_EDIT"
)
var AllNotificationEnum = []NotificationEnum{
NotificationEnumFavoritePerformerScene,
NotificationEnumFavoritePerformerEdit,
NotificationEnumFavoriteStudioScene,
NotificationEnumFavoriteStudioEdit,
NotificationEnumCommentOwnEdit,
NotificationEnumDownvoteOwnEdit,
NotificationEnumFailedOwnEdit,
NotificationEnumCommentCommentedEdit,
NotificationEnumCommentVotedEdit,
NotificationEnumUpdatedEdit,
NotificationEnumFingerprintedSceneEdit,
}
func (e NotificationEnum) IsValid() bool {
switch e {
case NotificationEnumFavoritePerformerScene, NotificationEnumFavoritePerformerEdit, NotificationEnumFavoriteStudioScene, NotificationEnumFavoriteStudioEdit, NotificationEnumCommentOwnEdit, NotificationEnumDownvoteOwnEdit, NotificationEnumFailedOwnEdit, NotificationEnumCommentCommentedEdit, NotificationEnumCommentVotedEdit, NotificationEnumUpdatedEdit, NotificationEnumFingerprintedSceneEdit:
return true
}
return false
}
func (e NotificationEnum) String() string {
return string(e)
}
func (e *NotificationEnum) UnmarshalGQL(v interface{}) error {
str, ok := v.(string)
if !ok {
return fmt.Errorf("enums must be strings")
}
*e = NotificationEnum(str)
if !e.IsValid() {
return fmt.Errorf("%s is not a valid NotificationEnum", str)
}
return nil
}
func (e NotificationEnum) MarshalGQL(w io.Writer) {
fmt.Fprint(w, strconv.Quote(e.String()))
}
type OperationEnum string
const (
@@ -2080,7 +1826,6 @@ type PerformerSortEnum string
const (
PerformerSortEnumName PerformerSortEnum = "NAME"
PerformerSortEnumBirthdate PerformerSortEnum = "BIRTHDATE"
PerformerSortEnumDeathdate PerformerSortEnum = "DEATHDATE"
PerformerSortEnumSceneCount PerformerSortEnum = "SCENE_COUNT"
PerformerSortEnumCareerStartYear PerformerSortEnum = "CAREER_START_YEAR"
PerformerSortEnumDebut PerformerSortEnum = "DEBUT"
@@ -2092,7 +1837,6 @@ const (
var AllPerformerSortEnum = []PerformerSortEnum{
PerformerSortEnumName,
PerformerSortEnumBirthdate,
PerformerSortEnumDeathdate,
PerformerSortEnumSceneCount,
PerformerSortEnumCareerStartYear,
PerformerSortEnumDebut,
@@ -2103,7 +1847,7 @@ var AllPerformerSortEnum = []PerformerSortEnum{
func (e PerformerSortEnum) IsValid() bool {
switch e {
case PerformerSortEnumName, PerformerSortEnumBirthdate, PerformerSortEnumDeathdate, PerformerSortEnumSceneCount, PerformerSortEnumCareerStartYear, PerformerSortEnumDebut, PerformerSortEnumLastScene, PerformerSortEnumCreatedAt, PerformerSortEnumUpdatedAt:
case PerformerSortEnumName, PerformerSortEnumBirthdate, PerformerSortEnumSceneCount, PerformerSortEnumCareerStartYear, PerformerSortEnumDebut, PerformerSortEnumLastScene, PerformerSortEnumCreatedAt, PerformerSortEnumUpdatedAt:
return true
}
return false
@@ -2447,55 +2191,6 @@ func (e TargetTypeEnum) MarshalGQL(w io.Writer) {
fmt.Fprint(w, strconv.Quote(e.String()))
}
type UserChangeEmailStatus string
const (
UserChangeEmailStatusConfirmOld UserChangeEmailStatus = "CONFIRM_OLD"
UserChangeEmailStatusConfirmNew UserChangeEmailStatus = "CONFIRM_NEW"
UserChangeEmailStatusExpired UserChangeEmailStatus = "EXPIRED"
UserChangeEmailStatusInvalidToken UserChangeEmailStatus = "INVALID_TOKEN"
UserChangeEmailStatusSuccess UserChangeEmailStatus = "SUCCESS"
UserChangeEmailStatusError UserChangeEmailStatus = "ERROR"
)
var AllUserChangeEmailStatus = []UserChangeEmailStatus{
UserChangeEmailStatusConfirmOld,
UserChangeEmailStatusConfirmNew,
UserChangeEmailStatusExpired,
UserChangeEmailStatusInvalidToken,
UserChangeEmailStatusSuccess,
UserChangeEmailStatusError,
}
func (e UserChangeEmailStatus) IsValid() bool {
switch e {
case UserChangeEmailStatusConfirmOld, UserChangeEmailStatusConfirmNew, UserChangeEmailStatusExpired, UserChangeEmailStatusInvalidToken, UserChangeEmailStatusSuccess, UserChangeEmailStatusError:
return true
}
return false
}
func (e UserChangeEmailStatus) String() string {
return string(e)
}
func (e *UserChangeEmailStatus) UnmarshalGQL(v interface{}) error {
str, ok := v.(string)
if !ok {
return fmt.Errorf("enums must be strings")
}
*e = UserChangeEmailStatus(str)
if !e.IsValid() {
return fmt.Errorf("%s is not a valid UserChangeEmailStatus", str)
}
return nil
}
func (e UserChangeEmailStatus) MarshalGQL(w io.Writer) {
fmt.Fprint(w, strconv.Quote(e.String()))
}
type UserVotedFilterEnum string
const (

View File

@@ -0,0 +1,13 @@
package stashbox
import "github.com/stashapp/stash/pkg/models"
type StashBoxStudioQueryResult struct {
Query string `json:"query"`
Results []*models.ScrapedStudio `json:"results"`
}
type StashBoxPerformerQueryResult struct {
Query string `json:"query"`
Results []*models.ScrapedPerformer `json:"results"`
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,75 +0,0 @@
package scraper
import (
"context"
"regexp"
"strings"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil"
)
func postProcessTags(ctx context.Context, tqb models.TagQueryer, scrapedTags []*models.ScrapedTag) (ret []*models.ScrapedTag, err error) {
ret = make([]*models.ScrapedTag, 0, len(scrapedTags))
for _, t := range scrapedTags {
err := match.ScrapedTag(ctx, tqb, t)
if err != nil {
return nil, err
}
ret = append(ret, t)
}
return ret, err
}
// FilterTags removes tags matching excluded tag patterns from the list of scraped tags
// It returns the filtered list of tags and a list of the excluded tags
func FilterTags(excludeRegexps []*regexp.Regexp, tags []*models.ScrapedTag) (newTags []*models.ScrapedTag, ignoredTags []string) {
if len(excludeRegexps) == 0 {
return tags, nil
}
for _, t := range tags {
ignore := false
for _, reg := range excludeRegexps {
if reg.MatchString(strings.ToLower(t.Name)) {
ignore = true
ignoredTags = sliceutil.AppendUnique(ignoredTags, t.Name)
break
}
}
if !ignore {
newTags = append(newTags, t)
}
}
return newTags, ignoredTags
}
// CompileExclusionRegexps compiles a list of tag exclusion patterns into a list of regular expressions
func CompileExclusionRegexps(patterns []string) []*regexp.Regexp {
excludePatterns := patterns
var excludeRegexps []*regexp.Regexp
for _, excludePattern := range excludePatterns {
reg, err := regexp.Compile(strings.ToLower(excludePattern))
if err != nil {
logger.Errorf("Invalid tag exclusion pattern: %v", err)
} else {
excludeRegexps = append(excludeRegexps, reg)
}
}
return excludeRegexps
}
// LogIgnoredTags logs the list of ignored tags
func LogIgnoredTags(ignoredTags []string) {
if len(ignoredTags) > 0 {
logger.Debugf("Tags ignored for matching exclusion patterns: %s", strings.Join(ignoredTags, ", "))
}
}

View File

@@ -83,12 +83,6 @@ func (s *xpathScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCon
return nil, err
}
return ret, nil
case ScrapeContentTypeImage:
ret, err := scraper.scrapeImage(ctx, q)
if err != nil || ret == nil {
return nil, err
}
return ret, nil
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
ret, err := scraper.scrapeGroup(ctx, q)
if err != nil || ret == nil {
@@ -151,7 +145,7 @@ func (s *xpathScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
return nil, ErrNotSupported
}
func (s *xpathScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
func (s *xpathScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
// construct the URL
queryURL := queryURLParametersFromScene(scene)
if s.scraper.QueryURLReplacements != nil {
@@ -210,7 +204,7 @@ func (s *xpathScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap
return scraper.scrapeScene(ctx, q)
}
func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
// construct the URL
queryURL := queryURLParametersFromGallery(gallery)
if s.scraper.QueryURLReplacements != nil {
@@ -234,30 +228,6 @@ func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
return scraper.scrapeGallery(ctx, q)
}
func (s *xpathScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
// construct the URL
queryURL := queryURLParametersFromImage(image)
if s.scraper.QueryURLReplacements != nil {
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
}
url := queryURL.constructURL(s.scraper.QueryURL)
scraper := s.getXpathScraper()
if scraper == nil {
return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config")
}
doc, err := s.loadURL(ctx, url)
if err != nil {
return nil, err
}
q := s.getXPathQuery(doc)
return scraper.scrapeImage(ctx, q)
}
func (s *xpathScraper) loadURL(ctx context.Context, url string) (*html.Node, error) {
r, err := loadURL(ctx, url, s.client, s.config, s.globalConfig)
if err != nil {

View File

@@ -32,7 +32,6 @@ const htmlDoc1 = `
</td>
<td class="paramvalue">
<a href="/html/m_links/Mia_Malkova/">Mia Malkova</a>&nbsp;
<a href="/html/m_links/Mia_Malkova/second_url">Mia Malkova</a>&nbsp;
</td>
</tr>
<tr>
@@ -207,8 +206,6 @@ func makeXPathConfig() mappedPerformerScraperConfig {
}
config.mappedConfig["Name"] = makeSimpleAttrConfig(makeCommonXPath("Babe Name:") + `/a`)
config.mappedConfig["URL"] = makeSimpleAttrConfig(makeCommonXPath("Babe Name:") + `/a/@href`)
config.mappedConfig["URLs"] = makeSimpleAttrConfig(makeCommonXPath("Babe Name:") + `/a/@href`)
config.mappedConfig["Ethnicity"] = makeSimpleAttrConfig(makeCommonXPath("Ethnicity:"))
config.mappedConfig["Aliases"] = makeSimpleAttrConfig(makeCommonXPath("Aliases:"))
config.mappedConfig["EyeColor"] = makeSimpleAttrConfig(makeCommonXPath("Eye Color:"))
@@ -324,8 +321,6 @@ func TestScrapePerformerXPath(t *testing.T) {
}
const performerName = "Mia Malkova"
const url = "/html/m_links/Mia_Malkova/"
const secondURL = "/html/m_links/Mia_Malkova/second_url"
const ethnicity = "Caucasian"
const country = "United States"
const birthdate = "1992-07-01"
@@ -343,16 +338,6 @@ func TestScrapePerformerXPath(t *testing.T) {
const weight = "57" // 126 lb
verifyField(t, performerName, performer.Name, "Name")
verifyField(t, url, performer.URL, "URL")
// #5294 - test multiple URLs
if len(performer.URLs) != 2 {
t.Errorf("Expected 2 URLs, got %d", len(performer.URLs))
} else {
verifyField(t, url, &performer.URLs[0], "URLs[0]")
verifyField(t, secondURL, &performer.URLs[1], "URLs[1]")
}
verifyField(t, gender, performer.Gender, "Gender")
verifyField(t, ethnicity, performer.Ethnicity, "Ethnicity")
verifyField(t, country, performer.Country, "Country")
@@ -584,7 +569,7 @@ func makeSceneXPathConfig() mappedScraper {
performerConfig := make(mappedConfig)
performerConfig["Name"] = makeSimpleAttrConfig(`$performerElem/@data-mxptext`)
performerConfig["URLs"] = makeSimpleAttrConfig(`$performerElem/@href`)
performerConfig["URL"] = makeSimpleAttrConfig(`$performerElem/@href`)
config.Performers.mappedConfig = performerConfig
studioConfig := make(mappedConfig)
@@ -668,8 +653,8 @@ func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []strin
}
if i < len(actualPerformers) {
actualName = *actualPerformers[i].Name
if len(actualPerformers[i].URLs) == 1 {
actualURL = actualPerformers[i].URLs[0]
if actualPerformers[i].URL != nil {
actualURL = *actualPerformers[i].URL
}
}
@@ -820,7 +805,7 @@ func TestLoadInvalidXPath(t *testing.T) {
doc: doc,
}
config.process(context.Background(), q, nil, nil)
config.process(context.Background(), q, nil)
}
type mockGlobalConfig struct{}

View File

@@ -167,13 +167,3 @@ func ValuesToPtrs[T any](vs []T) []*T {
}
return ret
}
// Flatten returns a single slice containing all elements of the provided
// slice of slices.
func Flatten[T any](vs [][]T) []T {
var ret []T
for _, v := range vs {
ret = append(ret, v...)
}
return ret
}

View File

@@ -816,7 +816,6 @@ func (db *Anonymiser) anonymiseTags(ctx context.Context) error {
query := dialect.From(table).Select(
table.Col(idColumn),
table.Col("name"),
table.Col("sort_name"),
table.Col("description"),
).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
@@ -827,14 +826,12 @@ func (db *Anonymiser) anonymiseTags(ctx context.Context) error {
var (
id int
name sql.NullString
sortName sql.NullString
description sql.NullString
)
if err := rows.Scan(
&id,
&name,
&sortName,
&description,
); err != nil {
return err
@@ -842,7 +839,6 @@ func (db *Anonymiser) anonymiseTags(ctx context.Context) error {
set := goqu.Record{}
db.obfuscateNullString(set, "name", name)
db.obfuscateNullString(set, "sort_name", sortName)
db.obfuscateNullString(set, "description", description)
if len(set) > 0 {

View File

@@ -34,7 +34,7 @@ const (
cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE"
)
var appSchemaVersion uint = 72
var appSchemaVersion uint = 71
//go:embed migrations/*.sql
var migrationsBox embed.FS
@@ -430,19 +430,7 @@ func (db *Database) Vacuum(ctx context.Context) error {
// Analyze runs an ANALYZE on the database to improve query performance.
func (db *Database) Analyze(ctx context.Context) error {
return analyze(ctx, db.writeDB)
}
// analyze runs an ANALYZE on the database to improve query performance.
func analyze(ctx context.Context, db *sqlx.DB) error {
_, err := db.ExecContext(ctx, "ANALYZE")
return err
}
// flushWAL flushes the Write-Ahead Log (WAL) to the main database file.
// It also truncates the WAL file to 0 bytes.
func flushWAL(ctx context.Context, db *sqlx.DB) error {
_, err := db.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
_, err := db.writeDB.ExecContext(ctx, "ANALYZE")
return err
}

View File

@@ -155,7 +155,7 @@ var (
},
fkColumn: "tag_id",
foreignTable: tagTable,
orderBy: "COALESCE(tags.sort_name, tags.name) ASC",
orderBy: "tags.name ASC",
},
images: joinRepository{
repository: repository{

View File

@@ -122,7 +122,7 @@ var (
},
fkColumn: tagIDColumn,
foreignTable: tagTable,
orderBy: "COALESCE(tags.sort_name, tags.name) ASC",
orderBy: "tags.name ASC",
},
}
)

View File

@@ -177,7 +177,7 @@ var (
},
fkColumn: tagIDColumn,
foreignTable: tagTable,
orderBy: "COALESCE(tags.sort_name, tags.name) ASC",
orderBy: "tags.name ASC",
},
}
)

View File

@@ -39,12 +39,6 @@ func NewMigrator(db *Database) (*Migrator, error) {
m.conn.SetConnMaxIdleTime(dbConnTimeout)
m.m, err = m.getMigrate()
// if error encountered, close the connection
if err != nil {
m.Close()
}
return m, err
}
@@ -130,27 +124,6 @@ func (m *Migrator) runCustomMigration(ctx context.Context, fn customMigrationFun
return nil
}
func (m *Migrator) PostMigrate(ctx context.Context) error {
// optimise the database
var err error
logger.Info("Running database analyze")
// don't use Optimize/vacuum as this adds a significant amount of time
// to the migration
err = analyze(ctx, m.conn)
if err == nil {
logger.Debug("Flushing WAL")
err = flushWAL(ctx, m.conn)
}
if err != nil {
return fmt.Errorf("error optimising database: %s", err)
}
return nil
}
func (db *Database) getDatabaseSchemaVersion() (uint, error) {
m, err := NewMigrator(db)
if err != nil {

View File

@@ -1,2 +0,0 @@
ALTER TABLE `tags` ADD COLUMN `sort_name` varchar(255);

View File

@@ -189,7 +189,7 @@ var (
},
fkColumn: tagIDColumn,
foreignTable: tagTable,
orderBy: "COALESCE(tags.sort_name, tags.name) ASC",
orderBy: "tags.name ASC",
},
stashIDs: stashIDRepository{
repository{

View File

@@ -121,14 +121,12 @@ func Test_PerformerStore_Create(t *testing.T) {
Aliases: models.NewRelatedStrings(aliases),
StashIDs: models.NewRelatedStashIDs([]models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
}),
CreatedAt: createdAt,
@@ -285,14 +283,12 @@ func Test_PerformerStore_Update(t *testing.T) {
TagIDs: models.NewRelatedIDs([]int{tagIDs[tagIdx1WithPerformer], tagIDs[tagIdx1WithDupName]}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
}),
CreatedAt: createdAt,
@@ -522,14 +518,12 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
StashIDs: &models.UpdateStashIDs{
StashIDs: []models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
},
Mode: models.RelationshipUpdateModeSet,
@@ -566,14 +560,12 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
TagIDs: models.NewRelatedIDs([]int{tagIDs[tagIdx1WithPerformer], tagIDs[tagIdx1WithDupName]}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
}),
CreatedAt: createdAt,
@@ -2083,9 +2075,8 @@ func testPerformerStashIDs(ctx context.Context, t *testing.T, s *models.Performe
const stashIDStr = "stashID"
const endpoint = "endpoint"
stashID := models.StashID{
StashID: stashIDStr,
Endpoint: endpoint,
UpdatedAt: epochTime,
StashID: stashIDStr,
Endpoint: endpoint,
}
qb := db.Performer

View File

@@ -201,7 +201,7 @@ var (
},
fkColumn: tagIDColumn,
foreignTable: tagTable,
orderBy: "COALESCE(tags.sort_name, tags.name) ASC",
orderBy: "tags.name ASC",
},
performers: joinRepository{
repository: repository{
@@ -1056,7 +1056,6 @@ func (qb *SceneStore) QueryCount(ctx context.Context, sceneFilter *models.SceneF
var sceneSortOptions = sortOptions{
"bitrate",
"created_at",
"code",
"date",
"file_count",
"filesize",

View File

@@ -133,14 +133,12 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
}),
ResumeTime: float64(resumeTime),
@@ -180,14 +178,12 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
}),
ResumeTime: resumeTime,
@@ -366,14 +362,12 @@ func Test_sceneQueryBuilder_Update(t *testing.T) {
}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
}),
ResumeTime: resumeTime,
@@ -596,14 +590,12 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
StashIDs: &models.UpdateStashIDs{
StashIDs: []models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
},
Mode: models.RelationshipUpdateModeSet,
@@ -642,14 +634,12 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
}),
ResumeTime: resumeTime,
@@ -751,14 +741,12 @@ func Test_sceneQueryBuilder_UpdatePartialRelationships(t *testing.T) {
stashIDs = []models.StashID{
{
StashID: stashID1,
Endpoint: endpoint1,
UpdatedAt: epochTime,
StashID: stashID1,
Endpoint: endpoint1,
},
{
StashID: stashID2,
Endpoint: endpoint2,
UpdatedAt: epochTime,
StashID: stashID2,
Endpoint: endpoint2,
},
}
)
@@ -4389,9 +4377,8 @@ func testSceneStashIDs(ctx context.Context, t *testing.T, s *models.Scene) {
const stashIDStr = "stashID"
const endpoint = "endpoint"
stashID := models.StashID{
StashID: stashIDStr,
Endpoint: endpoint,
UpdatedAt: epochTime,
StashID: stashIDStr,
Endpoint: endpoint,
}
qb := db.Scene

View File

@@ -24,8 +24,6 @@ import (
_ "github.com/stashapp/stash/pkg/sqlite/migrations"
)
var epochTime = time.Unix(0, 0).UTC()
const (
spacedSceneTitle = "zzz yyy xxx"
)
@@ -1030,9 +1028,8 @@ func getObjectDate(index int) *models.Date {
func sceneStashID(i int) models.StashID {
return models.StashID{
StashID: getSceneStringValue(i, "stashid"),
Endpoint: getSceneStringValue(i, "endpoint"),
UpdatedAt: epochTime,
StashID: getSceneStringValue(i, "stashid"),
Endpoint: getSceneStringValue(i, "endpoint"),
}
}

View File

@@ -133,7 +133,7 @@ var (
},
fkColumn: tagIDColumn,
foreignTable: tagTable,
orderBy: "COALESCE(tags.sort_name, tags.name) ASC",
orderBy: "tags.name ASC",
},
}
)
@@ -586,7 +586,6 @@ var studioSortOptions = sortOptions{
"scenes_count",
"random",
"rating",
"tag_count",
"updated_at",
}

View File

@@ -618,9 +618,6 @@ func testStudioStashIDs(ctx context.Context, t *testing.T, s *models.Studio) {
return
}
// #5563 - set the UpdatedAt field to epoch
stashID.UpdatedAt = epochTime
assert.Equal(t, []models.StashID{stashID}, s.StashIDs.List())
// remove stash ids and ensure was updated

View File

@@ -309,15 +309,7 @@ func (t *stashIDTable) get(ctx context.Context, id int) ([]models.StashID, error
return ret, nil
}
var epochTime = time.Unix(0, 0).UTC()
func (t *stashIDTable) insertJoin(ctx context.Context, id int, v models.StashID) (sql.Result, error) {
// #5563 - it's possible that zero-value updated at timestamps are provided via import
// replace them with the epoch time
if v.UpdatedAt.IsZero() {
v.UpdatedAt = epochTime
}
var q = dialect.Insert(t.table.table).Cols(t.idColumn.GetCol(), "endpoint", "stash_id", "updated_at").Vals(
goqu.Vals{id, v.Endpoint, v.StashID, v.UpdatedAt},
)

View File

@@ -33,7 +33,6 @@ const (
type tagRow struct {
ID int `db:"id" goqu:"skipinsert"`
Name null.String `db:"name"` // TODO: make schema non-nullable
SortName zero.String `db:"sort_name"`
Favorite bool `db:"favorite"`
Description zero.String `db:"description"`
IgnoreAutoTag bool `db:"ignore_auto_tag"`
@@ -47,7 +46,6 @@ type tagRow struct {
func (r *tagRow) fromTag(o models.Tag) {
r.ID = o.ID
r.Name = null.StringFrom(o.Name)
r.SortName = zero.StringFrom((o.SortName))
r.Favorite = o.Favorite
r.Description = zero.StringFrom(o.Description)
r.IgnoreAutoTag = o.IgnoreAutoTag
@@ -59,7 +57,6 @@ func (r *tagRow) resolve() *models.Tag {
ret := &models.Tag{
ID: r.ID,
Name: r.Name.String,
SortName: r.SortName.String,
Favorite: r.Favorite,
Description: r.Description.String,
IgnoreAutoTag: r.IgnoreAutoTag,
@@ -90,7 +87,6 @@ type tagRowRecord struct {
func (r *tagRowRecord) fromPartial(o models.TagPartial) {
r.setString("name", o.Name)
r.setNullString("sort_name", o.SortName)
r.setNullString("description", o.Description)
r.setBool("favorite", o.Favorite)
r.setBool("ignore_auto_tag", o.IgnoreAutoTag)
@@ -676,8 +672,6 @@ func (qb *TagStore) getTagSort(query *queryBuilder, findFilter *models.FindFilte
sortQuery := ""
switch sort {
case "name":
sortQuery += fmt.Sprintf(" ORDER BY COALESCE(tags.sort_name, tags.name) COLLATE NATURAL_CI %s", getSortDirection(direction))
case "scenes_count":
sortQuery += getCountSort(tagTable, scenesTagsTable, tagIDColumn, direction)
case "scene_markers_count":
@@ -696,8 +690,8 @@ func (qb *TagStore) getTagSort(query *queryBuilder, findFilter *models.FindFilte
sortQuery += getSort(sort, direction, "tags")
}
// Whatever the sorting, always use sort_name/name/id as a final sort
sortQuery += ", COALESCE(tags.sort_name, tags.name, tags.id) COLLATE NATURAL_CI ASC"
// Whatever the sorting, always use name/id as a final sort
sortQuery += ", COALESCE(tags.name, tags.id) COLLATE NATURAL_CI ASC"
return sortQuery, nil
}

View File

@@ -62,7 +62,6 @@ func (qb *tagFilterHandler) criterionHandler() criterionHandler {
tagFilter := qb.tagFilter
return compoundHandler{
stringCriterionHandler(tagFilter.Name, tagTable+".name"),
stringCriterionHandler(tagFilter.SortName, tagTable+".sort_name"),
qb.aliasCriterionHandler(tagFilter.Aliases),
boolCriterionHandler(tagFilter.Favorite, tagTable+".favorite", nil),

View File

@@ -1,101 +0,0 @@
// Package stashbox provides a client interface to a stash-box server instance.
package stashbox
import (
"context"
"net/http"
"regexp"
"github.com/Yamashou/gqlgenc/clientv2"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/stashbox/graphql"
"golang.org/x/time/rate"
)
// DefaultMaxRequestsPerMinute is the default maximum number of requests per minute.
const DefaultMaxRequestsPerMinute = 240
// Client represents the client interface to a stash-box server instance.
type Client struct {
client *graphql.Client
box models.StashBox
maxRequestsPerMinute int
// tag patterns to be excluded
excludeTagRE []*regexp.Regexp
}
type ClientOption func(*Client)
func ExcludeTagPatterns(patterns []string) ClientOption {
return func(c *Client) {
c.excludeTagRE = scraper.CompileExclusionRegexps(patterns)
}
}
func MaxRequestsPerMinute(n int) ClientOption {
return func(c *Client) {
if n > 0 {
c.maxRequestsPerMinute = n
}
}
}
func setApiKeyHeader(apiKey string) clientv2.RequestInterceptor {
return func(ctx context.Context, req *http.Request, gqlInfo *clientv2.GQLRequestInfo, res interface{}, next clientv2.RequestInterceptorFunc) error {
req.Header.Set("ApiKey", apiKey)
return next(ctx, req, gqlInfo, res)
}
}
func rateLimit(n int) clientv2.RequestInterceptor {
perSec := float64(n) / 60
limiter := rate.NewLimiter(rate.Limit(perSec), 1)
return func(ctx context.Context, req *http.Request, gqlInfo *clientv2.GQLRequestInfo, res interface{}, next clientv2.RequestInterceptorFunc) error {
if err := limiter.Wait(ctx); err != nil {
// should only happen if the context is canceled
return err
}
return next(ctx, req, gqlInfo, res)
}
}
// NewClient returns a new instance of a stash-box client.
func NewClient(box models.StashBox, options ...ClientOption) *Client {
ret := &Client{
box: box,
maxRequestsPerMinute: DefaultMaxRequestsPerMinute,
}
if box.MaxRequestsPerMinute > 0 {
ret.maxRequestsPerMinute = box.MaxRequestsPerMinute
}
for _, option := range options {
option(ret)
}
authHeader := setApiKeyHeader(box.APIKey)
limitRequests := rateLimit(ret.maxRequestsPerMinute)
client := &graphql.Client{
Client: clientv2.NewClient(http.DefaultClient, box.Endpoint, nil, authHeader, limitRequests),
}
ret.client = client
return ret
}
func (c Client) getHTTPClient() *http.Client {
return c.client.Client.Client
}
func (c Client) GetUser(ctx context.Context) (*graphql.Me, error) {
return c.client.Me(ctx)
}

View File

@@ -1,132 +0,0 @@
package stashbox
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"github.com/Yamashou/gqlgenc/clientv2"
"github.com/Yamashou/gqlgenc/graphqljson"
)
func (c *Client) submitDraft(ctx context.Context, query string, input interface{}, image io.Reader, ret interface{}) error {
vars := map[string]interface{}{
"input": input,
}
r := &clientv2.Request{
Query: query,
Variables: vars,
OperationName: "",
}
requestBody, err := json.Marshal(r)
if err != nil {
return fmt.Errorf("encode: %w", err)
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
if err := writer.WriteField("operations", string(requestBody)); err != nil {
return err
}
if image != nil {
if err := writer.WriteField("map", "{ \"0\": [\"variables.input.image\"] }"); err != nil {
return err
}
part, _ := writer.CreateFormFile("0", "draft")
if _, err := io.Copy(part, image); err != nil {
return err
}
} else if err := writer.WriteField("map", "{}"); err != nil {
return err
}
writer.Close()
req, _ := http.NewRequestWithContext(ctx, "POST", c.box.Endpoint, body)
req.Header.Add("Content-Type", writer.FormDataContentType())
req.Header.Set("ApiKey", c.box.APIKey)
httpClient := c.client.Client.Client
resp, err := httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
responseBytes, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
type response struct {
Data json.RawMessage `json:"data"`
Errors json.RawMessage `json:"errors"`
}
var respGQL response
if err := json.Unmarshal(responseBytes, &respGQL); err != nil {
return fmt.Errorf("failed to decode data %s: %w", string(responseBytes), err)
}
if len(respGQL.Errors) > 0 {
// try to parse standard graphql error
errors := &clientv2.GqlErrorList{}
if e := json.Unmarshal(responseBytes, errors); e != nil {
return fmt.Errorf("failed to parse graphql errors. Response content %s - %w ", string(responseBytes), e)
}
return errors
}
if err := graphqljson.UnmarshalData(respGQL.Data, ret); err != nil {
return err
}
return err
}
// we can't currently use this due to https://github.com/Yamashou/gqlgenc/issues/109
// func uploadImage(image io.Reader) client.HTTPRequestOption {
// return func(req *http.Request) {
// if image == nil {
// // return without changing anything
// return
// }
// // we can't handle errors in here, so if one happens, just return
// // without changing anything.
// // repackage the request to include the image
// bodyBytes, err := ioutil.ReadAll(req.Body)
// if err != nil {
// return
// }
// newBody := &bytes.Buffer{}
// writer := multipart.NewWriter(newBody)
// _ = writer.WriteField("operations", string(bodyBytes))
// if err := writer.WriteField("map", "{ \"0\": [\"variables.input.image\"] }"); err != nil {
// return
// }
// part, _ := writer.CreateFormFile("0", "draft")
// if _, err := io.Copy(part, image); err != nil {
// return
// }
// writer.Close()
// // now set the request body to this new body
// req.Body = io.NopCloser(newBody)
// req.ContentLength = int64(newBody.Len())
// req.Header.Set("Content-Type", writer.FormDataContentType())
// }
// }

Some files were not shown because too many files have changed in this diff Show More