mirror of
https://github.com/stashapp/CommunityScripts.git
synced 2026-04-21 22:37:24 -05:00
A few more features, funscripts matching, excluding markers with matching strings and adding a string to new markers.
Indexing funscripts is done with a task creating a database with the funscript path and a md5 of the script. Submitting funscripts is done by the submit tasks. On query a list of funscript hashes are returned by the api and if thre is a matching script it will copy the file to the correct name for stash to match it. Added an option to exclude markers that contain a string. Added an option to add the [Timestamp] tag to markers the plugin creates. Added an option to add the string [TsTrade] to markers eg "[TsTrade] marker name".
This commit is contained in:
Tweeticoats
@@ -8,6 +8,10 @@ import time
|
||||
import math
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import hashlib
|
||||
import shutil
|
||||
import re
|
||||
|
||||
per_page = 100
|
||||
request_s = requests.Session()
|
||||
@@ -40,13 +44,10 @@ def processSceneTimestamTrade(s):
|
||||
# log.debug(data)
|
||||
# log.debug(s["scene_markers"])
|
||||
# log.debug(len(s["scene_markers"]) > 0)
|
||||
if (
|
||||
settings["createMarkers"]
|
||||
and (len(s["scene_markers"]) == 0)
|
||||
or settings["overwriteMarkers"]
|
||||
):
|
||||
if settings["createMarkers"]:
|
||||
log.debug("creating markers")
|
||||
markers = []
|
||||
previous = {"title": "", "seconds": -1}
|
||||
for m in data["markers"]:
|
||||
|
||||
marker = {
|
||||
@@ -55,17 +56,38 @@ def processSceneTimestamTrade(s):
|
||||
"tags": [],
|
||||
"title": m["name"],
|
||||
}
|
||||
if settings["addTsTradeTag"]:
|
||||
marker["tags"].append(int(getTag("[Timestamp]")))
|
||||
|
||||
if m["tag_name"]:
|
||||
marker["primary_tag"] = m["tag_name"]
|
||||
else:
|
||||
marker["primary_tag"] = m["name"]
|
||||
if len(marker["primary_tag"]) > 0:
|
||||
|
||||
if settings["addTsTradeTitle"]:
|
||||
marker["title"] = f"[TsTrade] {m["name"]}"
|
||||
|
||||
#check for markers with a zero length title, skip adding
|
||||
if len(marker["primary_tag"]) == 0:
|
||||
True
|
||||
elif excluded_marker_tag(marker):
|
||||
True
|
||||
# check for duplicate markers
|
||||
elif marker["title"] == previous["title"] and marker["seconds"] == previous["seconds"]:
|
||||
True
|
||||
else:
|
||||
markers.append(marker)
|
||||
previous = marker
|
||||
# log.debug(marker)
|
||||
if len(markers) > 0:
|
||||
log.debug(markers)
|
||||
if settings["overwriteMarkers"]:
|
||||
stash.destroy_scene_markers(s["id"])
|
||||
mp.import_scene_markers(stash, markers, s["id"], 15)
|
||||
mp.import_scene_markers(stash, markers, s["id"], 15)
|
||||
elif len(s["scene_markers"]) == 0 or settings["mergeMarkers"]:
|
||||
mp.import_scene_markers(stash, markers, s["id"], 15)
|
||||
|
||||
|
||||
new_scene = {
|
||||
"id": s["id"],
|
||||
}
|
||||
@@ -273,13 +295,23 @@ def processSceneTimestamTrade(s):
|
||||
if m["movie_id"] not in [x["movie_id"] for x in new_scene["movies"]]:
|
||||
new_scene["movies"].append(m)
|
||||
needs_update = True
|
||||
log.debug(s)
|
||||
# log.debug(s)
|
||||
if getTag("[Timestamp: Auto Gallery]") in [x["id"] for x in s["tags"]]:
|
||||
autoGallery=True
|
||||
for g in s["galleries"]:
|
||||
gal=stash.find_gallery(g['id'])
|
||||
if getTag("[Timestamp: Auto Gallery]") in [x["id"] for x in gal["tags"]]:
|
||||
autoGallery=False
|
||||
if autoGallery:
|
||||
g1 = stash.find_galleries(f={"url": {"modifier": "EQUALS", "value": url }})
|
||||
if len(g1) > 0:
|
||||
if "gallery_ids" not in new_scene:
|
||||
new_scene["gallery_ids"] = [x["id"] for x in s["galleries"]]
|
||||
for g2 in g1:
|
||||
if g2['id'] not in new_scene["gallery_ids"]:
|
||||
new_scene["gallery_ids"].append(g2['id'])
|
||||
needs_update = True
|
||||
autoGallery=False
|
||||
if autoGallery:
|
||||
log.debug("creating auto gallery")
|
||||
# check the gallery if we have already
|
||||
@@ -300,11 +332,26 @@ def processSceneTimestamTrade(s):
|
||||
gal = stash.create_gallery(gallery_input)
|
||||
new_scene["gallery_ids"]=[x["id"] for x in s["galleries"]]
|
||||
new_scene["gallery_ids"].append(gal)
|
||||
needs_update=True
|
||||
|
||||
|
||||
needs_update= True
|
||||
else:
|
||||
log.debug("auto gallery already exists")
|
||||
log.debug(data.keys())
|
||||
if settings["matchFunscripts"] and "funscripts" in data:
|
||||
if not s["interactive"]:
|
||||
log.debug(s.keys())
|
||||
for fs in data["funscripts"]:
|
||||
log.debug(fs["md5"])
|
||||
conn = db_migrations()
|
||||
cur = conn.cursor()
|
||||
res = cur.execute("select id,filename,scene_id from script_index where md5=?", (fs["md5"],));
|
||||
for row in res.fetchall():
|
||||
if len(s["files"]) > 0:
|
||||
log.debug("found matching funscript, copying funscript")
|
||||
scriptfile_source=Path(row[1])
|
||||
video_file=Path(s["files"][0]["path"])
|
||||
scriptfile_destination = video_file.parent / (video_file.stem + ".funscript")
|
||||
log.info("copying funscript %s, to destination %s,"% (scriptfile_source,scriptfile_destination,))
|
||||
shutil.copyfile(scriptfile_source,scriptfile_destination)
|
||||
|
||||
if needs_update:
|
||||
log.debug("updating scene: %s" % (new_scene,))
|
||||
@@ -383,7 +430,8 @@ def processAll(query):
|
||||
|
||||
|
||||
def submitScene(query):
|
||||
scene_fgmt = """title
|
||||
scene_fgmt = """id
|
||||
title
|
||||
details
|
||||
urls
|
||||
date
|
||||
@@ -476,7 +524,8 @@ def submitScene(query):
|
||||
}
|
||||
"""
|
||||
if settings["schema"] >= 64:
|
||||
scene_fgmt = """title
|
||||
scene_fgmt = """id
|
||||
title
|
||||
details
|
||||
urls
|
||||
date
|
||||
@@ -584,7 +633,17 @@ def submitScene(query):
|
||||
)
|
||||
for s in scenes:
|
||||
log.debug("submitting scene: " + str(s))
|
||||
request_s.post("https://timestamp.trade/submit-stash", json=s)
|
||||
if settings["submitFunscriptHash"]:
|
||||
log.debug(s)
|
||||
s["funscriptHashes"]=[]
|
||||
conn=db_migrations()
|
||||
cur=conn.cursor()
|
||||
res=cur.execute("select id,filename,metadata,scene_id,md5 from script_index where scene_id=?",(s["id"],))
|
||||
for row in res.fetchall():
|
||||
s["funscriptHashes"].append({"filename": str(Path(row[1]).name),"metadata":json.loads(row[2]), "md5": row[4]})
|
||||
s.pop("id")
|
||||
log.debug(s)
|
||||
# request_s.post("https://timestamp.trade/submit-stash", json=s)
|
||||
i = i + 1
|
||||
log.progress((i / count))
|
||||
time.sleep(0.5)
|
||||
@@ -761,6 +820,8 @@ def processGallery(gallery):
|
||||
else:
|
||||
if "https://timestamp.trade/scene/" in [u[:30] for u in gallery["urls"]]:
|
||||
downloadGallery(gallery)
|
||||
stash.metadata_scan(paths=[settings["path"]])
|
||||
return
|
||||
# Process the gallery if it has the [Timestamp: Tag Gallery] tag
|
||||
tag_gallery_tag_id = stash.find_tag("[Timestamp: Tag Gallery]", create=True).get(
|
||||
"id"
|
||||
@@ -934,24 +995,82 @@ def downloadGallery(gallery):
|
||||
if scene:
|
||||
image_data["performer_ids"].extend([x["id"] for x in scene["performers"]])
|
||||
else:
|
||||
for p in gallery["performers"]:
|
||||
perf=stash.find_performers(p["name"])
|
||||
for p in data["performers"]:
|
||||
perf=stash.find_performers(q=p["name"])
|
||||
for p1 in perf:
|
||||
image_data["performer_ids"].append(p1["id"])
|
||||
|
||||
log.debug(image_data)
|
||||
log.info("Downloading image %s to file %s" % (i["url"],str(image_file),))
|
||||
r = requests.get(i["url"])
|
||||
if r.status_code==200:
|
||||
with open(metadata_file, "w") as f:
|
||||
json.dump(image_data, f)
|
||||
with open(image_file, "wb") as f:
|
||||
f.write(r.content)
|
||||
f.close()
|
||||
try:
|
||||
r = requests.get(i["url"])
|
||||
if r.status_code==200:
|
||||
with open(metadata_file, "w") as f:
|
||||
json.dump(image_data, f)
|
||||
with open(image_file, "wb") as f:
|
||||
f.write(r.content)
|
||||
f.close()
|
||||
except requests.RequestException as e:
|
||||
log.error(e)
|
||||
else:
|
||||
log.debug('img: %s' % (img[0],))
|
||||
|
||||
new_image= {"id": img[0]["id"]}
|
||||
needs_update=False
|
||||
if len(img[0]["performers"]) ==0:
|
||||
new_image["performer_ids"]= []
|
||||
needs_update= True
|
||||
for p in data["performers"]:
|
||||
log.debug(p["name"])
|
||||
perf= stash.find_performers(q=p["name"])
|
||||
for p1 in perf:
|
||||
new_image["performer_ids"].append(p1["id"])
|
||||
|
||||
if needs_update:
|
||||
log.debug(new_image)
|
||||
stash.update_image(new_image)
|
||||
|
||||
True
|
||||
|
||||
counts[i["type"]]=counts[i["type"]]+1
|
||||
|
||||
def reDownloadGallery():
|
||||
query={
|
||||
"tags": {
|
||||
"depth": 0,
|
||||
"excludes": [],
|
||||
"modifier": "INCLUDES_ALL",
|
||||
"value": [getTag("[Timestamp: Auto Gallery]")],
|
||||
}
|
||||
}
|
||||
log.info("Getting gallery count")
|
||||
count = stash.find_galleries(
|
||||
f=query,
|
||||
filter={"per_page": 1},
|
||||
get_count=True,
|
||||
)[0]
|
||||
log.info(str(count) + " galleries to process.")
|
||||
for r in range(1, int(count / per_page) + 2):
|
||||
i= (r-1)*per_page
|
||||
log.info(
|
||||
"fetching data: %s - %s %0.1f%%"
|
||||
% (
|
||||
(r - 1) * per_page,
|
||||
r * per_page,
|
||||
(i / count) * 100,
|
||||
)
|
||||
)
|
||||
galleries = stash.find_galleries(
|
||||
f=query,
|
||||
filter={"page": r, "per_page": per_page},
|
||||
)
|
||||
for g in galleries:
|
||||
downloadGallery(g)
|
||||
i = i + 1
|
||||
log.progress((i / count))
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
counts[i["type"]]=counts[i["type"]]+1
|
||||
stash.metadata_scan(paths=[settings["path"]])
|
||||
|
||||
|
||||
def getImages(gallery_id):
|
||||
@@ -1000,13 +1119,93 @@ def processImages(img):
|
||||
# log.debug(image_data)
|
||||
stash.update_image(image_data)
|
||||
|
||||
def db_migrations():
|
||||
con = sqlite3.connect(settings["funscript_dbpath"])
|
||||
cur = con.cursor()
|
||||
res=cur.execute("SELECT count(name) FROM sqlite_master WHERE type='table' and name='schema_migrations'")
|
||||
if res.fetchone()[0]==0:
|
||||
log.debug("creating table")
|
||||
cur.execute("CREATE TABLE schema_migrations (version uint64,start timestamp,dirty bool);")
|
||||
cur.execute("insert into schema_migrations (version,start,dirty ) values (0,datetime('now'),false);")
|
||||
con.commit()
|
||||
res=cur.execute("select max(version) from schema_migrations;")
|
||||
funscript_schema=res.fetchone()[0]
|
||||
if funscript_schema ==0:
|
||||
cur.execute("insert into schema_migrations (version,start,dirty ) values (1,datetime('now'),true);")
|
||||
cur.execute("CREATE TABLE script_index (id INTEGER PRIMARY KEY, filename text,metadata text,scene_id text,md5 text);")
|
||||
cur.execute('update schema_migrations set dirty=False where version=1')
|
||||
return con
|
||||
|
||||
|
||||
def funscript_index(path):
|
||||
conn= db_migrations()
|
||||
cur = conn.cursor()
|
||||
for file in path.glob("**/*.funscript"):
|
||||
log.info('indexing script file %s' % (file,))
|
||||
with open(file, 'rb') as f:
|
||||
data = f.read()
|
||||
hash = hashlib.md5(data).hexdigest()
|
||||
log.debug(hash)
|
||||
d = json.loads(data)
|
||||
metadata={}
|
||||
if "metadata" in d:
|
||||
metadata=d["metadata"]
|
||||
res=cur.execute('select count(*) from script_index where filename=? ', (str(file.resolve()),))
|
||||
if res.fetchone()[0]==0:
|
||||
cur.execute('insert into script_index (filename,metadata,md5)values (?,?,?)',(str(file.resolve()),json.dumps(metadata),hash))
|
||||
conn.commit()
|
||||
res = cur.execute('select count(*) from script_index ')
|
||||
funscript_count=res.fetchone()[0]
|
||||
log.info('finished indexing funscripts, %s scripts indexed, matching to scenes' % (funscript_count,))
|
||||
res=cur.execute("select id,filename,scene_id from script_index where scene_id is null;");
|
||||
for row in res.fetchall():
|
||||
id=row[0]
|
||||
filename=row[1]
|
||||
scenes=stash.find_scenes(f={"path": {"modifier": "INCLUDES", "value": Path(filename).stem}},fragment="id\nfiles{basename}")
|
||||
i=0
|
||||
for s in scenes:
|
||||
|
||||
for f in s["files"]:
|
||||
if Path(filename).stem == Path(f['basename']).stem:
|
||||
log.info("matching scene %s to script %s" % (s["id"], filename,))
|
||||
cur.execute("update script_index set scene_id=? where id=?", (s["id"], id))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def excluded_marker_tag(marker):
|
||||
"""
|
||||
Check if a marker should be excluded.
|
||||
|
||||
.. notes::
|
||||
- basic user input val is only alpha and len >= 4
|
||||
- startswith and len diff <= 3 is for inflectional endings, not perfect
|
||||
- primary_tag is split and cleaned of non-alphanumeric chars for comparison
|
||||
"""
|
||||
clean_input_pattern = re.compile(r"^[a-zA-Z]+$")
|
||||
excluded_words = {
|
||||
word.strip().lower()
|
||||
for word in settings.get("excludedMarkerWords", "").split(",")
|
||||
if len(word.strip()) >= 4 and clean_input_pattern.match(word.strip())
|
||||
}
|
||||
primary_tag_words = re.sub(r'[^\w\s]', '', marker["primary_tag"].lower()).split()
|
||||
if any(
|
||||
tag_word.startswith(word) and len(tag_word) - len(word) <= 3
|
||||
for word in excluded_words
|
||||
for tag_word in primary_tag_words
|
||||
):
|
||||
log.info(f'EXCLUDE: {marker["primary_tag"]} @ {marker["seconds"]}')
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
json_input = json.loads(sys.stdin.read())
|
||||
|
||||
FRAGMENT_SERVER = json_input["server_connection"]
|
||||
stash = StashInterface(FRAGMENT_SERVER)
|
||||
config = stash.get_configuration()["plugins"]
|
||||
config = stash.get_configuration()
|
||||
settings = {
|
||||
"createGalleryFromScene": False,
|
||||
"createMovieFromScene": False,
|
||||
@@ -1016,14 +1215,21 @@ settings = {
|
||||
"disableGalleryLookupHook": False,
|
||||
"createMarkers": True,
|
||||
"overwriteMarkers": False,
|
||||
"mergeMarkers": False,
|
||||
"createGalleries": True,
|
||||
"submitFunscriptHash":True,
|
||||
"excludedMarkerWords":"",
|
||||
"matchFunscripts":True,
|
||||
"addTsTradeTitle":False,
|
||||
}
|
||||
if "timestampTrade" in config:
|
||||
settings.update(config["timestampTrade"])
|
||||
if "timestampTrade" in config["plugins"]:
|
||||
settings.update(config["plugins"]["timestampTrade"])
|
||||
|
||||
|
||||
# check the schema version for features in the dev release
|
||||
res=stash.callGQL("{systemStatus {databaseSchema}}")
|
||||
res=stash.callGQL("{systemStatus {databaseSchema databasePath}}")
|
||||
settings["schema"]=res["systemStatus"]["databaseSchema"]
|
||||
settings["funscript_dbpath"]=Path (res["systemStatus"]["databasePath"]).parent / "funscript_index.sqlite"
|
||||
log.debug("settings: %s " % (settings,))
|
||||
|
||||
|
||||
@@ -1085,6 +1291,21 @@ if "mode" in json_input["args"]:
|
||||
"url": {"modifier": "INCLUDES", "value": "eroscripts.com"},
|
||||
}
|
||||
submitScene(query)
|
||||
elif "submitInteractiveScene" == PLUGIN_ARGS:
|
||||
skip_submit_tag_id = stash.find_tag(
|
||||
"[Timestamp: Skip Submit]", create=True
|
||||
).get("id")
|
||||
|
||||
query = {
|
||||
"tags": {
|
||||
"depth": 0,
|
||||
"excludes": [skip_submit_tag_id],
|
||||
"modifier": "INCLUDES_ALL",
|
||||
"value": [],
|
||||
},
|
||||
"interactive": True,
|
||||
}
|
||||
submitScene(query)
|
||||
|
||||
elif "submitGallery" == PLUGIN_ARGS:
|
||||
submitGallery()
|
||||
@@ -1150,7 +1371,12 @@ if "mode" in json_input["args"]:
|
||||
},
|
||||
}
|
||||
processAll(query)
|
||||
|
||||
elif "reauto" == PLUGIN_ARGS:
|
||||
reDownloadGallery()
|
||||
stash.metadata_scan(paths=[settings["path"]])
|
||||
elif "indexFunscripts" == PLUGIN_ARGS:
|
||||
for dir in config["general"]["stashes"]:
|
||||
funscript_index(Path(dir["path"]))
|
||||
|
||||
elif "hookContext" in json_input["args"]:
|
||||
_id = json_input["args"]["hookContext"]["id"]
|
||||
@@ -1176,4 +1402,3 @@ elif "hookContext" in json_input["args"]:
|
||||
if _type == "Image.Create.Post":
|
||||
img = stash.find_image(image_in=_id)
|
||||
processImages(img)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
name: Timestamp Trade
|
||||
description: Sync Markers with timestamp.trade, a new database for sharing markers.
|
||||
version: 0.7
|
||||
version: 0.8
|
||||
url: https://github.com/stashapp/CommunityScripts/
|
||||
exec:
|
||||
- python
|
||||
@@ -35,6 +35,26 @@ settings:
|
||||
displayName: Download parent folder
|
||||
description: Download location for files, note this should be in a different folder to stash and in a folder covered by stash. You may need to create a new library path to cover this directory.
|
||||
type: STRING
|
||||
addTsTradeTag:
|
||||
displayName: Add the [Timestamp] tag to created markers.
|
||||
type: BOOLEAN
|
||||
addTsTradeTitle:
|
||||
displayName: Add "TsTrade" to the start of generated marker titles.
|
||||
type: BOOLEAN
|
||||
excludedMarkerWords:
|
||||
displayName: Exclude markers containing these words
|
||||
description: Comma separated list of words to exclude
|
||||
type: STRING
|
||||
matchFunscripts:
|
||||
displayName: Match funscripts
|
||||
type: BOOLEAN
|
||||
overwriteMarkers:
|
||||
displayName: Overwrite Markers
|
||||
type: BOOLEAN
|
||||
mergeMarkers:
|
||||
displayName: Merge Markers
|
||||
type: BOOLEAN
|
||||
|
||||
hooks:
|
||||
- name: Add Marker to Scene
|
||||
description: Makes Markers checking against timestamp.trade
|
||||
@@ -61,6 +81,10 @@ tasks:
|
||||
description: Submit scenes with a eroscripts.com forum post
|
||||
defaultArgs:
|
||||
mode: submitEroscriptScene
|
||||
- name: "Submit Scenes with a funscript"
|
||||
description: Submit scenes with a funscript
|
||||
defaultArgs:
|
||||
mode: submitInteractiveScene
|
||||
- name: "Sync"
|
||||
description: Get markers for all scenes with a stashid
|
||||
defaultArgs:
|
||||
@@ -69,7 +93,7 @@ tasks:
|
||||
description: reprocess scenes with a timestamp.trade url
|
||||
defaultArgs:
|
||||
mode: reprocessScene
|
||||
- name: "Re-process All"
|
||||
- name: "Process All"
|
||||
description: reprocess all scenes with any stash-box id
|
||||
defaultArgs:
|
||||
mode: processAll
|
||||
@@ -81,3 +105,8 @@ tasks:
|
||||
description: get gallery info from timestamp.trade
|
||||
defaultArgs:
|
||||
mode: processGallery
|
||||
- name: "Index Funscript "
|
||||
description: scan for funscript files to be matched and submitted
|
||||
defaultArgs:
|
||||
mode: indexFunscripts
|
||||
|
||||
|
||||
Reference in New Issue
Block a user