Merge pull request #1538 from WithoutPants/package-management

Restructuring and index building

.github/workflows/deploy.yml

@@ -0,0 +1,51 @@
+name: Deploy repository to Github Pages
+
+on:
+  push:
+    branches: [ master, stable ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Checkout main
+      uses: actions/checkout@v2
+      with:
+        path: master
+        ref: master
+        fetch-depth: '0'
+    - run: |
+        cd master
+        ./build_site.sh ../_site/develop
+    # uncomment this once we have a stable branch
+    # - name: Checkout Stable
+    #   uses: actions/checkout@v2
+    #   with:
+    #     path: stable
+    #     ref: stable
+    #     fetch-depth: '0'
+    # - run: |
+    #     cd stable
+    #     ../master/build_site.sh ../_site/stable
+    - uses: actions/upload-pages-artifact@v2
+    
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-22.04
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
+    

.gitignore

@@ -5,3 +5,5 @@ yarn-error.log
 # Scraper-generated files
 /scrapers/*.ini
 **/__pycache__/
+
+/_site

build_site.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# builds a repository of scrapers
+# outputs to _site with the following structure:
+# index.yml
+# <scraper_id>.zip
+# Each zip file contains the scraper.yml file and any other files in the same directory
+
+outdir="$1"
+if [ -z "$outdir" ]; then
+    outdir="_site"
+fi
+
+rm -rf "$outdir"
+mkdir -p "$outdir"
+
+buildScraper() 
+{
+    f=$1
+    dir=$(dirname "$f")
+
+    # get the scraper id from the filename
+    scraper_id=$(basename "$f" .yml)
+    versionFile=$f
+    if [ "$scraper_id" == "package" ]; then
+        scraper_id=$(basename "$dir")
+    fi
+
+    if [ "$dir" != "./scrapers" ]; then
+        versionFile="$dir"
+    fi
+
+    echo "Processing $scraper_id"
+
+    # create a directory for the version
+    version=$(git log -n 1 --pretty=format:%h -- "$versionFile")
+    updated=$(git log -n 1 --date="format:%F %T %z" --pretty=format:%ad -- "$versionFile")
+    
+    # create the zip file
+    # copy other files
+    zipfile=$(realpath "$outdir/$scraper_id.zip")
+
+    name=$(grep "^name:" "$f" | cut -d' ' -f2- | sed -e 's/\r//' -e 's/^"\(.*\)"$/\1/')
+    ignore=$(grep "^# ignore:" "$f" | cut -c 10- | sed -e 's/\r//')
+    dep=$(grep "^# requires:" "$f" | cut -c 12- | sed -e 's/\r//')
+
+    # always ignore package file
+    ignore="-x $ignore package"
+
+    pushd "$dir" > /dev/null
+    if [ "$dir" != "./scrapers" ]; then
+        zip -r "$zipfile" . ${ignore} > /dev/null
+    else
+        zip "$zipfile" "$scraper_id.yml" > /dev/null
+    fi
+    popd > /dev/null
+
+    # write to spec index
+    echo "- id: $scraper_id
+  name: $name
+  version: $version
+  date: $updated
+  path: $scraper_id.zip
+  sha256: $(sha256sum "$zipfile" | cut -d' ' -f1)" >> "$outdir"/index.yml
+
+    # handle dependencies
+    if [ ! -z "$dep" ]; then
+        echo "  requires:" >> "$outdir"/index.yml
+        for d in ${dep//,/ }; do
+            echo "    - $d" >> "$outdir"/index.yml
+        done
+    fi
+
+    echo "" >> "$outdir"/index.yml
+}
+
+# find all yml files in ./scrapers - these are packages individually
+for f in ./scrapers/*.yml; do 
+    buildScraper "$f"
+done
+
+find ./scrapers/ -mindepth 2 -name *.yml -print0 | while read -d $'\0' f; do
+    buildScraper "$f"
+done
+
+# handle dependency packages
+find ./scrapers/ -mindepth 2 -name package -print0 | while read -d $'\0' f; do
+    buildScraper "$f"
+done

scrapers/21Naturals/21Naturals.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "21Naturals"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - 21naturals.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21naturals
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21naturals
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21naturals
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21naturals
       - validName
 # Last Updated March 23, 2022

scrapers/21Sextreme/21Sextreme.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "21Sextreme"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - 21sextreme.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextreme
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextreme
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextreme
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextreme
       - validName
 # Last Updated March 23, 2022

scrapers/21Sextury/21Sextury.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "21Sextury"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - 21sextury.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - 21sextury.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
       - gallery
 # Last Updated December 22, 2022

scrapers/ATKGirlfriends/ATKGirlfriends.yml

@@ -1,4 +1,5 @@
 name: ATK Girlfriends
+# requires: py_common
 sceneByFragment:
   action: script
   script:

scrapers/ActiveDuty/ActiveDuty.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "Active Duty"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - activeduty.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - activeduty
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - activeduty
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - activeduty
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - activeduty
     - validName
 # Last Updated September 26, 2023

scrapers/AddictedToGirls/AddictedToGirls.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "addicted 2 Girls"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - addicted2girls.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - addicted2girls.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
       - gallery
 # Last Updated December 22, 2022

scrapers/Adultime/Adultime.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "Adultime"
 sceneByURL:
   - action: script
@@ -56,33 +57,33 @@ sceneByURL:
       - wolfwagner.com/en/video/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlsway
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlsway
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlsway
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlsway
       - validName
 galleryByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - puretaboo
       - gallery
 galleryByURL:
@@ -100,7 +101,7 @@ galleryByURL:
       - webyoung.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - puretaboo
       - gallery
 movieByURL:
@@ -112,7 +113,7 @@ movieByURL:
       - outofthefamily.com/en/dvd/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - puretaboo
       - movie
 # Last Updated October 12, 2023

scrapers/Algolia/Algolia.py

@@ -8,6 +8,13 @@ import sys
 from configparser import ConfigParser, NoSectionError
 from urllib.parse import urlparse
 
+# to import from a parent directory we need to add that directory to the system path
+csd = os.path.dirname(os.path.realpath(__file__))  # get current script directory
+parent = os.path.dirname(csd)  # parent directory (should be the scrapers one)
+sys.path.append(
+    parent
+)  # add parent dir to sys path so that we can import py_common from there
+
 try:
     from bs4 import BeautifulSoup as bs
     import requests

scrapers/Algolia/package

@@ -0,0 +1,3 @@
+# script used as a dependency only
+# requires: py_common
+name: Algolia

scrapers/AllGirlMassage/AllGirlMassage.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "AllGirlMassage"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - allgirlmassage.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - allgirlmassage
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - allgirlmassage
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - allgirlmassage
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - allgirlmassage
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - allgirlmassage.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - allgirlmassage
       - gallery
 # Last Updated December 22, 2022

scrapers/AnimeCharactersDatabase/AnimeCharactersDatabase.yml

@@ -1,4 +1,5 @@
 name: AnimeCharactersDatabase
+# requires: py_common
 
 performerByURL:
 - action: script

scrapers/AuntJudys/AuntJudys.yml

@@ -1,3 +1,4 @@
+# requires: py_common
 name: AuntJudys
 sceneByURL:
   - action: script

scrapers/Biphoria/Biphoria.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "Biphoria"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - biphoria.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - biphoria
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - biphoria
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - biphoria
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - biphoria
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - biphoria.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - biphoria
       - gallery
 # Last Updated February 07, 2023

scrapers/Blowpass/Blowpass.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: Blowpass
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - blowpass.com/en/video/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - blowpass
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - blowpass
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - blowpass
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - blowpass
     - validName
 # Last Updated October 04, 2023

scrapers/BurningAngel/BurningAngel.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "Burning Angel"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - burningangel.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - burningangel
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - burningangel
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - burningangel
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - burningangel
       - validName
 # Last Updated March 23, 2022

scrapers/ChaosMen/ChaosMen.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "ChaosMen"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - chaosmen.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - chaosmen
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - chaosmen
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - chaosmen
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - chaosmen
     - validName
 # Last Updated January 10, 2023

scrapers/ClubInfernoDungeon/ClubInfernoDungeon.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "ClubInfernoDungeon"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - clubinfernodungeon.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - clubinfernodungeon
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - clubinfernodungeon
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - clubinfernodungeon
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - clubinfernodungeon
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - clubinfernodungeon.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - clubinfernodungeon
       - gallery
 # Last Updated December 22, 2022

scrapers/ComicInfoXML/ComicInfoXML.yml

@@ -1,4 +1,5 @@
 name: "ComicInfo XML"
+# requires: py_common
 
 galleryByFragment:
   action: script

scrapers/CopyFromScene/CopyFromScene.yml

@@ -1,4 +1,6 @@
 name: "Copy from Scene"
+# requires: py_common
+
 galleryByFragment:
   action: script
   script:

scrapers/CopyToGallery/CopyToGallery.py

@@ -1,123 +1,123 @@
-import json
-import sys
-import os
-
-try:
-    import py_common.graphql as graphql
-    import py_common.log as log
-except ModuleNotFoundError:
-    print("You need to download the folder 'py_common' from the community repo! (CommunityScrapers/tree/master/scrapers/py_common)", file=sys.stderr)
-    sys.exit()
-
-find_gallery = False
-
-def call_graphql(query, variables=None):
-    return graphql.callGraphQL(query, variables)
-
-def get_gallery_id_by_path(gallery_path):
-    log.debug("get_gallery_by_path gallery_path " + str(gallery_path))
-    query = """
-            query FindGalleries($galleries_filter: GalleryFilterType) {
-              findGalleries(gallery_filter: $galleries_filter filter: {per_page: -1}) {
-                count
-                    galleries {
-                        id
-                    }
-              }
-            }
-            """
-    variables = {"galleries_filter": {"path": {'value': gallery_path, "modifier": "EQUALS"}}}
-    result = call_graphql(query, variables)
-    log.debug("get_gallery_by_path callGraphQL result " + str(result))
-    return result['findGalleries']['galleries'][0]['id']
-
-def update_gallery(input):
-    log.debug("gallery input " + str(input))
-    query = """
-                mutation GalleryUpdate($input : GalleryUpdateInput!) {
-                  galleryUpdate(input: $input) {
-                    id
-                    title
-                  }
-                }
-                """
-    variables = {
-        "input": input
-    }
-    result = call_graphql(query, variables)
-    if result:
-        g_id = result['galleryUpdate'].get('id')
-        g_title = result['galleryUpdate'].get('title')
-        log.info(f"updated Gallery ({g_id}): {g_title}")
-    return result
-
-def get_id(obj):
-    ids = []
-    for item in obj:
-        ids.append(item['id'])
-    return ids
-
-def find_galleries(scene_id, scene_path):
-    ids = []
-    directory_path = os.path.dirname(scene_path)
-    for (cur, dirs, files) in os.walk(directory_path):
-
-        for file in files:
-            if file.endswith('.zip'):
-                gallery_path = os.path.join(cur, file)
-                id = get_gallery_id_by_path(gallery_path)
-                updateScene_with_gallery(scene_id, id)
-                ids.append(id)
-        break
-    log.debug("find_galleries ids' found " + str(ids))
-    return ids
-
-def updateScene_with_gallery(scene_id, gallery_id):
-    data = {'id': scene_id, 'gallery_ids': [gallery_id]}
-    log.debug("data " + str(data))
-    query = """
-                mutation SceneUpdate($input : SceneUpdateInput!) {
-                  sceneUpdate(input: $input) {
-                    id
-                    title
-                  }
-                }
-                """
-    variables = {
-        "input": data
-    }
-    result = call_graphql(query, variables)
-    log.debug("graphql_updateGallery callGraphQL result " + str(result))
-
-FRAGMENT = json.loads(sys.stdin.read())
-SCENE_ID = FRAGMENT.get("id")
-
-scene = graphql.getScene(SCENE_ID)
-if scene:
-    scene_galleries = scene['galleries']
-    log.debug("scene_galleries " + str(scene_galleries))
-    gallery_ids = []
-    if len(scene_galleries) > 0:
-        for gallery_obj in scene_galleries:
-            gallery_ids.append(gallery_obj['id'])
-    elif find_gallery:
-        # if no galleries are associated see if any gallery zips exist in directory
-        gallery_ids = find_galleries(SCENE_ID, scene["path"])
-    log.debug("gallery_ids " + str(gallery_ids))
-
-    for gallery_id in gallery_ids:
-        studio = None
-        if scene['studio']:
-            studio = scene['studio']['id']
-        gallery_input = {'id': gallery_id,
-                         'urls': scene['urls'],
-                         'title': scene['title'],
-                         'date': scene["date"],
-                         'details': scene['details'],
-                         'studio_id': studio,
-                         'tag_ids': get_id(scene['tags']),
-                         'performer_ids': get_id(scene['performers'])}
-        update_gallery(gallery_input)
-
-    print(json.dumps({}))
-
+import json
+import sys
+import os
+
+try:
+    import py_common.graphql as graphql
+    import py_common.log as log
+except ModuleNotFoundError:
+    print("You need to download the folder 'py_common' from the community repo! (CommunityScrapers/tree/master/scrapers/py_common)", file=sys.stderr)
+    sys.exit()
+
+find_gallery = False
+
+def call_graphql(query, variables=None):
+    return graphql.callGraphQL(query, variables)
+
+def get_gallery_id_by_path(gallery_path):
+    log.debug("get_gallery_by_path gallery_path " + str(gallery_path))
+    query = """
+            query FindGalleries($galleries_filter: GalleryFilterType) {
+              findGalleries(gallery_filter: $galleries_filter filter: {per_page: -1}) {
+                count
+                    galleries {
+                        id
+                    }
+              }
+            }
+            """
+    variables = {"galleries_filter": {"path": {'value': gallery_path, "modifier": "EQUALS"}}}
+    result = call_graphql(query, variables)
+    log.debug("get_gallery_by_path callGraphQL result " + str(result))
+    return result['findGalleries']['galleries'][0]['id']
+
+def update_gallery(input):
+    log.debug("gallery input " + str(input))
+    query = """
+                mutation GalleryUpdate($input : GalleryUpdateInput!) {
+                  galleryUpdate(input: $input) {
+                    id
+                    title
+                  }
+                }
+                """
+    variables = {
+        "input": input
+    }
+    result = call_graphql(query, variables)
+    if result:
+        g_id = result['galleryUpdate'].get('id')
+        g_title = result['galleryUpdate'].get('title')
+        log.info(f"updated Gallery ({g_id}): {g_title}")
+    return result
+
+def get_id(obj):
+    ids = []
+    for item in obj:
+        ids.append(item['id'])
+    return ids
+
+def find_galleries(scene_id, scene_path):
+    ids = []
+    directory_path = os.path.dirname(scene_path)
+    for (cur, dirs, files) in os.walk(directory_path):
+
+        for file in files:
+            if file.endswith('.zip'):
+                gallery_path = os.path.join(cur, file)
+                id = get_gallery_id_by_path(gallery_path)
+                updateScene_with_gallery(scene_id, id)
+                ids.append(id)
+        break
+    log.debug("find_galleries ids' found " + str(ids))
+    return ids
+
+def updateScene_with_gallery(scene_id, gallery_id):
+    data = {'id': scene_id, 'gallery_ids': [gallery_id]}
+    log.debug("data " + str(data))
+    query = """
+                mutation SceneUpdate($input : SceneUpdateInput!) {
+                  sceneUpdate(input: $input) {
+                    id
+                    title
+                  }
+                }
+                """
+    variables = {
+        "input": data
+    }
+    result = call_graphql(query, variables)
+    log.debug("graphql_updateGallery callGraphQL result " + str(result))
+
+FRAGMENT = json.loads(sys.stdin.read())
+SCENE_ID = FRAGMENT.get("id")
+
+scene = graphql.getScene(SCENE_ID)
+if scene:
+    scene_galleries = scene['galleries']
+    log.debug("scene_galleries " + str(scene_galleries))
+    gallery_ids = []
+    if len(scene_galleries) > 0:
+        for gallery_obj in scene_galleries:
+            gallery_ids.append(gallery_obj['id'])
+    elif find_gallery:
+        # if no galleries are associated see if any gallery zips exist in directory
+        gallery_ids = find_galleries(SCENE_ID, scene["path"])
+    log.debug("gallery_ids " + str(gallery_ids))
+
+    for gallery_id in gallery_ids:
+        studio = None
+        if scene['studio']:
+            studio = scene['studio']['id']
+        gallery_input = {'id': gallery_id,
+                         'urls': scene['urls'],
+                         'title': scene['title'],
+                         'date': scene["date"],
+                         'details': scene['details'],
+                         'studio_id': studio,
+                         'tag_ids': get_id(scene['tags']),
+                         'performer_ids': get_id(scene['performers'])}
+        update_gallery(gallery_input)
+
+    print(json.dumps({}))
+

scrapers/CopyToGallery/CopyToGallery.yml

@@ -1,4 +1,6 @@
 name: "Copy to Gallery"
+# requires: py_common
+
 sceneByFragment:
   action: script
   script:

scrapers/DogfartNetwork/DogfartNetwork.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "DogfartNetwork"
 sceneByURL:
   - action: script
@@ -19,26 +20,26 @@ sceneByURL:
       - zebragirls.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - dogfartnetwork
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - dogfartnetwork
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - dogfartnetwork
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - dogfartnetwork
       - validName
 galleryByURL:
@@ -61,7 +62,7 @@ galleryByURL:
       - zebragirls.com/en/photo
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - dogfartnetwork
       - gallery
 # Last Updated July 27, 2023

scrapers/EvilAngel/EvilAngel.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "EvilAngel"
 sceneByURL:
   - action: script
@@ -9,7 +10,7 @@ sceneByURL:
       - tsfactor.com/en/video/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - evilangel
 movieByURL:
   - action: script
@@ -21,27 +22,27 @@ movieByURL:
       - tsfactor.com/en/movie/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - evilangel
       - movie
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - evilangel
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - evilangel
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - evilangel
     - validName
 galleryByURL:
@@ -54,7 +55,7 @@ galleryByURL:
       - tsfactor.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - evilangel
       - gallery
 # Last Updated December 26, 2022

scrapers/FalconStudios/FalconStudios.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: Falcon Studios
 sceneByURL:
   - action: script
@@ -6,26 +7,26 @@ sceneByURL:
       - hothouse.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - falconstudios
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - falconstudios
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - falconstudios
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - falconstudios
     - validName
 # Last Updated September 09, 2023

scrapers/FantasyMassage/FantasyMassage.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "FantasyMassage"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - fantasymassage.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - fantasymassage
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - fantasymassage
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - fantasymassage
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - fantasymassage
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - fantasymassage.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - fantasymassage
       - gallery
 # Last Updated December 22, 2022

scrapers/Filename/Filename.yml

@@ -1,4 +1,6 @@
 name: Filename
+# requires: py_common
+
 sceneByFragment:
   action: script
   script:

scrapers/FistingInferno/FistingInferno.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "FistingInferno"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - fistinginferno.com/en/video/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - fistinginferno
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - fistinginferno
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - fistinginferno
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - fistinginferno
     - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - fistinginferno.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - fistinginferno
       - gallery
 # Last Updated September 24, 2023

scrapers/Fit18/Fit18.yml

@@ -1,4 +1,6 @@
 name: Fit18
+# requires: py_common
+
 sceneByURL:
   - url:
       - fit18.com/videos/

scrapers/FratX/FratX.yml

@@ -1,4 +1,6 @@
 name: FratX
+# requires: py_common
+
 sceneByURL:
   - action: script
     script:

scrapers/GangBangCreampie/GangBangCreampie.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "GangbangCreampie"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - gangbangcreampie.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - gangbangcreampie
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - gangbangcreampie
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - gangbangcreampie
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - gangbangcreampie
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - gangbangcreampie.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - gangbangcreampie
       - gallery
 # Last Updated December 22, 2022

scrapers/GenderXFilms/GenderXFilms.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: GenderX Films
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - genderxfilms.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - genderxfilms
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - genderxfilms
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - genderxfilms
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - genderxfilms
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - genderxfilms.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - genderxfilms
       - gallery
 movieByURL:
@@ -42,7 +43,7 @@ movieByURL:
       - genderxfilms.com/en/movie/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - genderxfilms
       - movie
 # Last Updated December 22, 2022

scrapers/Girlfriendsfilms/Girlfriendsfilms.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "Girlfriends Films"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - girlfriendsfilms.com/en/video/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlfriendsfilms
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlfriendsfilms
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlfriendsfilms
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlfriendsfilms
       - validName
 movieByURL:
@@ -33,7 +34,7 @@ movieByURL:
       - girlfriendsfilms.com/en/movie/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlfriendsfilms
       - movie
 galleryByURL:
@@ -42,7 +43,7 @@ galleryByURL:
       - girlfriendsfilms.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlfriendsfilms
       - gallery
 # Last Updated December 22, 2022

scrapers/Girlsway/Girlsway.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "GirlsWay"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - girlsway.com/en/video/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlsway
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlsway
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlsway
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlsway
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - girlsway.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - girlsway
       - gallery
 # Last Updated December 22, 2022

scrapers/Gloryholesecrets/Gloryholesecrets.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: GloryholeSecrets
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - gloryholesecrets.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - gloryholesecrets
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - gloryholesecrets
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - gloryholesecrets
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - gloryholesecrets
     - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - gloryholesecrets.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - gloryholesecrets
       - gallery
 # Last Updated August 22, 2023

scrapers/IAFD/IAFD.yml

@@ -1,4 +1,6 @@
 name: IAFD
+# requires: py_common
+
 performerByName:
   action: script
   script:

scrapers/JacquieEtMichelTV/JacquieEtMichelTV.yml

@@ -1,4 +1,6 @@
 name: JacquieEtMicaelTV
+# requires: py_common
+
 sceneByFragment:
     action: script
     script:

scrapers/JavLibrary_python/JavLibrary_python.yml

@@ -1,4 +1,6 @@
 name: "JavLibrary Python"
+# requires: py_common
+
 sceneByFragment:
   action: script
   script:

scrapers/Johnnyrapid/Johnnyrapid.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "Johnny Rapid"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - johnnyrapid.com/en/video/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - johnnyrapid
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - johnnyrapid
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - johnnyrapid
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - johnnyrapid
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - johnnyrapid.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - johnnyrapid
       - gallery
 # Last Updated December 22, 2022

scrapers/KBProductions/KBProductions.yml

@@ -1,4 +1,6 @@
 name: "KB Productions"
+# requires: py_common
+
 sceneByURL:
   - url:
       # Keeping this to allow for updates using old urls

scrapers/Loyalfans/Loyalfans.yml

@@ -1,4 +1,6 @@
 name: Loyalfans
+# requires: py_common
+
 sceneByURL:
   - url:
       - loyalfans.com

scrapers/ManyVids/ManyVids.yml

@@ -1,4 +1,6 @@
 name: ManyVids
+# requires: py_common
+
 sceneByURL:
   - url:
       - manyvids.com/Video

scrapers/MenOver30/MenOver30.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "MenOver30"
 sceneByURL:
   - action: script
@@ -6,26 +7,26 @@ sceneByURL:
       - pridestudios.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - menover30
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - menover30
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - menover30
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - menover30
     - validName
 galleryByURL:
@@ -35,7 +36,7 @@ galleryByURL:
       - pridestudios.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - menover30
       - gallery
 # Last Updated December 22, 2022

scrapers/MindGeekAPI/MindGeekAPI.yml

@@ -1,4 +1,6 @@
 name: "MindGeekAPI"
+# requires: py_common
+
 sceneByFragment:
     action: script
     script:

scrapers/MissaX/MissaX.yml

@@ -1,4 +1,6 @@
 name: "MissaX"
+# requires: py_common
+
 sceneByURL:
   - action: script
     script:

scrapers/ModelCentroAPI/ModelCentroAPI.yml

@@ -1,4 +1,6 @@
 name: "ModelCentroAPI"
+# requires: py_common
+
 sceneByURL:
   - url:
       - beingphoenixmarie.com/scene/

scrapers/NextDoorStudios/NextDoorStudios.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: Next Door Studios
 sceneByURL:
   - action: script
@@ -23,26 +24,26 @@ sceneByURL:
       - trystanbull.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - nextdoorstudios
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - nextdoorstudios
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - nextdoorstudios
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - nextdoorstudios
     - validName
 # Last Updated August 18, 2023

scrapers/RagingStallion/RagingStallion.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: Raging Stallion
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - ragingstallion.com/en/scene
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - ragingstallion
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - ragingstallion
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - ragingstallion
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - ragingstallion
     - validName
 # Last Updated September 09, 2023

scrapers/RealityLovers/RealityLovers.yml

@@ -1,4 +1,6 @@
 name: RealityLovers
+# requires: py_common
+
 performerByURL:
   - action: script
     url:

scrapers/Redgifs/Redgifs.yml

@@ -1,4 +1,6 @@
 name: Redgifs
+# requires: py_common
+
 sceneByURL:
   - action: script
     script:

scrapers/RoccoSiffredi/RoccoSiffredi.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "RoccoSiffredi"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - roccosiffredi.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - roccosiffredi
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - roccosiffredi
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - roccosiffredi
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - roccosiffredi
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - roccosiffredi.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - roccosiffredi
       - gallery
 # Last Updated December 22, 2022

scrapers/SARJ-LLC/SARJ-LLC.yml

@@ -1,4 +1,5 @@
 name: SARJ LLC
+# requires: py_common
 
 sceneByName:
   action: script

scrapers/ScrapeWithURL/ScrapeWithURL.py

@@ -1,81 +1,81 @@
-import json
-import sys
-
-try:
-    import py_common.graphql as graphql
-    import py_common.log as log
-except ModuleNotFoundError:
-    print(
-        "You need to download the folder 'py_common' from the community repo! (CommunityScrapers/tree/master/scrapers/py_common)",
-        file=sys.stderr,
-    )
-    sys.exit()
-
-
-def scrape_scene(url):
-    query = """
-query scrapeSceneURL($url: String!) {
-    scrapeSceneURL(url: $url) {
-        title
-        details
-        code
-        date
-        image
-        urls
-        studio {
-            name
-            url
-            image
-            parent {
-                name
-                url
-                image
-            }
-        }
-        tags {
-            name
-        }
-        performers {
-            aliases
-            birthdate
-            career_length
-            country
-            death_date
-            details
-            ethnicity
-            eye_color
-            fake_tits
-            gender
-            hair_color
-            height
-            instagram
-            images
-            measurements
-            name
-            piercings
-            tags {
-                name
-            }
-            tattoos
-            twitter
-            url
-            weight
-        }
-    }
-}"""
-
-    variables = {"url": url}
-    result = graphql.callGraphQL(query, variables)
-    log.debug(f"result {result}")
-    if result:
-        return result["scrapeSceneURL"]
-
-
-FRAGMENT = json.loads(sys.stdin.read())
-url = FRAGMENT.get("url")
-
-if url:
-    result = scrape_scene(url)
-    print(json.dumps(result))
-else:
-    print("null")
+import json
+import sys
+
+try:
+    import py_common.graphql as graphql
+    import py_common.log as log
+except ModuleNotFoundError:
+    print(
+        "You need to download the folder 'py_common' from the community repo! (CommunityScrapers/tree/master/scrapers/py_common)",
+        file=sys.stderr,
+    )
+    sys.exit()
+
+
+def scrape_scene(url):
+    query = """
+query scrapeSceneURL($url: String!) {
+    scrapeSceneURL(url: $url) {
+        title
+        details
+        code
+        date
+        image
+        urls
+        studio {
+            name
+            url
+            image
+            parent {
+                name
+                url
+                image
+            }
+        }
+        tags {
+            name
+        }
+        performers {
+            aliases
+            birthdate
+            career_length
+            country
+            death_date
+            details
+            ethnicity
+            eye_color
+            fake_tits
+            gender
+            hair_color
+            height
+            instagram
+            images
+            measurements
+            name
+            piercings
+            tags {
+                name
+            }
+            tattoos
+            twitter
+            url
+            weight
+        }
+    }
+}"""
+
+    variables = {"url": url}
+    result = graphql.callGraphQL(query, variables)
+    log.debug(f"result {result}")
+    if result:
+        return result["scrapeSceneURL"]
+
+
+FRAGMENT = json.loads(sys.stdin.read())
+url = FRAGMENT.get("url")
+
+if url:
+    result = scrape_scene(url)
+    print(json.dumps(result))
+else:
+    print("null")

scrapers/ScrapeWithURL/ScrapeWithURL.yml

@@ -1,8 +1,10 @@
-name: Scrape with URL
-sceneByFragment:
-    action: script
-    script:
-      - python
-      - ScrapeWithURL.py
-      
-# Last Updated April 16, 2022
+name: Scrape with URL
+# requires: py_common
+
+sceneByFragment:
+    action: script
+    script:
+      - python
+      - ScrapeWithURL.py
+      
+# Last Updated April 16, 2022

scrapers/ShokoAPI/ShokoAPI.yml

@@ -1,4 +1,6 @@
 name: "ShokoAPI"
+# requires: py_common
+
 sceneByFragment:
     action: script
     script:

scrapers/TabooHeat/TabooHeat.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "TabooHeat"
 sceneByURL:
   - action: script
@@ -5,26 +6,26 @@ sceneByURL:
       - tabooheat.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - tabooheat
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - tabooheat
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - tabooheat
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - tabooheat
     - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - tabooheat.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - tabooheat
       - gallery
 # Last Updated December 22, 2022

scrapers/Teamskeet/Teamskeet.yml

@@ -1,4 +1,6 @@
 name: "TeamskeetAPI"
+# requires: py_common
+
 sceneByURL:
   - url:
       - teamskeet.com/movies/

scrapers/Tokyohot/Tokyohot.yml

@@ -1,4 +1,6 @@
 name: tokyohot
+# requires: py_common
+
 sceneByURL:
   - action: scrapeXPath
     url:

scrapers/TopWebModels/TopWebModels.yml

@@ -1,4 +1,6 @@
 name: "Top Web Models"
+# requires: py_common
+
 sceneByURL:
   - url:
       - tour.2girls1camera.com/scenes

scrapers/Traxxx/Traxxx.yml

@@ -1,4 +1,6 @@
 name: "Traxxx"
+# requires: py_common
+
 sceneByURL:
   - url:
      - traxxx.me/scene/

scrapers/Wicked/Wicked.yml

@@ -1,3 +1,4 @@
+# requires: Algolia
 name: "Wicked"
 sceneByURL:
   - action: script
@@ -5,7 +6,7 @@ sceneByURL:
       - wicked.com/en/video/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - wicked
 movieByURL:
   - action: script
@@ -13,27 +14,27 @@ movieByURL:
       - wicked.com/en/movie/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - wicked
       - movie
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - wicked
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - wicked
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - wicked
       - validName
 galleryByURL:
@@ -42,7 +43,7 @@ galleryByURL:
       - wicked.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - wicked
       - gallery
 # Last Updated December 22, 2022

scrapers/WowNetworkVenus/WowNetworkVenus.yml

@@ -1,4 +1,6 @@
 name: WowNetworkVenus
+# requires: py_common
+
 sceneByURL:
   - action: script
     script:

scrapers/dc-onlyfans/dc-onlyfans.yml

@@ -1,4 +1,6 @@
 name: "DC Onlyfans"
+# requires: py_common
+
 sceneByFragment:
     action: script
     script: