diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..15e6e311 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,51 @@ +name: Deploy repository to Github Pages + +on: + push: + branches: [ master, stable ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: Checkout main + uses: actions/checkout@v2 + with: + path: master + ref: master + fetch-depth: '0' + - run: | + cd master + ./build_site.sh ../_site/develop + # uncomment this once we have a stable branch + # - name: Checkout Stable + # uses: actions/checkout@v2 + # with: + # path: stable + # ref: stable + # fetch-depth: '0' + # - run: | + # cd stable + # ../master/build_site.sh ../_site/stable + - uses: actions/upload-pages-artifact@v2 + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-22.04 + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 + diff --git a/.gitignore b/.gitignore index 9377605f..393112e6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ yarn-error.log # Scraper-generated files /scrapers/*.ini **/__pycache__/ + +/_site \ No newline at end of file diff --git a/build_site.sh b/build_site.sh new file mode 100755 index 00000000..4b3b9a3e --- /dev/null +++ b/build_site.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# builds a repository of scrapers +# outputs to _site with the following structure: +# index.yml +# .zip +# Each zip file contains the scraper.yml file and any other files in the same directory + +outdir="$1" +if [ -z "$outdir" ]; then + outdir="_site" +fi + +rm -rf "$outdir" +mkdir -p "$outdir" + +buildScraper() +{ + f=$1 + dir=$(dirname "$f") + + # get the scraper id from the filename + scraper_id=$(basename "$f" .yml) + versionFile=$f + if [ "$scraper_id" == "package" ]; then + scraper_id=$(basename "$dir") + fi + + if [ "$dir" != "./scrapers" ]; then + versionFile="$dir" + fi + + echo "Processing $scraper_id" + + # create a directory for the version + version=$(git log -n 1 --pretty=format:%h -- "$versionFile") + updated=$(git log -n 1 --date="format:%F %T %z" --pretty=format:%ad -- "$versionFile") + + # create the zip file + # copy other files + zipfile=$(realpath "$outdir/$scraper_id.zip") + + name=$(grep "^name:" "$f" | cut -d' ' -f2- | sed -e 's/\r//' -e 's/^"\(.*\)"$/\1/') + ignore=$(grep "^# ignore:" "$f" | cut -c 10- | sed -e 's/\r//') + dep=$(grep "^# requires:" "$f" | cut -c 12- | sed -e 's/\r//') + + # always ignore package file + ignore="-x $ignore package" + + pushd "$dir" > /dev/null + if [ "$dir" != "./scrapers" ]; then + zip -r "$zipfile" . ${ignore} > /dev/null + else + zip "$zipfile" "$scraper_id.yml" > /dev/null + fi + popd > /dev/null + + # write to spec index + echo "- id: $scraper_id + name: $name + version: $version + date: $updated + path: $scraper_id.zip + sha256: $(sha256sum "$zipfile" | cut -d' ' -f1)" >> "$outdir"/index.yml + + # handle dependencies + if [ ! -z "$dep" ]; then + echo " requires:" >> "$outdir"/index.yml + for d in ${dep//,/ }; do + echo " - $d" >> "$outdir"/index.yml + done + fi + + echo "" >> "$outdir"/index.yml +} + +# find all yml files in ./scrapers - these are packages individually +for f in ./scrapers/*.yml; do + buildScraper "$f" +done + +find ./scrapers/ -mindepth 2 -name *.yml -print0 | while read -d $'\0' f; do + buildScraper "$f" +done + +# handle dependency packages +find ./scrapers/ -mindepth 2 -name package -print0 | while read -d $'\0' f; do + buildScraper "$f" +done diff --git a/scrapers/Algolia_21Naturals.yml b/scrapers/21Naturals/21Naturals.yml similarity index 76% rename from scrapers/Algolia_21Naturals.yml rename to scrapers/21Naturals/21Naturals.yml index 131b7bf6..8f0dceda 100644 --- a/scrapers/Algolia_21Naturals.yml +++ b/scrapers/21Naturals/21Naturals.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "21Naturals" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - 21naturals.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21naturals sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21naturals sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21naturals - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21naturals - validName # Last Updated March 23, 2022 diff --git a/scrapers/Algolia_21Sextreme.yml b/scrapers/21Sextreme/21Sextreme.yml similarity index 76% rename from scrapers/Algolia_21Sextreme.yml rename to scrapers/21Sextreme/21Sextreme.yml index 1dfaa360..34fda954 100644 --- a/scrapers/Algolia_21Sextreme.yml +++ b/scrapers/21Sextreme/21Sextreme.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "21Sextreme" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - 21sextreme.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21sextreme sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21sextreme sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21sextreme - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21sextreme - validName # Last Updated March 23, 2022 diff --git a/scrapers/Algolia_21Sextury.yml b/scrapers/21Sextury/21Sextury.yml similarity index 77% rename from scrapers/Algolia_21Sextury.yml rename to scrapers/21Sextury/21Sextury.yml index 62d1df2e..afc0b531 100644 --- a/scrapers/Algolia_21Sextury.yml +++ b/scrapers/21Sextury/21Sextury.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "21Sextury" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - 21sextury.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21sextury sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21sextury sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21sextury - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21sextury - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - 21sextury.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - 21sextury - gallery # Last Updated December 22, 2022 diff --git a/scrapers/ATKGirlfriends.py b/scrapers/ATKGirlfriends/ATKGirlfriends.py similarity index 100% rename from scrapers/ATKGirlfriends.py rename to scrapers/ATKGirlfriends/ATKGirlfriends.py diff --git a/scrapers/ATKGirlfriends.yml b/scrapers/ATKGirlfriends/ATKGirlfriends.yml similarity index 99% rename from scrapers/ATKGirlfriends.yml rename to scrapers/ATKGirlfriends/ATKGirlfriends.yml index a0c8d8b3..782e4fef 100644 --- a/scrapers/ATKGirlfriends.yml +++ b/scrapers/ATKGirlfriends/ATKGirlfriends.yml @@ -1,4 +1,5 @@ name: ATK Girlfriends +# requires: py_common sceneByFragment: action: script script: diff --git a/scrapers/Algolia_ActiveDuty.yml b/scrapers/ActiveDuty/ActiveDuty.yml similarity index 76% rename from scrapers/Algolia_ActiveDuty.yml rename to scrapers/ActiveDuty/ActiveDuty.yml index 68833fcf..895fea7a 100644 --- a/scrapers/Algolia_ActiveDuty.yml +++ b/scrapers/ActiveDuty/ActiveDuty.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "Active Duty" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - activeduty.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - activeduty sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - activeduty sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - activeduty - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - activeduty - validName # Last Updated September 26, 2023 diff --git a/scrapers/Algolia_AddictedToGirls.yml b/scrapers/AddictedToGirls/AddictedToGirls.yml similarity index 78% rename from scrapers/Algolia_AddictedToGirls.yml rename to scrapers/AddictedToGirls/AddictedToGirls.yml index 296ac220..8eb060de 100644 --- a/scrapers/Algolia_AddictedToGirls.yml +++ b/scrapers/AddictedToGirls/AddictedToGirls.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "addicted 2 Girls" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - addicted2girls.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - addicted2girls sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - addicted2girls sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - addicted2girls - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - addicted2girls - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - addicted2girls.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - addicted2girls - gallery # Last Updated December 22, 2022 diff --git a/scrapers/Algolia_Adultime.yml b/scrapers/Adultime/Adultime.yml similarity index 92% rename from scrapers/Algolia_Adultime.yml rename to scrapers/Adultime/Adultime.yml index 028706f3..4a01a9fc 100644 --- a/scrapers/Algolia_Adultime.yml +++ b/scrapers/Adultime/Adultime.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "Adultime" sceneByURL: - action: script @@ -56,33 +57,33 @@ sceneByURL: - wolfwagner.com/en/video/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlsway sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlsway sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlsway - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlsway - validName galleryByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - puretaboo - gallery galleryByURL: @@ -100,7 +101,7 @@ galleryByURL: - webyoung.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - puretaboo - gallery movieByURL: @@ -112,7 +113,7 @@ movieByURL: - outofthefamily.com/en/dvd/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - puretaboo - movie # Last Updated October 12, 2023 diff --git a/scrapers/Algolia.py b/scrapers/Algolia/Algolia.py similarity index 99% rename from scrapers/Algolia.py rename to scrapers/Algolia/Algolia.py index e0fd6da6..3295a118 100644 --- a/scrapers/Algolia.py +++ b/scrapers/Algolia/Algolia.py @@ -8,6 +8,13 @@ import sys from configparser import ConfigParser, NoSectionError from urllib.parse import urlparse +# to import from a parent directory we need to add that directory to the system path +csd = os.path.dirname(os.path.realpath(__file__)) # get current script directory +parent = os.path.dirname(csd) # parent directory (should be the scrapers one) +sys.path.append( + parent +) # add parent dir to sys path so that we can import py_common from there + try: from bs4 import BeautifulSoup as bs import requests diff --git a/scrapers/Algolia/package b/scrapers/Algolia/package new file mode 100644 index 00000000..acffc82b --- /dev/null +++ b/scrapers/Algolia/package @@ -0,0 +1,3 @@ +# script used as a dependency only +# requires: py_common +name: Algolia diff --git a/scrapers/Algolia_AllGirlMassage.yml b/scrapers/AllGirlMassage/AllGirlMassage.yml similarity index 78% rename from scrapers/Algolia_AllGirlMassage.yml rename to scrapers/AllGirlMassage/AllGirlMassage.yml index e648981e..45c23adb 100644 --- a/scrapers/Algolia_AllGirlMassage.yml +++ b/scrapers/AllGirlMassage/AllGirlMassage.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "AllGirlMassage" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - allgirlmassage.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - allgirlmassage sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - allgirlmassage sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - allgirlmassage - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - allgirlmassage - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - allgirlmassage.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - allgirlmassage - gallery # Last Updated December 22, 2022 diff --git a/scrapers/AnalVids.py b/scrapers/AnalVids/AnalVids.py similarity index 100% rename from scrapers/AnalVids.py rename to scrapers/AnalVids/AnalVids.py diff --git a/scrapers/AnalVids.yml b/scrapers/AnalVids/AnalVids.yml similarity index 100% rename from scrapers/AnalVids.yml rename to scrapers/AnalVids/AnalVids.yml diff --git a/scrapers/AnimeCharactersDatabase.py b/scrapers/AnimeCharactersDatabase/AnimeCharactersDatabase.py similarity index 100% rename from scrapers/AnimeCharactersDatabase.py rename to scrapers/AnimeCharactersDatabase/AnimeCharactersDatabase.py diff --git a/scrapers/AnimeCharactersDatabase.yml b/scrapers/AnimeCharactersDatabase/AnimeCharactersDatabase.yml similarity index 93% rename from scrapers/AnimeCharactersDatabase.yml rename to scrapers/AnimeCharactersDatabase/AnimeCharactersDatabase.yml index f83e3525..22d778cf 100644 --- a/scrapers/AnimeCharactersDatabase.yml +++ b/scrapers/AnimeCharactersDatabase/AnimeCharactersDatabase.yml @@ -1,4 +1,5 @@ name: AnimeCharactersDatabase +# requires: py_common performerByURL: - action: script diff --git a/scrapers/Arx.py b/scrapers/Arx/Arx.py similarity index 100% rename from scrapers/Arx.py rename to scrapers/Arx/Arx.py diff --git a/scrapers/Arx.yml b/scrapers/Arx/Arx.yml similarity index 100% rename from scrapers/Arx.yml rename to scrapers/Arx/Arx.yml diff --git a/scrapers/AuntJudys.py b/scrapers/AuntJudys/AuntJudys.py similarity index 100% rename from scrapers/AuntJudys.py rename to scrapers/AuntJudys/AuntJudys.py diff --git a/scrapers/AuntJudys.yml b/scrapers/AuntJudys/AuntJudys.yml similarity index 90% rename from scrapers/AuntJudys.yml rename to scrapers/AuntJudys/AuntJudys.yml index fc38558d..acf463c1 100644 --- a/scrapers/AuntJudys.yml +++ b/scrapers/AuntJudys/AuntJudys.yml @@ -1,3 +1,4 @@ +# requires: py_common name: AuntJudys sceneByURL: - action: script diff --git a/scrapers/Algolia_Biphoria.yml b/scrapers/Biphoria/Biphoria.yml similarity index 77% rename from scrapers/Algolia_Biphoria.yml rename to scrapers/Biphoria/Biphoria.yml index 94d6441c..8a8c7230 100644 --- a/scrapers/Algolia_Biphoria.yml +++ b/scrapers/Biphoria/Biphoria.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "Biphoria" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - biphoria.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - biphoria sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - biphoria sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - biphoria - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - biphoria - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - biphoria.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - biphoria - gallery # Last Updated February 07, 2023 diff --git a/scrapers/Algolia_Blowpass.yml b/scrapers/Blowpass/Blowpass.yml similarity index 75% rename from scrapers/Algolia_Blowpass.yml rename to scrapers/Blowpass/Blowpass.yml index 76ada89b..077459c2 100644 --- a/scrapers/Algolia_Blowpass.yml +++ b/scrapers/Blowpass/Blowpass.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: Blowpass sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - blowpass.com/en/video/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - blowpass sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - blowpass sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - blowpass - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - blowpass - validName # Last Updated October 04, 2023 diff --git a/scrapers/Algolia_BurningAngel.yml b/scrapers/BurningAngel/BurningAngel.yml similarity index 76% rename from scrapers/Algolia_BurningAngel.yml rename to scrapers/BurningAngel/BurningAngel.yml index 2b0f93cf..b3e861a0 100644 --- a/scrapers/Algolia_BurningAngel.yml +++ b/scrapers/BurningAngel/BurningAngel.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "Burning Angel" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - burningangel.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - burningangel sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - burningangel sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - burningangel - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - burningangel - validName # Last Updated March 23, 2022 diff --git a/scrapers/Algolia_ChaosMen.yml b/scrapers/ChaosMen/ChaosMen.yml similarity index 75% rename from scrapers/Algolia_ChaosMen.yml rename to scrapers/ChaosMen/ChaosMen.yml index 15652114..ee8f9b1d 100644 --- a/scrapers/Algolia_ChaosMen.yml +++ b/scrapers/ChaosMen/ChaosMen.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "ChaosMen" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - chaosmen.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - chaosmen sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - chaosmen sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - chaosmen - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - chaosmen - validName # Last Updated January 10, 2023 diff --git a/scrapers/Algolia_ClubInfernoDungeon.yml b/scrapers/ClubInfernoDungeon/ClubInfernoDungeon.yml similarity index 79% rename from scrapers/Algolia_ClubInfernoDungeon.yml rename to scrapers/ClubInfernoDungeon/ClubInfernoDungeon.yml index 60d1ed6d..d02a370a 100644 --- a/scrapers/Algolia_ClubInfernoDungeon.yml +++ b/scrapers/ClubInfernoDungeon/ClubInfernoDungeon.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "ClubInfernoDungeon" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - clubinfernodungeon.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - clubinfernodungeon sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - clubinfernodungeon sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - clubinfernodungeon - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - clubinfernodungeon - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - clubinfernodungeon.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - clubinfernodungeon - gallery # Last Updated December 22, 2022 diff --git a/scrapers/ComicInfoXML.py b/scrapers/ComicInfoXML/ComicInfoXML.py similarity index 100% rename from scrapers/ComicInfoXML.py rename to scrapers/ComicInfoXML/ComicInfoXML.py diff --git a/scrapers/ComicInfoXML.yml b/scrapers/ComicInfoXML/ComicInfoXML.yml similarity index 70% rename from scrapers/ComicInfoXML.yml rename to scrapers/ComicInfoXML/ComicInfoXML.yml index 561d22bc..24004427 100644 --- a/scrapers/ComicInfoXML.yml +++ b/scrapers/ComicInfoXML/ComicInfoXML.yml @@ -1,4 +1,5 @@ name: "ComicInfo XML" +# requires: py_common galleryByFragment: action: script diff --git a/scrapers/CopyFromScene.py b/scrapers/CopyFromScene/CopyFromScene.py similarity index 100% rename from scrapers/CopyFromScene.py rename to scrapers/CopyFromScene/CopyFromScene.py diff --git a/scrapers/CopyFromScene.yml b/scrapers/CopyFromScene/CopyFromScene.yml similarity index 87% rename from scrapers/CopyFromScene.yml rename to scrapers/CopyFromScene/CopyFromScene.yml index 7801dd1b..266918b9 100644 --- a/scrapers/CopyFromScene.yml +++ b/scrapers/CopyFromScene/CopyFromScene.yml @@ -1,4 +1,6 @@ name: "Copy from Scene" +# requires: py_common + galleryByFragment: action: script script: diff --git a/scrapers/CopyToGallery.py b/scrapers/CopyToGallery/CopyToGallery.py similarity index 97% rename from scrapers/CopyToGallery.py rename to scrapers/CopyToGallery/CopyToGallery.py index b3d16af4..90bd4109 100644 --- a/scrapers/CopyToGallery.py +++ b/scrapers/CopyToGallery/CopyToGallery.py @@ -1,123 +1,123 @@ -import json -import sys -import os - -try: - import py_common.graphql as graphql - import py_common.log as log -except ModuleNotFoundError: - print("You need to download the folder 'py_common' from the community repo! (CommunityScrapers/tree/master/scrapers/py_common)", file=sys.stderr) - sys.exit() - -find_gallery = False - -def call_graphql(query, variables=None): - return graphql.callGraphQL(query, variables) - -def get_gallery_id_by_path(gallery_path): - log.debug("get_gallery_by_path gallery_path " + str(gallery_path)) - query = """ - query FindGalleries($galleries_filter: GalleryFilterType) { - findGalleries(gallery_filter: $galleries_filter filter: {per_page: -1}) { - count - galleries { - id - } - } - } - """ - variables = {"galleries_filter": {"path": {'value': gallery_path, "modifier": "EQUALS"}}} - result = call_graphql(query, variables) - log.debug("get_gallery_by_path callGraphQL result " + str(result)) - return result['findGalleries']['galleries'][0]['id'] - -def update_gallery(input): - log.debug("gallery input " + str(input)) - query = """ - mutation GalleryUpdate($input : GalleryUpdateInput!) { - galleryUpdate(input: $input) { - id - title - } - } - """ - variables = { - "input": input - } - result = call_graphql(query, variables) - if result: - g_id = result['galleryUpdate'].get('id') - g_title = result['galleryUpdate'].get('title') - log.info(f"updated Gallery ({g_id}): {g_title}") - return result - -def get_id(obj): - ids = [] - for item in obj: - ids.append(item['id']) - return ids - -def find_galleries(scene_id, scene_path): - ids = [] - directory_path = os.path.dirname(scene_path) - for (cur, dirs, files) in os.walk(directory_path): - - for file in files: - if file.endswith('.zip'): - gallery_path = os.path.join(cur, file) - id = get_gallery_id_by_path(gallery_path) - updateScene_with_gallery(scene_id, id) - ids.append(id) - break - log.debug("find_galleries ids' found " + str(ids)) - return ids - -def updateScene_with_gallery(scene_id, gallery_id): - data = {'id': scene_id, 'gallery_ids': [gallery_id]} - log.debug("data " + str(data)) - query = """ - mutation SceneUpdate($input : SceneUpdateInput!) { - sceneUpdate(input: $input) { - id - title - } - } - """ - variables = { - "input": data - } - result = call_graphql(query, variables) - log.debug("graphql_updateGallery callGraphQL result " + str(result)) - -FRAGMENT = json.loads(sys.stdin.read()) -SCENE_ID = FRAGMENT.get("id") - -scene = graphql.getScene(SCENE_ID) -if scene: - scene_galleries = scene['galleries'] - log.debug("scene_galleries " + str(scene_galleries)) - gallery_ids = [] - if len(scene_galleries) > 0: - for gallery_obj in scene_galleries: - gallery_ids.append(gallery_obj['id']) - elif find_gallery: - # if no galleries are associated see if any gallery zips exist in directory - gallery_ids = find_galleries(SCENE_ID, scene["path"]) - log.debug("gallery_ids " + str(gallery_ids)) - - for gallery_id in gallery_ids: - studio = None - if scene['studio']: - studio = scene['studio']['id'] - gallery_input = {'id': gallery_id, - 'urls': scene['urls'], - 'title': scene['title'], - 'date': scene["date"], - 'details': scene['details'], - 'studio_id': studio, - 'tag_ids': get_id(scene['tags']), - 'performer_ids': get_id(scene['performers'])} - update_gallery(gallery_input) - - print(json.dumps({})) - +import json +import sys +import os + +try: + import py_common.graphql as graphql + import py_common.log as log +except ModuleNotFoundError: + print("You need to download the folder 'py_common' from the community repo! (CommunityScrapers/tree/master/scrapers/py_common)", file=sys.stderr) + sys.exit() + +find_gallery = False + +def call_graphql(query, variables=None): + return graphql.callGraphQL(query, variables) + +def get_gallery_id_by_path(gallery_path): + log.debug("get_gallery_by_path gallery_path " + str(gallery_path)) + query = """ + query FindGalleries($galleries_filter: GalleryFilterType) { + findGalleries(gallery_filter: $galleries_filter filter: {per_page: -1}) { + count + galleries { + id + } + } + } + """ + variables = {"galleries_filter": {"path": {'value': gallery_path, "modifier": "EQUALS"}}} + result = call_graphql(query, variables) + log.debug("get_gallery_by_path callGraphQL result " + str(result)) + return result['findGalleries']['galleries'][0]['id'] + +def update_gallery(input): + log.debug("gallery input " + str(input)) + query = """ + mutation GalleryUpdate($input : GalleryUpdateInput!) { + galleryUpdate(input: $input) { + id + title + } + } + """ + variables = { + "input": input + } + result = call_graphql(query, variables) + if result: + g_id = result['galleryUpdate'].get('id') + g_title = result['galleryUpdate'].get('title') + log.info(f"updated Gallery ({g_id}): {g_title}") + return result + +def get_id(obj): + ids = [] + for item in obj: + ids.append(item['id']) + return ids + +def find_galleries(scene_id, scene_path): + ids = [] + directory_path = os.path.dirname(scene_path) + for (cur, dirs, files) in os.walk(directory_path): + + for file in files: + if file.endswith('.zip'): + gallery_path = os.path.join(cur, file) + id = get_gallery_id_by_path(gallery_path) + updateScene_with_gallery(scene_id, id) + ids.append(id) + break + log.debug("find_galleries ids' found " + str(ids)) + return ids + +def updateScene_with_gallery(scene_id, gallery_id): + data = {'id': scene_id, 'gallery_ids': [gallery_id]} + log.debug("data " + str(data)) + query = """ + mutation SceneUpdate($input : SceneUpdateInput!) { + sceneUpdate(input: $input) { + id + title + } + } + """ + variables = { + "input": data + } + result = call_graphql(query, variables) + log.debug("graphql_updateGallery callGraphQL result " + str(result)) + +FRAGMENT = json.loads(sys.stdin.read()) +SCENE_ID = FRAGMENT.get("id") + +scene = graphql.getScene(SCENE_ID) +if scene: + scene_galleries = scene['galleries'] + log.debug("scene_galleries " + str(scene_galleries)) + gallery_ids = [] + if len(scene_galleries) > 0: + for gallery_obj in scene_galleries: + gallery_ids.append(gallery_obj['id']) + elif find_gallery: + # if no galleries are associated see if any gallery zips exist in directory + gallery_ids = find_galleries(SCENE_ID, scene["path"]) + log.debug("gallery_ids " + str(gallery_ids)) + + for gallery_id in gallery_ids: + studio = None + if scene['studio']: + studio = scene['studio']['id'] + gallery_input = {'id': gallery_id, + 'urls': scene['urls'], + 'title': scene['title'], + 'date': scene["date"], + 'details': scene['details'], + 'studio_id': studio, + 'tag_ids': get_id(scene['tags']), + 'performer_ids': get_id(scene['performers'])} + update_gallery(gallery_input) + + print(json.dumps({})) + diff --git a/scrapers/CopyToGallery.yml b/scrapers/CopyToGallery/CopyToGallery.yml similarity index 85% rename from scrapers/CopyToGallery.yml rename to scrapers/CopyToGallery/CopyToGallery.yml index 732b8ab3..226b2319 100644 --- a/scrapers/CopyToGallery.yml +++ b/scrapers/CopyToGallery/CopyToGallery.yml @@ -1,4 +1,6 @@ name: "Copy to Gallery" +# requires: py_common + sceneByFragment: action: script script: diff --git a/scrapers/Algolia_DogfartNetwork.yml b/scrapers/DogfartNetwork/DogfartNetwork.yml similarity index 90% rename from scrapers/Algolia_DogfartNetwork.yml rename to scrapers/DogfartNetwork/DogfartNetwork.yml index 06496844..4bf3304d 100644 --- a/scrapers/Algolia_DogfartNetwork.yml +++ b/scrapers/DogfartNetwork/DogfartNetwork.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "DogfartNetwork" sceneByURL: - action: script @@ -19,26 +20,26 @@ sceneByURL: - zebragirls.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - dogfartnetwork sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - dogfartnetwork sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - dogfartnetwork - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - dogfartnetwork - validName galleryByURL: @@ -61,7 +62,7 @@ galleryByURL: - zebragirls.com/en/photo script: - python - - Algolia.py + - ../Algolia/Algolia.py - dogfartnetwork - gallery # Last Updated July 27, 2023 diff --git a/scrapers/Algolia_EvilAngel.yml b/scrapers/EvilAngel/EvilAngel.yml similarity index 84% rename from scrapers/Algolia_EvilAngel.yml rename to scrapers/EvilAngel/EvilAngel.yml index c13bbdb3..c9bb7d23 100644 --- a/scrapers/Algolia_EvilAngel.yml +++ b/scrapers/EvilAngel/EvilAngel.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "EvilAngel" sceneByURL: - action: script @@ -9,7 +10,7 @@ sceneByURL: - tsfactor.com/en/video/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - evilangel movieByURL: - action: script @@ -21,27 +22,27 @@ movieByURL: - tsfactor.com/en/movie/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - evilangel - movie sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - evilangel sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - evilangel - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - evilangel - validName galleryByURL: @@ -54,7 +55,7 @@ galleryByURL: - tsfactor.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - evilangel - gallery # Last Updated December 26, 2022 diff --git a/scrapers/Algolia_FalconStudios.yml b/scrapers/FalconStudios/FalconStudios.yml similarity index 77% rename from scrapers/Algolia_FalconStudios.yml rename to scrapers/FalconStudios/FalconStudios.yml index 10ac8b05..61bc6196 100644 --- a/scrapers/Algolia_FalconStudios.yml +++ b/scrapers/FalconStudios/FalconStudios.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: Falcon Studios sceneByURL: - action: script @@ -6,26 +7,26 @@ sceneByURL: - hothouse.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - falconstudios sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - falconstudios sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - falconstudios - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - falconstudios - validName # Last Updated September 09, 2023 diff --git a/scrapers/Algolia_FantasyMassage.yml b/scrapers/FantasyMassage/FantasyMassage.yml similarity index 78% rename from scrapers/Algolia_FantasyMassage.yml rename to scrapers/FantasyMassage/FantasyMassage.yml index 22e38736..f5bdf03e 100644 --- a/scrapers/Algolia_FantasyMassage.yml +++ b/scrapers/FantasyMassage/FantasyMassage.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "FantasyMassage" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - fantasymassage.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - fantasymassage sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - fantasymassage sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - fantasymassage - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - fantasymassage - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - fantasymassage.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - fantasymassage - gallery # Last Updated December 22, 2022 diff --git a/scrapers/Filename.py b/scrapers/Filename/Filename.py similarity index 100% rename from scrapers/Filename.py rename to scrapers/Filename/Filename.py diff --git a/scrapers/Filename.yml b/scrapers/Filename/Filename.yml similarity index 87% rename from scrapers/Filename.yml rename to scrapers/Filename/Filename.yml index 41380fe6..0a8e037a 100644 --- a/scrapers/Filename.yml +++ b/scrapers/Filename/Filename.yml @@ -1,4 +1,6 @@ name: Filename +# requires: py_common + sceneByFragment: action: script script: diff --git a/scrapers/Algolia_FistingInferno.yml b/scrapers/FistingInferno/FistingInferno.yml similarity index 78% rename from scrapers/Algolia_FistingInferno.yml rename to scrapers/FistingInferno/FistingInferno.yml index 591d3868..f4a90075 100644 --- a/scrapers/Algolia_FistingInferno.yml +++ b/scrapers/FistingInferno/FistingInferno.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "FistingInferno" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - fistinginferno.com/en/video/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - fistinginferno sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - fistinginferno sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - fistinginferno - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - fistinginferno - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - fistinginferno.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - fistinginferno - gallery # Last Updated September 24, 2023 diff --git a/scrapers/Fit18.py b/scrapers/Fit18/Fit18.py similarity index 100% rename from scrapers/Fit18.py rename to scrapers/Fit18/Fit18.py diff --git a/scrapers/Fit18.yml b/scrapers/Fit18/Fit18.yml similarity index 88% rename from scrapers/Fit18.yml rename to scrapers/Fit18/Fit18.yml index e21d0667..c6be98c4 100644 --- a/scrapers/Fit18.yml +++ b/scrapers/Fit18/Fit18.yml @@ -1,4 +1,6 @@ name: Fit18 +# requires: py_common + sceneByURL: - url: - fit18.com/videos/ diff --git a/scrapers/FratX.py b/scrapers/FratX/FratX.py similarity index 100% rename from scrapers/FratX.py rename to scrapers/FratX/FratX.py diff --git a/scrapers/FratX.yml b/scrapers/FratX/FratX.yml similarity index 92% rename from scrapers/FratX.yml rename to scrapers/FratX/FratX.yml index 1e9081f6..90ab0329 100644 --- a/scrapers/FratX.yml +++ b/scrapers/FratX/FratX.yml @@ -1,4 +1,6 @@ name: FratX +# requires: py_common + sceneByURL: - action: script script: diff --git a/scrapers/Algolia_GangBangCreampie.yml b/scrapers/GangBangCreampie/GangBangCreampie.yml similarity index 78% rename from scrapers/Algolia_GangBangCreampie.yml rename to scrapers/GangBangCreampie/GangBangCreampie.yml index 1eae61aa..f49d827e 100644 --- a/scrapers/Algolia_GangBangCreampie.yml +++ b/scrapers/GangBangCreampie/GangBangCreampie.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "GangbangCreampie" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - gangbangcreampie.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - gangbangcreampie sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - gangbangcreampie sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - gangbangcreampie - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - gangbangcreampie - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - gangbangcreampie.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - gangbangcreampie - gallery # Last Updated December 22, 2022 diff --git a/scrapers/Algolia_GenderXFilms.yml b/scrapers/GenderXFilms/GenderXFilms.yml similarity index 78% rename from scrapers/Algolia_GenderXFilms.yml rename to scrapers/GenderXFilms/GenderXFilms.yml index fd567c4c..aad91c96 100644 --- a/scrapers/Algolia_GenderXFilms.yml +++ b/scrapers/GenderXFilms/GenderXFilms.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: GenderX Films sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - genderxfilms.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - genderxfilms sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - genderxfilms sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - genderxfilms - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - genderxfilms - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - genderxfilms.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - genderxfilms - gallery movieByURL: @@ -42,7 +43,7 @@ movieByURL: - genderxfilms.com/en/movie/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - genderxfilms - movie # Last Updated December 22, 2022 diff --git a/scrapers/Algolia_Girlfriendsfilms.yml b/scrapers/Girlfriendsfilms/Girlfriendsfilms.yml similarity index 79% rename from scrapers/Algolia_Girlfriendsfilms.yml rename to scrapers/Girlfriendsfilms/Girlfriendsfilms.yml index 8e02c4bc..933251fa 100644 --- a/scrapers/Algolia_Girlfriendsfilms.yml +++ b/scrapers/Girlfriendsfilms/Girlfriendsfilms.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "Girlfriends Films" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - girlfriendsfilms.com/en/video/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlfriendsfilms sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlfriendsfilms sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlfriendsfilms - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlfriendsfilms - validName movieByURL: @@ -33,7 +34,7 @@ movieByURL: - girlfriendsfilms.com/en/movie/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlfriendsfilms - movie galleryByURL: @@ -42,7 +43,7 @@ galleryByURL: - girlfriendsfilms.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlfriendsfilms - gallery # Last Updated December 22, 2022 diff --git a/scrapers/Algolia_Girlsway.yml b/scrapers/Girlsway/Girlsway.yml similarity index 77% rename from scrapers/Algolia_Girlsway.yml rename to scrapers/Girlsway/Girlsway.yml index d81ad7db..4b37bb8b 100644 --- a/scrapers/Algolia_Girlsway.yml +++ b/scrapers/Girlsway/Girlsway.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "GirlsWay" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - girlsway.com/en/video/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlsway sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlsway sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlsway - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlsway - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - girlsway.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - girlsway - gallery # Last Updated December 22, 2022 diff --git a/scrapers/Algolia_Gloryholesecrets.yml b/scrapers/Gloryholesecrets/Gloryholesecrets.yml similarity index 78% rename from scrapers/Algolia_Gloryholesecrets.yml rename to scrapers/Gloryholesecrets/Gloryholesecrets.yml index c69808d6..4fbc1885 100644 --- a/scrapers/Algolia_Gloryholesecrets.yml +++ b/scrapers/Gloryholesecrets/Gloryholesecrets.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: GloryholeSecrets sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - gloryholesecrets.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - gloryholesecrets sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - gloryholesecrets sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - gloryholesecrets - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - gloryholesecrets - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - gloryholesecrets.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - gloryholesecrets - gallery # Last Updated August 22, 2023 diff --git a/scrapers/IAFD.py b/scrapers/IAFD/IAFD.py similarity index 100% rename from scrapers/IAFD.py rename to scrapers/IAFD/IAFD.py diff --git a/scrapers/IAFD.yml b/scrapers/IAFD/IAFD.yml similarity index 95% rename from scrapers/IAFD.yml rename to scrapers/IAFD/IAFD.yml index 3084f927..1a3acdc3 100644 --- a/scrapers/IAFD.yml +++ b/scrapers/IAFD/IAFD.yml @@ -1,4 +1,6 @@ name: IAFD +# requires: py_common + performerByName: action: script script: diff --git a/scrapers/IFeelMyself.py b/scrapers/IFeelMyself/IFeelMyself.py similarity index 100% rename from scrapers/IFeelMyself.py rename to scrapers/IFeelMyself/IFeelMyself.py diff --git a/scrapers/IFeelMyself.yml b/scrapers/IFeelMyself/IFeelMyself.yml similarity index 100% rename from scrapers/IFeelMyself.yml rename to scrapers/IFeelMyself/IFeelMyself.yml diff --git a/scrapers/JacquieEtMichelTV.py b/scrapers/JacquieEtMichelTV/JacquieEtMichelTV.py similarity index 100% rename from scrapers/JacquieEtMichelTV.py rename to scrapers/JacquieEtMichelTV/JacquieEtMichelTV.py diff --git a/scrapers/JacquieEtMichelTV.yml b/scrapers/JacquieEtMichelTV/JacquieEtMichelTV.yml similarity index 94% rename from scrapers/JacquieEtMichelTV.yml rename to scrapers/JacquieEtMichelTV/JacquieEtMichelTV.yml index 30a90c91..b92cafe2 100644 --- a/scrapers/JacquieEtMichelTV.yml +++ b/scrapers/JacquieEtMichelTV/JacquieEtMichelTV.yml @@ -1,4 +1,6 @@ name: JacquieEtMicaelTV +# requires: py_common + sceneByFragment: action: script script: diff --git a/scrapers/JavLibrary_python.py b/scrapers/JavLibrary_python/JavLibrary_python.py similarity index 100% rename from scrapers/JavLibrary_python.py rename to scrapers/JavLibrary_python/JavLibrary_python.py diff --git a/scrapers/JavLibrary_python.yml b/scrapers/JavLibrary_python/JavLibrary_python.yml similarity index 93% rename from scrapers/JavLibrary_python.yml rename to scrapers/JavLibrary_python/JavLibrary_python.yml index 457df64d..5066baa7 100644 --- a/scrapers/JavLibrary_python.yml +++ b/scrapers/JavLibrary_python/JavLibrary_python.yml @@ -1,4 +1,6 @@ name: "JavLibrary Python" +# requires: py_common + sceneByFragment: action: script script: diff --git a/scrapers/Algolia_Johnnyrapid.yml b/scrapers/Johnnyrapid/Johnnyrapid.yml similarity index 77% rename from scrapers/Algolia_Johnnyrapid.yml rename to scrapers/Johnnyrapid/Johnnyrapid.yml index cb6d91c0..00840164 100644 --- a/scrapers/Algolia_Johnnyrapid.yml +++ b/scrapers/Johnnyrapid/Johnnyrapid.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "Johnny Rapid" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - johnnyrapid.com/en/video/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - johnnyrapid sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - johnnyrapid sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - johnnyrapid - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - johnnyrapid - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - johnnyrapid.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - johnnyrapid - gallery # Last Updated December 22, 2022 diff --git a/scrapers/KBProductions.py b/scrapers/KBProductions/KBProductions.py similarity index 100% rename from scrapers/KBProductions.py rename to scrapers/KBProductions/KBProductions.py diff --git a/scrapers/KBProductions.yml b/scrapers/KBProductions/KBProductions.yml similarity index 96% rename from scrapers/KBProductions.yml rename to scrapers/KBProductions/KBProductions.yml index cdb300a2..d100ae75 100644 --- a/scrapers/KBProductions.yml +++ b/scrapers/KBProductions/KBProductions.yml @@ -1,4 +1,6 @@ name: "KB Productions" +# requires: py_common + sceneByURL: - url: # Keeping this to allow for updates using old urls diff --git a/scrapers/Loyalfans/Loyalfans.yml b/scrapers/Loyalfans/Loyalfans.yml index 74f0c006..e32fedd3 100644 --- a/scrapers/Loyalfans/Loyalfans.yml +++ b/scrapers/Loyalfans/Loyalfans.yml @@ -1,4 +1,6 @@ name: Loyalfans +# requires: py_common + sceneByURL: - url: - loyalfans.com diff --git a/scrapers/ManyVids/ManyVids.yml b/scrapers/ManyVids/ManyVids.yml index a17d87fc..0d492db4 100644 --- a/scrapers/ManyVids/ManyVids.yml +++ b/scrapers/ManyVids/ManyVids.yml @@ -1,4 +1,6 @@ name: ManyVids +# requires: py_common + sceneByURL: - url: - manyvids.com/Video diff --git a/scrapers/Algolia_MenOver30.yml b/scrapers/MenOver30/MenOver30.yml similarity index 79% rename from scrapers/Algolia_MenOver30.yml rename to scrapers/MenOver30/MenOver30.yml index f0a36291..8342bd99 100644 --- a/scrapers/Algolia_MenOver30.yml +++ b/scrapers/MenOver30/MenOver30.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "MenOver30" sceneByURL: - action: script @@ -6,26 +7,26 @@ sceneByURL: - pridestudios.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - menover30 sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - menover30 sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - menover30 - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - menover30 - validName galleryByURL: @@ -35,7 +36,7 @@ galleryByURL: - pridestudios.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - menover30 - gallery # Last Updated December 22, 2022 diff --git a/scrapers/MindGeekAPI.py b/scrapers/MindGeekAPI/MindGeekAPI.py similarity index 100% rename from scrapers/MindGeekAPI.py rename to scrapers/MindGeekAPI/MindGeekAPI.py diff --git a/scrapers/MindGeekAPI.yml b/scrapers/MindGeekAPI/MindGeekAPI.yml similarity index 94% rename from scrapers/MindGeekAPI.yml rename to scrapers/MindGeekAPI/MindGeekAPI.yml index 70d9af86..bf83410c 100644 --- a/scrapers/MindGeekAPI.yml +++ b/scrapers/MindGeekAPI/MindGeekAPI.yml @@ -1,4 +1,6 @@ name: "MindGeekAPI" +# requires: py_common + sceneByFragment: action: script script: diff --git a/scrapers/MissaX.py b/scrapers/MissaX/MissaX.py similarity index 100% rename from scrapers/MissaX.py rename to scrapers/MissaX/MissaX.py diff --git a/scrapers/MissaX.yml b/scrapers/MissaX/MissaX.yml similarity index 88% rename from scrapers/MissaX.yml rename to scrapers/MissaX/MissaX.yml index 931949e2..cfca1a5f 100644 --- a/scrapers/MissaX.yml +++ b/scrapers/MissaX/MissaX.yml @@ -1,4 +1,6 @@ name: "MissaX" +# requires: py_common + sceneByURL: - action: script script: diff --git a/scrapers/ModelCentroAPI/ModelCentroAPI.yml b/scrapers/ModelCentroAPI/ModelCentroAPI.yml index 65ddd4e8..86b1a782 100644 --- a/scrapers/ModelCentroAPI/ModelCentroAPI.yml +++ b/scrapers/ModelCentroAPI/ModelCentroAPI.yml @@ -1,4 +1,6 @@ name: "ModelCentroAPI" +# requires: py_common + sceneByURL: - url: - beingphoenixmarie.com/scene/ diff --git a/scrapers/Algolia_NextDoorStudios.yml b/scrapers/NextDoorStudios/NextDoorStudios.yml similarity index 88% rename from scrapers/Algolia_NextDoorStudios.yml rename to scrapers/NextDoorStudios/NextDoorStudios.yml index 07099fcd..61263df9 100644 --- a/scrapers/Algolia_NextDoorStudios.yml +++ b/scrapers/NextDoorStudios/NextDoorStudios.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: Next Door Studios sceneByURL: - action: script @@ -23,26 +24,26 @@ sceneByURL: - trystanbull.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - nextdoorstudios sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - nextdoorstudios sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - nextdoorstudios - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - nextdoorstudios - validName # Last Updated August 18, 2023 diff --git a/scrapers/PerfectGonzo.py b/scrapers/PerfectGonzo/PerfectGonzo.py similarity index 100% rename from scrapers/PerfectGonzo.py rename to scrapers/PerfectGonzo/PerfectGonzo.py diff --git a/scrapers/PerfectGonzo.yml b/scrapers/PerfectGonzo/PerfectGonzo.yml similarity index 100% rename from scrapers/PerfectGonzo.yml rename to scrapers/PerfectGonzo/PerfectGonzo.yml diff --git a/scrapers/PremiumBukkake.py b/scrapers/PremiumBukkake/PremiumBukkake.py similarity index 100% rename from scrapers/PremiumBukkake.py rename to scrapers/PremiumBukkake/PremiumBukkake.py diff --git a/scrapers/PremiumBukkake.yml b/scrapers/PremiumBukkake/PremiumBukkake.yml similarity index 100% rename from scrapers/PremiumBukkake.yml rename to scrapers/PremiumBukkake/PremiumBukkake.yml diff --git a/scrapers/Algolia_RagingStallion.yml b/scrapers/RagingStallion/RagingStallion.yml similarity index 77% rename from scrapers/Algolia_RagingStallion.yml rename to scrapers/RagingStallion/RagingStallion.yml index e6a94864..373a88b3 100644 --- a/scrapers/Algolia_RagingStallion.yml +++ b/scrapers/RagingStallion/RagingStallion.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: Raging Stallion sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - ragingstallion.com/en/scene script: - python - - Algolia.py + - ../Algolia/Algolia.py - ragingstallion sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - ragingstallion sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - ragingstallion - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - ragingstallion - validName # Last Updated September 09, 2023 diff --git a/scrapers/RealityLovers.py b/scrapers/RealityLovers/RealityLovers.py similarity index 100% rename from scrapers/RealityLovers.py rename to scrapers/RealityLovers/RealityLovers.py diff --git a/scrapers/RealityLovers.yml b/scrapers/RealityLovers/RealityLovers.yml similarity index 93% rename from scrapers/RealityLovers.yml rename to scrapers/RealityLovers/RealityLovers.yml index 0c70b608..9ea81438 100644 --- a/scrapers/RealityLovers.yml +++ b/scrapers/RealityLovers/RealityLovers.yml @@ -1,4 +1,6 @@ name: RealityLovers +# requires: py_common + performerByURL: - action: script url: diff --git a/scrapers/Redgifs.py b/scrapers/Redgifs/Redgifs.py similarity index 100% rename from scrapers/Redgifs.py rename to scrapers/Redgifs/Redgifs.py diff --git a/scrapers/Redgifs.yml b/scrapers/Redgifs/Redgifs.yml similarity index 95% rename from scrapers/Redgifs.yml rename to scrapers/Redgifs/Redgifs.yml index d62e9c3a..73ff5b56 100644 --- a/scrapers/Redgifs.yml +++ b/scrapers/Redgifs/Redgifs.yml @@ -1,4 +1,6 @@ name: Redgifs +# requires: py_common + sceneByURL: - action: script script: diff --git a/scrapers/Algolia_RoccoSiffredi.yml b/scrapers/RoccoSiffredi/RoccoSiffredi.yml similarity index 78% rename from scrapers/Algolia_RoccoSiffredi.yml rename to scrapers/RoccoSiffredi/RoccoSiffredi.yml index 6c22a531..1c41ad74 100644 --- a/scrapers/Algolia_RoccoSiffredi.yml +++ b/scrapers/RoccoSiffredi/RoccoSiffredi.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "RoccoSiffredi" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - roccosiffredi.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - roccosiffredi sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - roccosiffredi sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - roccosiffredi - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - roccosiffredi - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - roccosiffredi.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - roccosiffredi - gallery # Last Updated December 22, 2022 diff --git a/scrapers/SARJ-LLC.py b/scrapers/SARJ-LLC/SARJ-LLC.py similarity index 100% rename from scrapers/SARJ-LLC.py rename to scrapers/SARJ-LLC/SARJ-LLC.py diff --git a/scrapers/SARJ-LLC.yml b/scrapers/SARJ-LLC/SARJ-LLC.yml similarity index 98% rename from scrapers/SARJ-LLC.yml rename to scrapers/SARJ-LLC/SARJ-LLC.yml index cf7b5fe6..01bef872 100644 --- a/scrapers/SARJ-LLC.yml +++ b/scrapers/SARJ-LLC/SARJ-LLC.yml @@ -1,4 +1,5 @@ name: SARJ LLC +# requires: py_common sceneByName: action: script diff --git a/scrapers/ScrapeWithURL.py b/scrapers/ScrapeWithURL/ScrapeWithURL.py similarity index 95% rename from scrapers/ScrapeWithURL.py rename to scrapers/ScrapeWithURL/ScrapeWithURL.py index 25312d9e..1498ded7 100644 --- a/scrapers/ScrapeWithURL.py +++ b/scrapers/ScrapeWithURL/ScrapeWithURL.py @@ -1,81 +1,81 @@ -import json -import sys - -try: - import py_common.graphql as graphql - import py_common.log as log -except ModuleNotFoundError: - print( - "You need to download the folder 'py_common' from the community repo! (CommunityScrapers/tree/master/scrapers/py_common)", - file=sys.stderr, - ) - sys.exit() - - -def scrape_scene(url): - query = """ -query scrapeSceneURL($url: String!) { - scrapeSceneURL(url: $url) { - title - details - code - date - image - urls - studio { - name - url - image - parent { - name - url - image - } - } - tags { - name - } - performers { - aliases - birthdate - career_length - country - death_date - details - ethnicity - eye_color - fake_tits - gender - hair_color - height - instagram - images - measurements - name - piercings - tags { - name - } - tattoos - twitter - url - weight - } - } -}""" - - variables = {"url": url} - result = graphql.callGraphQL(query, variables) - log.debug(f"result {result}") - if result: - return result["scrapeSceneURL"] - - -FRAGMENT = json.loads(sys.stdin.read()) -url = FRAGMENT.get("url") - -if url: - result = scrape_scene(url) - print(json.dumps(result)) -else: - print("null") +import json +import sys + +try: + import py_common.graphql as graphql + import py_common.log as log +except ModuleNotFoundError: + print( + "You need to download the folder 'py_common' from the community repo! (CommunityScrapers/tree/master/scrapers/py_common)", + file=sys.stderr, + ) + sys.exit() + + +def scrape_scene(url): + query = """ +query scrapeSceneURL($url: String!) { + scrapeSceneURL(url: $url) { + title + details + code + date + image + urls + studio { + name + url + image + parent { + name + url + image + } + } + tags { + name + } + performers { + aliases + birthdate + career_length + country + death_date + details + ethnicity + eye_color + fake_tits + gender + hair_color + height + instagram + images + measurements + name + piercings + tags { + name + } + tattoos + twitter + url + weight + } + } +}""" + + variables = {"url": url} + result = graphql.callGraphQL(query, variables) + log.debug(f"result {result}") + if result: + return result["scrapeSceneURL"] + + +FRAGMENT = json.loads(sys.stdin.read()) +url = FRAGMENT.get("url") + +if url: + result = scrape_scene(url) + print(json.dumps(result)) +else: + print("null") diff --git a/scrapers/ScrapeWithURL.yml b/scrapers/ScrapeWithURL/ScrapeWithURL.yml similarity index 86% rename from scrapers/ScrapeWithURL.yml rename to scrapers/ScrapeWithURL/ScrapeWithURL.yml index dfc7541d..fbd2e459 100644 --- a/scrapers/ScrapeWithURL.yml +++ b/scrapers/ScrapeWithURL/ScrapeWithURL.yml @@ -1,8 +1,10 @@ -name: Scrape with URL -sceneByFragment: - action: script - script: - - python - - ScrapeWithURL.py - -# Last Updated April 16, 2022 +name: Scrape with URL +# requires: py_common + +sceneByFragment: + action: script + script: + - python + - ScrapeWithURL.py + +# Last Updated April 16, 2022 diff --git a/scrapers/ShokoAPI/ShokoAPI.yml b/scrapers/ShokoAPI/ShokoAPI.yml index bab4dc55..64777953 100644 --- a/scrapers/ShokoAPI/ShokoAPI.yml +++ b/scrapers/ShokoAPI/ShokoAPI.yml @@ -1,4 +1,6 @@ name: "ShokoAPI" +# requires: py_common + sceneByFragment: action: script script: diff --git a/scrapers/Algolia_TabooHeat.yml b/scrapers/TabooHeat/TabooHeat.yml similarity index 77% rename from scrapers/Algolia_TabooHeat.yml rename to scrapers/TabooHeat/TabooHeat.yml index e695f589..455d42d3 100644 --- a/scrapers/Algolia_TabooHeat.yml +++ b/scrapers/TabooHeat/TabooHeat.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "TabooHeat" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - tabooheat.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - tabooheat sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - tabooheat sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - tabooheat - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - tabooheat - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - tabooheat.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - tabooheat - gallery # Last Updated December 22, 2022 diff --git a/scrapers/Teamskeet.yml b/scrapers/Teamskeet/Teamskeet.yml similarity index 90% rename from scrapers/Teamskeet.yml rename to scrapers/Teamskeet/Teamskeet.yml index 35eb892e..dbca726b 100644 --- a/scrapers/Teamskeet.yml +++ b/scrapers/Teamskeet/Teamskeet.yml @@ -1,4 +1,6 @@ name: "TeamskeetAPI" +# requires: py_common + sceneByURL: - url: - teamskeet.com/movies/ diff --git a/scrapers/TeamskeetAPI.py b/scrapers/Teamskeet/TeamskeetAPI.py similarity index 100% rename from scrapers/TeamskeetAPI.py rename to scrapers/Teamskeet/TeamskeetAPI.py diff --git a/scrapers/Tokyohot.py b/scrapers/Tokyohot/Tokyohot.py similarity index 100% rename from scrapers/Tokyohot.py rename to scrapers/Tokyohot/Tokyohot.py diff --git a/scrapers/Tokyohot.yml b/scrapers/Tokyohot/Tokyohot.yml similarity index 97% rename from scrapers/Tokyohot.yml rename to scrapers/Tokyohot/Tokyohot.yml index 32cbb837..1f976b3d 100644 --- a/scrapers/Tokyohot.yml +++ b/scrapers/Tokyohot/Tokyohot.yml @@ -1,4 +1,6 @@ name: tokyohot +# requires: py_common + sceneByURL: - action: scrapeXPath url: diff --git a/scrapers/TopWebModels.py b/scrapers/TopWebModels/TopWebModels.py similarity index 100% rename from scrapers/TopWebModels.py rename to scrapers/TopWebModels/TopWebModels.py diff --git a/scrapers/TopWebModels.yml b/scrapers/TopWebModels/TopWebModels.yml similarity index 95% rename from scrapers/TopWebModels.yml rename to scrapers/TopWebModels/TopWebModels.yml index 0fb0532a..ae9a847e 100644 --- a/scrapers/TopWebModels.yml +++ b/scrapers/TopWebModels/TopWebModels.yml @@ -1,4 +1,6 @@ name: "Top Web Models" +# requires: py_common + sceneByURL: - url: - tour.2girls1camera.com/scenes diff --git a/scrapers/Traxxx.py b/scrapers/Traxxx/Traxxx.py similarity index 100% rename from scrapers/Traxxx.py rename to scrapers/Traxxx/Traxxx.py diff --git a/scrapers/Traxxx.yml b/scrapers/Traxxx/Traxxx.yml similarity index 97% rename from scrapers/Traxxx.yml rename to scrapers/Traxxx/Traxxx.yml index 2186c9a1..f3d6a851 100644 --- a/scrapers/Traxxx.yml +++ b/scrapers/Traxxx/Traxxx.yml @@ -1,4 +1,6 @@ name: "Traxxx" +# requires: py_common + sceneByURL: - url: - traxxx.me/scene/ diff --git a/scrapers/traxxx_interface.py b/scrapers/Traxxx/traxxx_interface.py similarity index 100% rename from scrapers/traxxx_interface.py rename to scrapers/Traxxx/traxxx_interface.py diff --git a/scrapers/WAPdB.py b/scrapers/WAPdB/WAPdB.py similarity index 100% rename from scrapers/WAPdB.py rename to scrapers/WAPdB/WAPdB.py diff --git a/scrapers/WAPdB.yml b/scrapers/WAPdB/WAPdB.yml similarity index 100% rename from scrapers/WAPdB.yml rename to scrapers/WAPdB/WAPdB.yml diff --git a/scrapers/Algolia_Wicked.yml b/scrapers/Wicked/Wicked.yml similarity index 77% rename from scrapers/Algolia_Wicked.yml rename to scrapers/Wicked/Wicked.yml index 236c3bf7..390020a4 100644 --- a/scrapers/Algolia_Wicked.yml +++ b/scrapers/Wicked/Wicked.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "Wicked" sceneByURL: - action: script @@ -5,7 +6,7 @@ sceneByURL: - wicked.com/en/video/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - wicked movieByURL: - action: script @@ -13,27 +14,27 @@ movieByURL: - wicked.com/en/movie/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - wicked - movie sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - wicked sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - wicked - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - wicked - validName galleryByURL: @@ -42,7 +43,7 @@ galleryByURL: - wicked.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - wicked - gallery # Last Updated December 22, 2022 diff --git a/scrapers/WowNetworkVenus/WowNetworkVenus.yml b/scrapers/WowNetworkVenus/WowNetworkVenus.yml index c8fa2aa4..5f4a8672 100644 --- a/scrapers/WowNetworkVenus/WowNetworkVenus.yml +++ b/scrapers/WowNetworkVenus/WowNetworkVenus.yml @@ -1,4 +1,6 @@ name: WowNetworkVenus +# requires: py_common + sceneByURL: - action: script script: diff --git a/scrapers/dc-onlyfans.py b/scrapers/dc-onlyfans/dc-onlyfans.py similarity index 100% rename from scrapers/dc-onlyfans.py rename to scrapers/dc-onlyfans/dc-onlyfans.py diff --git a/scrapers/dc-onlyfans.yml b/scrapers/dc-onlyfans/dc-onlyfans.yml similarity index 93% rename from scrapers/dc-onlyfans.yml rename to scrapers/dc-onlyfans/dc-onlyfans.yml index 3e9edb52..16df8e81 100644 --- a/scrapers/dc-onlyfans.yml +++ b/scrapers/dc-onlyfans/dc-onlyfans.yml @@ -1,4 +1,6 @@ name: "DC Onlyfans" +# requires: py_common + sceneByFragment: action: script script: diff --git a/scrapers/Algolia_disruptivefilms.yml b/scrapers/disruptivefilms/disruptivefilms.yml similarity index 78% rename from scrapers/Algolia_disruptivefilms.yml rename to scrapers/disruptivefilms/disruptivefilms.yml index f857fb8e..47373a0d 100644 --- a/scrapers/Algolia_disruptivefilms.yml +++ b/scrapers/disruptivefilms/disruptivefilms.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "DisruptiveFilms" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - disruptivefilms.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - disruptivefilms sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - disruptivefilms sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - disruptivefilms - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - disruptivefilms - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - disruptivefilms.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - disruptivefilms - gallery # Last Updated December 22, 2022 diff --git a/scrapers/Algolia_filthykings.yml b/scrapers/filthykings/filthykings.yml similarity index 77% rename from scrapers/Algolia_filthykings.yml rename to scrapers/filthykings/filthykings.yml index 55dee88d..8c0941b7 100644 --- a/scrapers/Algolia_filthykings.yml +++ b/scrapers/filthykings/filthykings.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "FilthyKings" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - filthykings.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - filthykings sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - filthykings sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - filthykings - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - filthykings - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - filthykings.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - filthykings - gallery # Last Updated December 22, 2022 diff --git a/scrapers/jellyfin.py b/scrapers/jellyfin/jellyfin.py similarity index 100% rename from scrapers/jellyfin.py rename to scrapers/jellyfin/jellyfin.py diff --git a/scrapers/jellyfin.yml b/scrapers/jellyfin/jellyfin.yml similarity index 97% rename from scrapers/jellyfin.yml rename to scrapers/jellyfin/jellyfin.yml index 9017d10e..0efe6754 100644 --- a/scrapers/jellyfin.yml +++ b/scrapers/jellyfin/jellyfin.yml @@ -1,4 +1,6 @@ name: Jellyfin +# requires: py_common + performerByURL: - url: #Replace this with you local jellyfin host diff --git a/scrapers/multiscrape.py b/scrapers/multiscrape/multiscrape.py similarity index 100% rename from scrapers/multiscrape.py rename to scrapers/multiscrape/multiscrape.py diff --git a/scrapers/multiscrape.yml b/scrapers/multiscrape/multiscrape.yml similarity index 100% rename from scrapers/multiscrape.yml rename to scrapers/multiscrape/multiscrape.yml diff --git a/scrapers/Algolia_mypervyfamily.yml b/scrapers/mypervyfamily/mypervyfamily.yml similarity index 78% rename from scrapers/Algolia_mypervyfamily.yml rename to scrapers/mypervyfamily/mypervyfamily.yml index 2474ce32..74e353a1 100644 --- a/scrapers/Algolia_mypervyfamily.yml +++ b/scrapers/mypervyfamily/mypervyfamily.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "MyPervyFamily" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - mypervyfamily.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - mypervyfamily sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - mypervyfamily sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - mypervyfamily - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - mypervyfamily - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - mypervyfamily.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - mypervyfamily - gallery # Last Updated December 22, 2022 diff --git a/scrapers/performer-image-by-scene.py b/scrapers/performer-image-by-scene/performer-image-by-scene.py similarity index 100% rename from scrapers/performer-image-by-scene.py rename to scrapers/performer-image-by-scene/performer-image-by-scene.py diff --git a/scrapers/performer-image-by-scene.yml b/scrapers/performer-image-by-scene/performer-image-by-scene.yml similarity index 92% rename from scrapers/performer-image-by-scene.yml rename to scrapers/performer-image-by-scene/performer-image-by-scene.yml index fbbacc26..c046fb2e 100644 --- a/scrapers/performer-image-by-scene.yml +++ b/scrapers/performer-image-by-scene/performer-image-by-scene.yml @@ -1,4 +1,5 @@ name: Performer Image by scene cover +# requires: py_common performerByFragment: action: script diff --git a/scrapers/performer-image-dir.py b/scrapers/performer-image-dir/performer-image-dir.py similarity index 100% rename from scrapers/performer-image-dir.py rename to scrapers/performer-image-dir/performer-image-dir.py diff --git a/scrapers/performer-image-dir.yml b/scrapers/performer-image-dir/performer-image-dir.yml similarity index 100% rename from scrapers/performer-image-dir.yml rename to scrapers/performer-image-dir/performer-image-dir.yml diff --git a/scrapers/py_common/package b/scrapers/py_common/package new file mode 100644 index 00000000..6c171ab0 --- /dev/null +++ b/scrapers/py_common/package @@ -0,0 +1,2 @@ +# script used as a dependency only +name: py_common diff --git a/scrapers/rb_common/package b/scrapers/rb_common/package new file mode 100644 index 00000000..363e2f4f --- /dev/null +++ b/scrapers/rb_common/package @@ -0,0 +1,2 @@ +# script used as a dependency only +name: rb_common diff --git a/scrapers/stash-sqlite.py b/scrapers/stash-sqlite/stash-sqlite.py similarity index 100% rename from scrapers/stash-sqlite.py rename to scrapers/stash-sqlite/stash-sqlite.py diff --git a/scrapers/stash-sqlite.yml b/scrapers/stash-sqlite/stash-sqlite.yml similarity index 100% rename from scrapers/stash-sqlite.yml rename to scrapers/stash-sqlite/stash-sqlite.yml diff --git a/scrapers/torrent.py b/scrapers/torrent/torrent.py similarity index 100% rename from scrapers/torrent.py rename to scrapers/torrent/torrent.py diff --git a/scrapers/torrent.yml b/scrapers/torrent/torrent.yml similarity index 84% rename from scrapers/torrent.yml rename to scrapers/torrent/torrent.yml index 785ce247..ab8f49a7 100644 --- a/scrapers/torrent.yml +++ b/scrapers/torrent/torrent.yml @@ -1,4 +1,6 @@ name: Torrent +# requires: py_common + sceneByFragment: action: script script: diff --git a/scrapers/Algolia_touchmywife.yml b/scrapers/touchmywife/touchmywife.yml similarity index 77% rename from scrapers/Algolia_touchmywife.yml rename to scrapers/touchmywife/touchmywife.yml index bf1ac810..20081577 100644 --- a/scrapers/Algolia_touchmywife.yml +++ b/scrapers/touchmywife/touchmywife.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "TouchMyWife" sceneByURL: - action: script @@ -5,26 +6,26 @@ sceneByURL: - touchmywife.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - touchmywife sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - touchmywife sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - touchmywife - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - touchmywife - validName galleryByURL: @@ -33,7 +34,7 @@ galleryByURL: - touchmywife.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - touchmywife - gallery # Last Updated December 22, 2022 diff --git a/scrapers/vixenNetwork.py b/scrapers/vixenNetwork/vixenNetwork.py similarity index 100% rename from scrapers/vixenNetwork.py rename to scrapers/vixenNetwork/vixenNetwork.py diff --git a/scrapers/vixenNetwork.yml b/scrapers/vixenNetwork/vixenNetwork.yml similarity index 97% rename from scrapers/vixenNetwork.yml rename to scrapers/vixenNetwork/vixenNetwork.yml index bebff994..5515e8b5 100644 --- a/scrapers/vixenNetwork.yml +++ b/scrapers/vixenNetwork/vixenNetwork.yml @@ -1,4 +1,6 @@ name: "Vixen Media Group" +# requires: py_common + sceneByURL: - url: - blacked.com/videos diff --git a/scrapers/Algolia_xEmpire.yml b/scrapers/xEmpire/xEmpire.yml similarity index 83% rename from scrapers/Algolia_xEmpire.yml rename to scrapers/xEmpire/xEmpire.yml index 75f6c400..8ff92c4b 100644 --- a/scrapers/Algolia_xEmpire.yml +++ b/scrapers/xEmpire/xEmpire.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "xEmpire" sceneByURL: - action: script @@ -10,26 +11,26 @@ sceneByURL: - xempire.com/en/video/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - xempire sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - xempire sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - xempire - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - xempire - validName galleryByURL: @@ -43,7 +44,7 @@ galleryByURL: - xempire.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - xempire - gallery # Last Updated December 22, 2022 diff --git a/scrapers/xbvrdb.py b/scrapers/xbvrdb/xbvrdb.py similarity index 100% rename from scrapers/xbvrdb.py rename to scrapers/xbvrdb/xbvrdb.py diff --git a/scrapers/xbvrdb.yml b/scrapers/xbvrdb/xbvrdb.yml similarity index 100% rename from scrapers/xbvrdb.yml rename to scrapers/xbvrdb/xbvrdb.yml diff --git a/scrapers/Algolia_zerotolerancefilms.yml b/scrapers/zerotolerancefilms/zerotolerancefilms.yml similarity index 81% rename from scrapers/Algolia_zerotolerancefilms.yml rename to scrapers/zerotolerancefilms/zerotolerancefilms.yml index 7015c48a..2a4fbac9 100644 --- a/scrapers/Algolia_zerotolerancefilms.yml +++ b/scrapers/zerotolerancefilms/zerotolerancefilms.yml @@ -1,3 +1,4 @@ +# requires: Algolia name: "ZeroToleranceFilms" sceneByURL: - action: script @@ -6,26 +7,26 @@ sceneByURL: - zerotolerancefilms.com/en/video script: - python - - Algolia.py + - ../Algolia/Algolia.py - zerotolerancefilms sceneByFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - zerotolerancefilms sceneByName: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - zerotolerancefilms - searchName sceneByQueryFragment: action: script script: - python - - Algolia.py + - ../Algolia/Algolia.py - zerotolerancefilms - validName movieByURL: @@ -35,7 +36,7 @@ movieByURL: - zerotolerancefilms.com/en/movie script: - python - - Algolia.py + - ../Algolia/Algolia.py - zerotolerancefilms - movie galleryByURL: @@ -44,7 +45,7 @@ galleryByURL: - zerotolerancefilms.com/en/photo/ script: - python - - Algolia.py + - ../Algolia/Algolia.py - zerotolerancefilms - gallery # Last Updated December 22, 2022