feat(indexers): update PTP announce to new format (#1738)

* feat(indexers): update PTP to new format

* fix: update expect line

* feat: use unique key for dl link

* feat: update pattern

* fix: definition tests

* feat: rename var

* feat: add custom vars map

* feat: start mapCustomVars

* Update internal/indexer/definitions/ptp.yaml

Co-authored-by: nuxen <felix.schubert1998@gmail.com>

* feat(indexers): map custom indexer vars

* feat: support upload unix epoch time

* feat(releases): update mapvars

* feat(indexers): remove ptp api init

* feat(indexers): update ptp category mapping

* feat(releases): show announce type in details

* feat(releases): mapvars announcetype

---------

Co-authored-by: nuxen <felix.schubert1998@gmail.com>

internal/domain/indexer.go

@@ -232,6 +232,7 @@ type IndexerIRCParse struct {
 	ForceSizeUnit string                                  `json:"forcesizeunit"`
 	Lines         []IndexerIRCParseLine                   `json:"lines"`
 	Match         IndexerIRCParseMatch                    `json:"match"`
+	Mappings      map[string]map[string]map[string]string `json:"mappings"`
 }
 
 type LineTest struct {
@@ -352,7 +353,31 @@ func (p *IndexerIRCParseMatch) ParseTorrentName(vars map[string]string, rls *Rel
 	return nil
 }
 
+func (p *IndexerIRCParse) MapCustomVariables(vars map[string]string) error {
+	for varsKey, varsKeyMap := range p.Mappings {
+		varsValue, ok := vars[varsKey]
+		if !ok {
+			continue
+		}
+
+		keyValueMap, ok := varsKeyMap[varsValue]
+		if !ok {
+			continue
+		}
+
+		for k, v := range keyValueMap {
+			vars[k] = v
+		}
+	}
+
+	return nil
+}
+
 func (p *IndexerIRCParse) Parse(def *IndexerDefinition, vars map[string]string, rls *Release) error {
+	if err := p.MapCustomVariables(vars); err != nil {
+		return errors.Wrap(err, "could not map custom variables for release")
+	}
+
 	if err := rls.MapVars(def, vars); err != nil {
 		return errors.Wrap(err, "could not map variables for release")
 	}

internal/domain/indexer_test.go

@@ -375,3 +375,107 @@ func TestIRCParserOrpheus_Parse(t *testing.T) {
 		})
 	}
 }
+
+func TestIndexerIRCParse_MapCustomVariables1(t *testing.T) {
+	type fields struct {
+		Type          string
+		ForceSizeUnit string
+		Lines         []IndexerIRCParseLine
+		Match         IndexerIRCParseMatch
+		Mappings      map[string]map[string]map[string]string
+	}
+	type args struct {
+		vars       map[string]string
+		expectVars map[string]string
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "",
+			fields: fields{
+				Mappings: map[string]map[string]map[string]string{
+					"announceType": {
+						"0": map[string]string{
+							"announceType": "NEW",
+						},
+						"1": map[string]string{
+							"announceType": "PROMO",
+						},
+					},
+					"categoryEnum": {
+						"0": map[string]string{
+							"category": "Feature Film",
+						},
+						"1": map[string]string{
+							"category": "Short Film",
+						},
+						"2": map[string]string{
+							"category": "Miniseries",
+						},
+						"3": map[string]string{
+							"category": "Stand-up Comedy",
+						},
+						"4": map[string]string{
+							"category": "Live Performance",
+						},
+						"5": map[string]string{
+							"category": "Movie Collection",
+						},
+					},
+					"freeleechEnum": {
+						"0": map[string]string{
+							"downloadVolumeFactor": "1.0",
+							"uploadVolumeFactor":   "1.0",
+						},
+						"1": map[string]string{
+							"downloadVolumeFactor": "0",
+							"uploadVolumeFactor":   "1.0",
+						},
+						"2": map[string]string{
+							"downloadVolumeFactor": "0.5",
+							"uploadVolumeFactor":   "1.0",
+						},
+						"3": map[string]string{
+							"downloadVolumeFactor": "0",
+							"uploadVolumeFactor":   "0",
+						},
+					},
+				},
+			},
+			args: args{
+				vars: map[string]string{
+					"announceType":  "1",
+					"categoryEnum":  "0",
+					"freeleechEnum": "1",
+				},
+				expectVars: map[string]string{
+					"announceType":         "PROMO",
+					"category":             "Feature Film",
+					"categoryEnum":         "0",
+					"freeleechEnum":        "1",
+					"downloadVolumeFactor": "0",
+					"uploadVolumeFactor":   "1.0",
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &IndexerIRCParse{
+				Type:          tt.fields.Type,
+				ForceSizeUnit: tt.fields.ForceSizeUnit,
+				Lines:         tt.fields.Lines,
+				Match:         tt.fields.Match,
+				Mappings:      tt.fields.Mappings,
+			}
+			err := p.MapCustomVariables(tt.args.vars)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.args.expectVars, tt.args.vars)
+		})
+	}
+}

internal/domain/release.go

@@ -733,7 +733,7 @@ func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) erro
 		r.Category = category
 	}
 
-	if announceType, err := getStringMapValue(varMap, "announceTypeEnum"); err == nil {
+	if announceType, err := getStringMapValue(varMap, "announceType"); err == nil {
 		annType, parseErr := ParseAnnounceType(announceType)
 		if parseErr == nil {
 			r.AnnounceType = annType
@@ -764,11 +764,8 @@ func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) erro
 		freeleechPercent = strings.Replace(freeleechPercent, "%", "", -1)
 		freeleechPercent = strings.Trim(freeleechPercent, " ")
 
-		freeleechPercentInt, err := strconv.Atoi(freeleechPercent)
+		freeleechPercentInt, parseErr := strconv.Atoi(freeleechPercent)
-		if err != nil {
+		if parseErr == nil {
-			//log.Debug().Msgf("bad freeleechPercent var: %v", year)
-		}
-
 			if freeleechPercentInt > 0 {
 				r.Freeleech = true
 				r.FreeleechPercent = freeleechPercentInt
@@ -787,6 +784,52 @@ func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) erro
 				}
 			}
 		}
+	}
+
+	//if downloadVolumeFactor, err := getStringMapValue(varMap, "downloadVolumeFactor"); err == nil {
+	if downloadVolumeFactor, ok := varMap["downloadVolumeFactor"]; ok {
+		// special handling for BHD to map their freeleech into percent
+		//if def.Identifier == "beyondhd" {
+		//	if freeleechPercent == "Capped FL" {
+		//		freeleechPercent = "100%"
+		//	} else if strings.Contains(freeleechPercent, "% FL") {
+		//		freeleechPercent = strings.Replace(freeleechPercent, " FL", "", -1)
+		//	}
+		//}
+
+		//r.downloadVolumeFactor = downloadVolumeFactor
+
+		value, parseErr := strconv.ParseInt(downloadVolumeFactor, 10, 64)
+		if parseErr == nil {
+			percentage := value * 100
+			r.FreeleechPercent = int(percentage)
+		}
+
+		r.Freeleech = true
+	}
+
+	//if uploadVolumeFactor, err := getStringMapValue(varMap, "uploadVolumeFactor"); err == nil {
+	//	// special handling for BHD to map their freeleech into percent
+	//	//if def.Identifier == "beyondhd" {
+	//	//	if freeleechPercent == "Capped FL" {
+	//	//		freeleechPercent = "100%"
+	//	//	} else if strings.Contains(freeleechPercent, "% FL") {
+	//	//		freeleechPercent = strings.Replace(freeleechPercent, " FL", "", -1)
+	//	//	}
+	//	//}
+	//
+	//	r.uploadVolumeFactor = uploadVolumeFactor
+	//
+	//	//freeleechPercentInt, err := strconv.Atoi(freeleechPercent)
+	//	//if err != nil {
+	//	//	//log.Debug().Msgf("bad freeleechPercent var: %v", year)
+	//	//}
+	//	//
+	//	//if freeleechPercentInt > 0 {
+	//	//	r.Freeleech = true
+	//	//	r.FreeleechPercent = freeleechPercentInt
+	//	//}
+	//}
 
 	if uploader, err := getStringMapValue(varMap, "uploader"); err == nil {
 		r.Uploader = uploader
@@ -801,12 +844,18 @@ func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) erro
 			torrentSize = fmt.Sprintf("%s %s", torrentSize, def.IRC.Parse.ForceSizeUnit)
 		}
 
-		size, err := humanize.ParseBytes(torrentSize)
+		size, parseErr := humanize.ParseBytes(torrentSize)
-		if err != nil {
+		if parseErr == nil {
-			// log could not parse into bytes
-		}
 			r.Size = size
 		}
+	}
+
+	if torrentSizeBytes, err := getStringMapValue(varMap, "torrentSizeBytes"); err == nil {
+		size, parseErr := strconv.ParseUint(torrentSizeBytes, 10, 64)
+		if parseErr == nil {
+			r.Size = size
+		}
+	}
 
 	if scene, err := getStringMapValue(varMap, "scene"); err == nil {
 		if StringEqualFoldMulti(scene, "true", "yes", "1") {
@@ -826,14 +875,14 @@ func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) erro
 	}
 
 	if yearVal, err := getStringMapValue(varMap, "year"); err == nil {
-		year, err := strconv.Atoi(yearVal)
+		year, parseErr := strconv.Atoi(yearVal)
-		if err != nil {
+		if parseErr == nil {
-			//log.Debug().Msgf("bad year var: %v", year)
-		}
 			r.Year = year
 		}
+	}
 
 	if tags, err := getStringMapValue(varMap, "tags"); err == nil {
+		if tags != "" && tags != "*" {
 			tagsArr := []string{}
 			s := strings.Split(tags, ",")
 			for _, t := range s {
@@ -841,10 +890,13 @@ func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) erro
 			}
 			r.Tags = tagsArr
 		}
+	}
 
 	if title, err := getStringMapValue(varMap, "title"); err == nil {
+		if title != "" && title != "*" {
 			r.Title = title
 		}
+	}
 
 	// handle releaseTags. Most of them are redundant but some are useful
 	if releaseTags, err := getStringMapValue(varMap, "releaseTags"); err == nil {
@@ -864,6 +916,10 @@ func (r *Release) MapVars(def *IndexerDefinition, varMap map[string]string) erro
 		r.Episode = episode
 	}
 
+	//if metaImdb, err := getStringMapValue(varMap, "imdb"); err == nil {
+	//	r.MetaIMDB = metaImdb
+	//}
+
 	return nil
 }
 

internal/indexer/api.go

@@ -13,7 +13,6 @@ import (
 	"github.com/autobrr/autobrr/pkg/errors"
 	"github.com/autobrr/autobrr/pkg/ggn"
 	"github.com/autobrr/autobrr/pkg/ops"
-	"github.com/autobrr/autobrr/pkg/ptp"
 	"github.com/autobrr/autobrr/pkg/red"
 
 	"github.com/rs/zerolog"
@@ -94,18 +93,6 @@ func (s *apiService) AddClient(indexer string, settings map[string]string) error
 		}
 		s.apiClients[indexer] = btn.NewClient(key)
 
-	case "ptp":
-		user, ok := settings["api_user"]
-		if !ok || user == "" {
-			return errors.New("api.Service.AddClient: could not initialize ptp client: missing var 'api_user'")
-		}
-
-		key, ok := settings["api_key"]
-		if !ok || key == "" {
-			return errors.New("api.Service.AddClient: could not initialize ptp client: missing var 'api_key'")
-		}
-		s.apiClients[indexer] = ptp.NewClient(user, key)
-
 	case "ggn":
 		key, ok := settings["api_key"]
 		if !ok || key == "" {
@@ -156,16 +143,6 @@ func (s *apiService) getClientForTest(req domain.IndexerTestApiRequest) (apiClie
 		}
 		return btn.NewClient(req.ApiKey), nil
 
-	case "ptp":
-		if req.ApiUser == "" {
-			return nil, errors.New("api.Service.AddClient: could not initialize ptp client: missing var 'api_user'")
-		}
-
-		if req.ApiKey == "" {
-			return nil, errors.New("api.Service.AddClient: could not initialize ptp client: missing var 'api_key'")
-		}
-		return ptp.NewClient(req.ApiUser, req.ApiKey), nil
-
 	case "ggn":
 		if req.ApiKey == "" {
 			return nil, errors.New("api.Service.AddClient: could not initialize ggn client: missing var 'api_key'")

internal/indexer/definitions/ptp.yaml

@@ -11,7 +11,6 @@ protocol: torrent
 supports:
   - irc
   - rss
-  - api
 # source: gazelle
 settings:
   - name: authkey
@@ -26,34 +25,6 @@ settings:
     label: Torrent pass
     help: Right click DL on a torrent and get the torrent_pass.
 
-  - name: api_user
-    type: secret
-    required: true
-    label: API User
-    help: Edit profile -> Security -> Generate new api keys
-
-  - name: api_key
-    type: secret
-    required: true
-    label: API Key
-    help: Edit profile -> Security -> Generate new api keys
-
-# api:
-#   url: https://passthepopcorn.me/
-#   type: json
-#   limits:
-#     max: 60
-#     per: minute
-#   settings:
-#     - name: api_user
-#       type: secret
-#       label: API User
-#       help: Edit profile -> Security -> Generate new api keys
-#     - name: api_key
-#       type: secret
-#       label: API Key
-#       help: Edit profile -> Security -> Generate new api keys
-
 irc:
   network: PassThePopcorn
   server: irc.passthepopcorn.me
@@ -93,31 +64,131 @@ irc:
     type: single
     lines:
       - tests:
-        - line: That Movie [1972] by Some Director | x264 / Blu-ray / MKV / 1080p | 204371 | 964303 | That.Movie.1972.1080p.BluRay.FLAC.x264-GROUP | comedy, drama, romance
+        - line: 0:2:0:0:357805:1332009:9k6p:tt24249072:1899875175:1728081334|H.264/MKV/WEB/720p||2023|Last Straw|Last.Straw.2023.REPACK.720p.AMZN.WEB-DL.DDP5.1.H.264-FLUX|thriller,horror
           expect:
-            year: "1972"
+            announceTypeEnum: 0
-            releaseTags: x264 / Blu-ray / MKV / 1080p
+            categoryEnum: 2
-            freeleech: ""
+            originEnum: 0
-            torrentId: "964303"
+            freeleechEnum: 0
-            torrentName: That.Movie.1972.1080p.BluRay.FLAC.x264-GROUP
+            groupId: "357805"
-            tags: comedy, drama, romance
+            torrentId: "1332009"
-        - line: That Other Movie [1972] | x264 / Blu-ray / MKV / 1080p / Freeleech! | 204371 | 964303 | That.Other.Movie.1972.1080p.BluRay.FLAC.x264-GROUP | comedy, drama, romance
+            key: "9k6p"
+            imdb: "tt24249072"
+            torrentSizeBytes: "1899875175"
+            uploadedDate: "1728081334"
+            releaseTags: H.264/MKV/WEB/720p
+            editionTags: ""
+            year: "2023"
+            title: "Last Straw"
+            torrentName: Last.Straw.2023.REPACK.720p.AMZN.WEB-DL.DDP5.1.H.264-FLUX
+            tags: thriller,horror
+        - line: 2:1:0:1:357805:1332009:9k6p:tt24249072:1899875175:1728081334|H.264/MKV/WEB/720p||2023|Last Straw|Last.Straw.2023.REPACK.720p.AMZN.WEB-DL.DDP5.1.H.264-FLUX|thriller,horror
           expect:
-            year: "1972"
+            announceTypeEnum: 2
-            releaseTags: x264 / Blu-ray / MKV / 1080p / Freeleech!
+            categoryEnum: 1
-            freeleech: Freeleech
+            originEnum: 0
-            torrentId: "964303"
+            freeleechEnum: 1
-            torrentName: That.Other.Movie.1972.1080p.BluRay.FLAC.x264-GROUP
+            groupId: "357805"
-            tags: comedy, drama, romance
+            torrentId: "1332009"
-        pattern: '.* \[(.*)\] (?:by .*)?\| (.*?(?: \/ (Freeleech)!)?) \| .* \| (.*) \| (.*) \| (.*)'
+            key: "9k6p"
+            imdb: "tt24249072"
+            torrentSizeBytes: "1899875175"
+            uploadedDate: "1728081334"
+            releaseTags: H.264/MKV/WEB/720p
+            editionTags: ""
+            year: "2023"
+            title: "Last Straw"
+            torrentName: Last.Straw.2023.REPACK.720p.AMZN.WEB-DL.DDP5.1.H.264-FLUX
+            tags: thriller,horror
+        - line: 1:1:2:1:21108:1332339:ncfe:tt0107426:63996413804:1728081334|BD66/m2ts/Blu-ray/2160p|Dolby Atmos/Dolby Vision|1993|Little Buddha|Little.Buddha.1993.2160p.FRA.UHD.Blu-ray.DV.HDR.HEVC.DTS-HD.MA.5.1|drama,italian
+          expect:
+            announceTypeEnum: 1
+            categoryEnum: 1
+            originEnum: 2
+            freeleechEnum: 1
+            groupId: "21108"
+            torrentId: "1332339"
+            key: "ncfe"
+            imdb: "tt0107426"
+            torrentSizeBytes: "63996413804"
+            uploadedDate: "1728081334"
+            releaseTags: BD66/m2ts/Blu-ray/2160p
+            editionTags: "Dolby Atmos/Dolby Vision"
+            year: "1993"
+            title: "Little Buddha"
+            torrentName: Little.Buddha.1993.2160p.FRA.UHD.Blu-ray.DV.HDR.HEVC.DTS-HD.MA.5.1
+            tags: drama,italian
+        # <announceType>:<category>:<origin>:<freeleech_type>:<gid>:<tid>:<key>:tt<imdbid>:<sizebytes>|<format>/<container>/<source>/<resolution>|<edition_tags>|<year>|<title>|<releasename>|<taglist>
+        pattern: '(\d):(\d):(\d):(\d):(\d+):(\d+):(\w+):(tt\d+)?:(\d+):(\d+)\|(.*)\|(.*)\|(\d+)\|(.+)\|(.+)\|(.+)'
         vars:
-          - year
+          - announceTypeEnum
-          - releaseTags
+          - categoryEnum
-          - freeleech
+          - originEnum
+          - freeleechEnum
+          - groupId
           - torrentId
+          - key
+          - imdb
+          - torrentSizeBytes
+          - uploadedDate
+          - releaseTags
+          - editionTags
+          - year
+          - title
           - torrentName
           - tags
 
+    mappings:
+      announceTypeEnum:
+        "0":
+          announceType: NEW
+        "1":
+          announceType: CHECKED
+        "2":
+          announceType: PROMO # X-leeched (could be neutral/half/free)
+        "3":
+          announceType: PROMO_GP # marked golden popcorn
+        "4":
+          announceType: RESURRECTED # reseeded/revived from dead
+
+      # Category IDs are Zero+1 indexed from:
+      # ( 'Feature Film', 'Short Film', 'Miniseries', 'Stand-up Comedy', 'Live Performance', 'Movie Collection' )
+      categoryEnum:
+        "1":
+          category: "Feature Film"
+        "2":
+          category: "Short Film"
+        "3":
+          category: "Miniseries"
+        "4":
+          category: "Stand-up Comedy"
+        "5":
+          category: "Live Performance"
+        "6":
+          category: "Movie Collection"
+
+      originEnum:
+        "0":
+          origin: "P2P"
+        "1":
+          origin: "SCENE"
+        "2":
+          origin: "INTERNAL" # PERSONAL RIP
+
+      freeleechEnum:
+        "0": # Normal
+          downloadVolumeFactor: 1
+          uploadVolumeFactor: 1
+        "1": # Freeleech
+          downloadVolumeFactor: 0
+          uploadVolumeFactor: 1
+        "2": # Halfleech
+          downloadVolumeFactor: 0.5
+          uploadVolumeFactor: 1
+        "3": # Neutral
+          downloadVolumeFactor: 0
+          uploadVolumeFactor: 0
+
     match:
       infourl: "/torrents.php?torrentid={{ .torrentId }}"
-      torrenturl: "/torrents.php?action=download&id={{ .torrentId }}&authkey={{ .authkey }}&torrent_pass={{ .torrent_pass }}"
+      torrenturl: "/torrents.php?action=download&id={{ .torrentId }}&authkey={{ .authkey }}&torrent_pass={{ .torrent_pass }}&key={{ .key }}"

web/src/components/data-table/Cells.tsx

@@ -63,6 +63,7 @@ export const LinksCell = (props: CellContext<Release, unknown>) => {
             <CellLine title="Indexer">{props.row.original.indexer.identifier}</CellLine>
             <CellLine title="Protocol">{props.row.original.protocol}</CellLine>
             <CellLine title="Implementation">{props.row.original.implementation}</CellLine>
+            <CellLine title="Announce Type">{props.row.original.announce_type}</CellLine>
             <CellLine title="Category">{props.row.original.category}</CellLine>
             <CellLine title="Uploader">{props.row.original.uploader}</CellLine>
             <CellLine title="Size">{humanFileSize(props.row.original.size)}</CellLine>