Merge branch 'dev-0.1.1' into 'master'

0.1.1 Reworked Spotify, Cache values, Update package name

See merge request goscrobble/goscrobble-api!3

.gitlab-ci.yml

@@ -2,10 +2,10 @@ stages:
  - build
 
 variables:
-  VERSION: 0.1.0
+  VERSION: 0.1.1
 
 build-go:
-  image: golang:1.16
+  image: golang:1.17
   stage: build
   only:
     - master

cmd/go-scrobble/main.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/joho/godotenv"
-	"gitlab.com/idanoo/go-scrobble/internal/goscrobble"
+	"gitlab.com/goscrobble/goscrobble-api/internal/goscrobble"
 )
 
 func init() {

docs/changelog.md

@@ -1,3 +1,9 @@
+# 0.1.1
+- Cached all config values
+- Updated spotify sdk package to v2
+- Changed package name to gitlab.com/goscrobble/goscrobble-api to match repo
+- Updated duplicate scrobble logic to never log the same song twice
+
 # 0.1.0
 - Split frontend/backend code into separate repos (https://gitlab.com/goscrobble/goscrobble-web)
 - Added new ENV VARS to support unique configurations: DATA_DIRECTORY, FRONTEND_DIRECTORY, API_DOCS_DIRECTORY

go.mod

@@ -1,33 +1,46 @@
-module gitlab.com/idanoo/go-scrobble
+module gitlab.com/goscrobble/goscrobble-api
 
-go 1.16
+go 1.17
 
 require (
-	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
-	github.com/containerd/containerd v1.4.1 // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
-	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/docker v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
 	github.com/go-redis/redis/v8 v8.8.0
 	github.com/go-sql-driver/mysql v1.5.0
-	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang-migrate/migrate v3.5.4+incompatible
 	github.com/google/uuid v1.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/joho/godotenv v1.3.0
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rs/cors v1.7.0
-	github.com/sendgrid/rest v2.6.3+incompatible // indirect
 	github.com/sendgrid/sendgrid-go v3.8.0+incompatible
-	github.com/sirupsen/logrus v1.7.0 // indirect
+	github.com/zmb3/spotify/v2 v2.0.1
-	github.com/zmb3/spotify/v2 v2.0.1 // indirect
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/oauth2 v0.0.0-20210810183815-faf39c7919d5
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+)
+
+require (
+	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/containerd/containerd v1.4.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/gogo/protobuf v1.3.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sendgrid/rest v2.6.3+incompatible // indirect
+	github.com/sirupsen/logrus v1.7.0 // indirect
+	go.opentelemetry.io/otel v0.19.0 // indirect
+	go.opentelemetry.io/otel/metric v0.19.0 // indirect
+	go.opentelemetry.io/otel/trace v0.19.0 // indirect
+	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20201030142918-24207fddd1c3 // indirect
 	google.golang.org/grpc v1.33.1 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
 )

internal/goscrobble/config.go

@@ -59,19 +59,36 @@ func updateConfigValue(key string, value string) error {
 		return errors.New("Failed to update config value.")
 	}
 
+	// Set cached config
+	redisKey := "config:" + key
+	setRedisVal(redisKey, value)
+
 	return nil
 }
 
 func getConfigValue(key string) (string, error) {
 	var value string
 
-	err := db.QueryRow("SELECT `value` FROM `config` "+
+	// Check if cached first
-		"WHERE `key` = ?",
+	redisKey := "config:" + key
-		key).Scan(&value)
 
-	if err == sql.ErrNoRows {
+	// TODO: Handle unset vals in DB to prevent excess calls if not using spotify/etc.
-		return value, errors.New("Config key doesn't exist")
+	configKey := getRedisVal(redisKey)
+	if configKey == "" {
+		err := db.QueryRow("SELECT `value` FROM `config` "+
+			"WHERE `key` = ?",
+			key).Scan(&value)
+
+		if err == sql.ErrNoRows {
+			return value, errors.New("Config key doesn't exist")
+		}
+
+		if value != "" {
+			setRedisVal(redisKey, value)
+		}
+
+		return value, nil
 	}
 
-	return value, nil
+	return configKey, nil
 }

internal/goscrobble/ingress_jellyfin.go

@@ -47,10 +47,10 @@ func ParseJellyfinInput(userUUID string, jf JellyfinRequest, ip net.IP, tx *sql.
 	// Debugging
 	// fmt.Printf("%+v", jf)
 
-	// Prevents scrobbling same song twice!
+	// Custom cache key - never log the same song twice in a row for now... (:
-	cacheKey := jf.UserID + ":" + jf.Name + ":" + jf.Artist + ":" + jf.Album + ":" + jf.ServerID
+	lastPlayedTitle := getUserLastPlayed(userUUID)
-	redisKey := getMd5(cacheKey + userUUID)
+	if lastPlayedTitle == jf.Name+":"+jf.Album {
-	if getRedisKeyExists(redisKey) {
+		// If it matches last played song, skip it
 		return nil
 	}
 
@@ -103,8 +103,7 @@ func ParseJellyfinInput(userUUID string, jf JellyfinRequest, ip net.IP, tx *sql.
 	}
 
 	// Add cache key!
-	ttl := time.Duration(timestampToSeconds(jf.RunTime)) * time.Second
+	setUserLastPlayed(userUUID, jf.Name+":"+jf.Album)
-	setRedisValTtl(redisKey, "1", ttl)
 
 	return nil
 }

internal/goscrobble/ingress_multiscrobbler.go

@@ -3,7 +3,6 @@ package goscrobble
 import (
 	"database/sql"
 	"errors"
-	"fmt"
 	"log"
 	"net"
 	"time"
@@ -19,10 +18,10 @@ type MultiScrobblerRequest struct {
 
 // ParseMultiScrobblerInput - Transform API data
 func ParseMultiScrobblerInput(userUUID string, data MultiScrobblerRequest, ip net.IP, tx *sql.Tx) error {
-	// Cache key
+	// Custom cache key - never log the same song twice in a row for now... (:
-	json := fmt.Sprintf("%s:%s:%s", data.PlayedAt, data.Track, userUUID)
+	lastPlayedTitle := getUserLastPlayed(userUUID)
-	redisKey := getMd5(json)
+	if lastPlayedTitle == data.Track+":"+data.Album {
-	if getRedisKeyExists(redisKey) {
+		// If it matches last played song, skip it
 		return nil
 	}
 
@@ -61,8 +60,7 @@ func ParseMultiScrobblerInput(userUUID string, data MultiScrobblerRequest, ip ne
 		return errors.New("Failed to map track")
 	}
 
-	ttl := time.Duration(24) * time.Hour
+	setUserLastPlayed(userUUID, data.Track+":"+data.Album)
-	setRedisValTtl(redisKey, "1", ttl)
 
 	return nil
 }

internal/goscrobble/ingress_navidrome.go

@@ -138,10 +138,10 @@ func validateNavidromeConnection(url string, username string, hash string, salt
 
 // ParseNavidromeInput - Transform API data
 func ParseNavidromeInput(userUUID string, data NavidromeNowPlaying, ip net.IP, tx *sql.Tx) error {
-	// Cache key
+	// Custom cache key - never log the same song twice in a row for now... (:
-	json := fmt.Sprintf("%s:%s:%s", data.ID, data.Parent, userUUID)
+	lastPlayedTitle := getUserLastPlayed(userUUID)
-	redisKey := getMd5(json)
+	if lastPlayedTitle == data.Title+":"+data.Album {
-	if getRedisKeyExists(redisKey) {
+		// If it matches last played song, skip it
 		return nil
 	}
 
@@ -177,9 +177,7 @@ func ParseNavidromeInput(userUUID string, data NavidromeNowPlaying, ip net.IP, t
 		return errors.New("Failed to map track")
 	}
 
-	// Todo: Find a better way to check dupes
+	setUserLastPlayed(userUUID, data.Title+":"+data.Album)
-	ttl := time.Duration(data.Duration*2) * time.Second
-	setRedisValTtl(redisKey, "1", ttl)
 
 	return nil
 }

internal/goscrobble/ingress_spotify.go

@@ -64,11 +64,15 @@ func getSpotifyAuthHandler() spotifyauth.Authenticator {
 
 func connectSpotifyResponse(r *http.Request) error {
 	urlParams := r.URL.Query()
-	userUuid := urlParams["state"][0]
+	userUUID := urlParams["state"][0]
+
+	_, err := getUserByUUID(userUUID)
+	if err != nil {
+		return err
+	}
 
-	// TODO: Add validation user exists here
 	auth := getSpotifyAuthHandler()
-	token, err := auth.Token(r.Context(), userUuid, r)
+	token, err := auth.Token(r.Context(), userUUID, r)
 	if err != nil {
 		return err
 	}
@@ -79,7 +83,7 @@ func connectSpotifyResponse(r *http.Request) error {
 
 	// Lets pull in last 30 minutes
 	time := time.Now().UTC().Add(-(time.Duration(30) * time.Minute))
-	err = insertOauthToken(userUuid, "spotify", token.AccessToken, token.RefreshToken, token.Expiry, spotifyUser.DisplayName, time, "")
+	err = insertOauthToken(userUUID, "spotify", token.AccessToken, token.RefreshToken, token.Expiry, spotifyUser.DisplayName, time, "")
 	if err != nil {
 		return err
 	}

internal/goscrobble/server.go

@@ -133,24 +133,21 @@ func HandleRequests(port string) {
 // API ENDPOINT HANDLING
 // handleRegister - Does as it says!
 func handleRegister(w http.ResponseWriter, r *http.Request) {
-	cachedRegistrationEnabled := getRedisVal("REGISTRATION_ENABLED")
+	registrationEnabled, err := getConfigValue("REGISTRATION_ENABLED")
-	if cachedRegistrationEnabled == "" {
+	if err != nil {
-		registrationEnabled, err := getConfigValue("REGISTRATION_ENABLED")
+		log.Printf("%+v", err)
-		if err != nil {
+		throwOkError(w, "Registration is currently disabled")
-			throwOkError(w, "Error checking if registration is enabled")
+		return
-		}
-		setRedisVal("REGISTRATION_ENABLED", registrationEnabled)
-		cachedRegistrationEnabled = registrationEnabled
 	}
 
-	if cachedRegistrationEnabled == "0" {
+	if registrationEnabled == "0" {
 		throwOkError(w, "Registration is currently disabled")
 		return
 	}
 
 	regReq := RequestRequest{}
 	decoder := json.NewDecoder(r.Body)
-	err := decoder.Decode(&regReq)
+	err = decoder.Decode(&regReq)
 	if err != nil {
 		throwBadReq(w, err.Error())
 		return
@@ -778,19 +775,15 @@ func deleteNavidrome(w http.ResponseWriter, r *http.Request, claims CustomClaims
 }
 
 func getServerInfo(w http.ResponseWriter, r *http.Request) {
-	cachedRegistrationEnabled := getRedisVal("REGISTRATION_ENABLED")
+	registrationEnabled, err := getConfigValue("REGISTRATION_ENABLED")
-	if cachedRegistrationEnabled == "" {
+	if err != nil {
-		registrationEnabled, err := getConfigValue("REGISTRATION_ENABLED")
+		log.Printf("%+v", err)
-		if err != nil {
+		registrationEnabled = "0"
-			throwOkError(w, "Error fetching serverinfo")
-		}
-		setRedisVal("REGISTRATION_ENABLED", registrationEnabled)
-		cachedRegistrationEnabled = registrationEnabled
 	}
 
 	info := ServerInfo{
-		Version:             "0.1.0",
+		Version:             "0.1.1",
-		RegistrationEnabled: cachedRegistrationEnabled,
+		RegistrationEnabled: registrationEnabled,
 	}
 
 	js, _ := json.Marshal(&info)

internal/goscrobble/track.go

@@ -317,8 +317,6 @@ func getTopUsersForTrackUUID(trackUUID string, limit int, page int) (TopUserTrac
 	response := TopUserTrackResponse{}
 	var count int
 
-	// Yeah this isn't great. But for now.. it works! Cache later
-	// TODO:  This is counting total scrobbles, not unique users
 	total, err := getDbCount(
 		"SELECT COUNT(*) FROM `scrobbles` WHERE `track` = UUID_TO_BIN(?, true) GROUP BY `track`, `user`", trackUUID)
 

internal/goscrobble/user.go

@@ -201,7 +201,7 @@ func getUserByUUID(uuid string) (User, error) {
 		uuid).Scan(&user.UUID, &user.CreatedAt, &user.CreatedIp, &user.ModifiedAt, &user.ModifiedIP, &user.Username, &user.Email, &user.Password, &user.Verified, &user.Admin, &user.Mod, &user.Timezone, &user.Token)
 
 	if err == sql.ErrNoRows {
-		return user, errors.New("Invalid JWT Token")
+		return user, errors.New("Invalid user UUID")
 	}
 
 	return user, nil
@@ -378,3 +378,11 @@ func getAllNavidromeUsers() ([]User, error) {
 
 	return users, nil
 }
+
+func getUserLastPlayed(userUUID string) string {
+	return getRedisVal("lastPlayed:" + userUUID)
+}
+
+func setUserLastPlayed(userUUID string, val string) {
+	setRedisVal("lastPlayed:"+userUUID, val)
+}