diff --git a/.env.development b/.env.development new file mode 100644 index 00000000..4a369c9b --- /dev/null +++ b/.env.development @@ -0,0 +1,29 @@ +MYSQL_HOST=mysql +MYSQL_USER=root +MYSQL_PASS=supersecretdatabasepassword1 +MYSQL_DB=goscrobble + +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_DB=4 +REDIS_PREFIX="gs:" +REDIS_AUTH="" + +JWT_SECRET=abcdefg +JWT_EXPIRY=604800 +REFRESH_EXPIRY=604800 + +REVERSE_PROXIES= +PORT=42069 + +SENDGRID_API_KEY= +MAIL_FROM_ADDRESS= +MAIL_FROM_NAME= + +DEV_MODE=true + +GOSCROBBLE_DOMAIN="http://127.0.0.1" + +DATA_DIRECTORY="/app/data" +FRONTEND_DIRECTORY="/app" +API_DOCS_DIRECTORY="/app/docs/api/build" \ No newline at end of file diff --git a/.env.production b/.env.production new file mode 100644 index 00000000..d8f029ff --- /dev/null +++ b/.env.production @@ -0,0 +1,29 @@ +MYSQL_HOST= +MYSQL_USER= +MYSQL_PASS= +MYSQL_DB= + +REDIS_HOST=127.0.0.1 +REDIS_PORT=6379 +REDIS_DB= +REDIS_PREFIX="gs:" +REDIS_AUTH="" + +JWT_SECRET= +JWT_EXPIRY=1800 +REFRESH_EXPIRY=604800 + +REVERSE_PROXIES=127.0.0.1 +PORT=42069 + +SENDGRID_API_KEY= +MAIL_FROM_ADDRESS= +MAIL_FROM_NAME= + +DEV_MODE=false + +GOSCROBBLE_DOMAIN="" + +DATA_DIRECTORY="/var/www/goscrobble-data" +FRONTEND_DIRECTORY="/var/www/goscrobble-web" +API_DOCS_DIRECTORY="/var/www/goscrobble-api/docs/api/build" diff --git a/.gitignore b/.gitignore index 88083e5e..86717f57 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,8 @@ *.so *.dylib .env -web/.env.production -web/.env.development -web/img/* +/data # Test binary, built with `go test -c` *.test diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15939cef..e092f188 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ stages: - build variables: - VERSION: 0.1.1 + VERSION: 0.1.2 build-go: image: golang:1.17 @@ -10,7 +10,7 @@ build-go: only: - master script: - - go build -o goscrobble cmd/go-scrobble/*.go + - go build -o goscrobble cmd/goscrobble/*.go artifacts: expire_in: 1 week paths: diff --git a/cmd/go-scrobble/main.go b/cmd/goscrobble/main.go similarity index 100% rename from cmd/go-scrobble/main.go rename to cmd/goscrobble/main.go diff --git a/data/img/placeholder.jpg b/data/img/placeholder.jpg new file mode 100644 index 00000000..ff8ff816 Binary files /dev/null and b/data/img/placeholder.jpg differ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..47b1c261 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,44 @@ +version: "3.9" + +services: + frontend: + image: node:16 + volumes: + - ../goscrobble-web:/app + restart: always + ports: + - "127.0.0.1:3000:3000" + environment: + - REACT_APP_API_URL=http://127.0.0.1:42069 + command: bash -c "cd /app && npm install && yarn start" + + backend: + image: golang:1.17 + volumes: + - ./:/app + ports: + - "127.0.0.1:42069:42069" + restart: always + command: bash -c "sleep 5 && cd /app && go mod tidy && go run cmd/goscrobble/*.go" + + mysql: + image: mysql:8.0.27 + command: --default-authentication-plugin=mysql_native_password --init-file /app/migrations/0_create_db.sql --sql_mode= + restart: always + cap_add: + - SYS_NICE + volumes: + - database-data:/var/lib/mysql + - ./migrations/0_create_db.sql:/app/migrations/0_create_db.sql + ports: + - "127.0.0.1:3306:3306" + environment: + - MYSQL_ROOT_PASSWORD=supersecretdatabasepassword1 + + redis: + image: redis:6.2 + ports: + - "127.0.0.1:6379:6379" + +volumes: + database-data: \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md index 2f798924..b6129e35 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,3 +1,8 @@ +# 0.1.2 +- Add docker-compose file for local dev +- Implemented top listeners for artist/album endpoints to match track +- Add recent endpoint + # 0.1.1 - Cached all config values - Updated spotify sdk package to v2 diff --git a/docs/removing_bad_data.md b/docs/removing_bad_data.md index b8a86409..6e1ebcc8 100644 --- a/docs/removing_bad_data.md +++ b/docs/removing_bad_data.md @@ -11,3 +11,32 @@ This is by no means recommended.. But during testing I somehow scrobbled movies. DELETE tracks FROM tracks LEFT JOIN track_artist ON track_artist.track = tracks.uuid WHERE track_artist.track IS NULL; DELETE scrobbles FROM scrobbles LEFT JOIN tracks ON tracks.uuid = scrobbles.track WHERE tracks.uuid is null; SET FOREIGN_KEY_CHECKS=1; + + + +Removing duplicates (based on same song played in same hour) + + -- backup stuff first + DROP TABLE BACKUP_scrobbles; + CREATE TABLE BACKUP_scrobbles (primary key (uuid)) as select * from scrobbles; + + SELECT BIN_TO_UUID(`user`, true), scrobbles.*, count(*) FROM scrobbles + -- WHERE `user`= UUID_TO_BIN('', true) + GROUP BY track, HOUR(created_at) + HAVING count(*) > 1 + ORDER BY COUNT(*) DESC; + + -- will only delete one set of dupes at a time, run until 0 updated rows + DELETE scrobbles + FROM scrobbles + WHERE uuid IN ( + SELECT uuid FROM ( + SELECT `uuid` FROM scrobbles + WHERE `user`= UUID_TO_BIN('', true) + GROUP BY track, HOUR(created_at) + HAVING count(*) > 1 + ) x + ); + + + diff --git a/internal/goscrobble/album.go b/internal/goscrobble/album.go index ab93dfb9..a0340380 100644 --- a/internal/goscrobble/album.go +++ b/internal/goscrobble/album.go @@ -128,3 +128,58 @@ func getAlbumByUUID(uuid string) (Album, error) { return album, nil } + +// getTopUsersForAlbumUUID - Returns list of top users for a track +func getTopUsersForAlbumUUID(albumUUID string, limit int, page int) (TopUserResponse, error) { + response := TopUserResponse{} + var count int + + total, err := getDbCount( + "SELECT COUNT(*) FROM `scrobbles` "+ + "JOIN `track_album` ON `track_album`.`track` = `scrobbles`.`track` "+ + "WHERE `track_album`.`album` = UUID_TO_BIN(?, true);", albumUUID) + + if err != nil { + log.Printf("Failed to fetch scrobble count: %+v", err) + return response, errors.New("Failed to fetch combined scrobbles") + } + + rows, err := db.Query( + "SELECT BIN_TO_UUID(`scrobbles`.`user`, true), `users`.`username`, COUNT(*) "+ + "FROM `scrobbles` "+ + "JOIN `users` ON `scrobbles`.`user` = `users`.`uuid` "+ + "JOIN `track_album` ON `track_album`.`track` = `scrobbles`.`track` "+ + "WHERE `track_album`.`album` = UUID_TO_BIN(?, true) "+ + "GROUP BY `scrobbles`.`user` "+ + "ORDER BY COUNT(*) DESC LIMIT ?", + albumUUID, limit) + + if err != nil { + log.Printf("Failed to fetch scrobbles: %+v", err) + return response, errors.New("Failed to fetch combined scrobbles") + } + defer rows.Close() + + for rows.Next() { + item := TopUserResponseItem{} + err := rows.Scan(&item.UserUUID, &item.UserName, &item.Count) + if err != nil { + log.Printf("Failed to fetch scrobbles: %+v", err) + return response, errors.New("Failed to fetch combined scrobbles") + } + count++ + response.Items = append(response.Items, item) + } + + err = rows.Err() + if err != nil { + log.Printf("Failed to fetch scrobbles: %+v", err) + return response, errors.New("Failed to fetch scrobbles") + } + + response.Meta.Count = count + response.Meta.Total = total + response.Meta.Page = page + + return response, nil +} diff --git a/internal/goscrobble/artist.go b/internal/goscrobble/artist.go index 9b82feb4..9b33d9ba 100644 --- a/internal/goscrobble/artist.go +++ b/internal/goscrobble/artist.go @@ -164,3 +164,59 @@ func getTopArtists(userUuid string) (TopArtists, error) { return topArtist, nil } + +// getTopUsersForArtistUUID - Returns list of top users for a track +func getTopUsersForArtistUUID(artistUUID string, limit int, page int) (TopUserResponse, error) { + response := TopUserResponse{} + + var count int + + total, err := getDbCount( + "SELECT COUNT(*) FROM `scrobbles` "+ + "JOIN `track_artist` ON `track_artist`.`track` = `scrobbles`.`track` "+ + "WHERE `track_artist`.`artist` = UUID_TO_BIN(?, true);", artistUUID) + + if err != nil { + log.Printf("Failed to fetch scrobble count: %+v", err) + return response, errors.New("Failed to fetch combined scrobbles") + } + + rows, err := db.Query( + "SELECT BIN_TO_UUID(`scrobbles`.`user`, true), `users`.`username`, COUNT(*) "+ + "FROM `scrobbles` "+ + "JOIN `users` ON `scrobbles`.`user` = `users`.`uuid` "+ + "JOIN `track_artist` ON `track_artist`.`track` = `scrobbles`.`track` "+ + "WHERE `track_artist`.`artist` = UUID_TO_BIN(?, true) "+ + "GROUP BY `scrobbles`.`user` "+ + "ORDER BY COUNT(*) DESC LIMIT ?", + artistUUID, limit) + + if err != nil { + log.Printf("Failed to fetch scrobbles: %+v", err) + return response, errors.New("Failed to fetch combined scrobbles") + } + defer rows.Close() + + for rows.Next() { + item := TopUserResponseItem{} + err := rows.Scan(&item.UserUUID, &item.UserName, &item.Count) + if err != nil { + log.Printf("Failed to fetch scrobbles: %+v", err) + return response, errors.New("Failed to fetch combined scrobbles") + } + count++ + response.Items = append(response.Items, item) + } + + err = rows.Err() + if err != nil { + log.Printf("Failed to fetch scrobbles: %+v", err) + return response, errors.New("Failed to fetch scrobbles") + } + + response.Meta.Count = count + response.Meta.Total = total + response.Meta.Page = page + + return response, nil +} diff --git a/internal/goscrobble/ingress_spotify.go b/internal/goscrobble/ingress_spotify.go index 62936f8b..78f45887 100644 --- a/internal/goscrobble/ingress_spotify.go +++ b/internal/goscrobble/ingress_spotify.go @@ -227,17 +227,17 @@ func ParseSpotifyInput(ctx context.Context, userUUID string, data spotify.Recent } // updateImageDataFromSpotify update artist/album images from spotify ;D -func (user *User) updateImageDataFromSpotify() error { +func (user *User) updateImageDataFromSpotify() { // Check that data is set before we attempt to pull val, _ := getConfigValue("SPOTIFY_API_SECRET") if val == "" { - return nil + return } // TO BE REWORKED TO NOT USE A DAMN USER ARGHHH dbToken, err := user.getSpotifyTokens() if err != nil { - return nil + return } token := new(oauth2.Token) @@ -252,7 +252,7 @@ func (user *User) updateImageDataFromSpotify() error { rows, err := db.Query("SELECT BIN_TO_UUID(`uuid`, true), `name` FROM `artists` WHERE IFNULL(`img`,'') NOT IN ('pending', 'complete') LIMIT 100") if err != nil { log.Printf("Failed to fetch config: %+v", err) - return errors.New("Failed to fetch artists") + return } toUpdate := make(map[string]string) @@ -263,7 +263,7 @@ func (user *User) updateImageDataFromSpotify() error { if err != nil { log.Printf("Failed to fetch artists: %+v", err) rows.Close() - return errors.New("Failed to fetch artist") + return } res, err := client.Search(ctx, name, spotify.SearchTypeArtist) if len(res.Artists.Artists) > 0 { @@ -296,7 +296,7 @@ func (user *User) updateImageDataFromSpotify() error { rows, err = db.Query("SELECT BIN_TO_UUID(`uuid`, true), `name` FROM `albums` WHERE IFNULL(`img`,'') NOT IN ('pending', 'complete') LIMIT 100") if err != nil { log.Printf("Failed to fetch config: %+v", err) - return errors.New("Failed to fetch artists") + return } toUpdate = make(map[string]string) @@ -307,7 +307,7 @@ func (user *User) updateImageDataFromSpotify() error { if err != nil { log.Printf("Failed to fetch albums: %+v", err) rows.Close() - return errors.New("Failed to fetch album") + return } res, err := client.Search(ctx, name, spotify.SearchTypeAlbum) if len(res.Albums.Albums) > 0 { @@ -331,5 +331,6 @@ func (user *User) updateImageDataFromSpotify() error { _ = album.updateAlbum("img", "pending", tx) } tx.Commit() - return nil + + return } diff --git a/internal/goscrobble/scrobble.go b/internal/goscrobble/scrobble.go index 3ae0c844..f83858a8 100644 --- a/internal/goscrobble/scrobble.go +++ b/internal/goscrobble/scrobble.go @@ -35,6 +35,7 @@ type ScrobbleResponseItem struct { Album string `json:"album"` Track ScrobbleTrackItem `json:"track"` Source string `json:"source"` + User ScrobbleTrackItem `json:"user"` } type ScrobbleTrackItem struct { @@ -127,3 +128,49 @@ func checkIfScrobbleExists(userUuid string, timestamp time.Time, source string) return count != 0 } + +func getRecentScrobbles() (ScrobbleResponse, error) { + scrobbleReq := ScrobbleResponse{} + var count int + limit := 50 + + rows, err := db.Query( + "SELECT BIN_TO_UUID(`scrobbles`.`uuid`, true), `scrobbles`.`created_at`, BIN_TO_UUID(`artists`.`uuid`, true), `artists`.`name`, `albums`.`name`, BIN_TO_UUID(`tracks`.`uuid`, true), `tracks`.`name`, `scrobbles`.`source`, BIN_TO_UUID(`scrobbles`.`user`, true), `users`.`username` FROM `scrobbles` "+ + "JOIN tracks ON scrobbles.track = tracks.uuid "+ + "JOIN track_artist ON track_artist.track = tracks.uuid "+ + "JOIN track_album ON track_album.track = tracks.uuid "+ + "JOIN artists ON track_artist.artist = artists.uuid "+ + "JOIN albums ON track_album.album = albums.uuid "+ + "JOIN users ON scrobbles.user = users.uuid "+ + "GROUP BY scrobbles.uuid, albums.uuid "+ + "ORDER BY scrobbles.created_at DESC LIMIT ?", limit) + + if err != nil { + log.Printf("Failed to fetch scrobbles: %+v", err) + return scrobbleReq, errors.New("Failed to fetch scrobbles") + } + defer rows.Close() + + for rows.Next() { + item := ScrobbleResponseItem{} + err := rows.Scan(&item.UUID, &item.Timestamp, &item.Artist.UUID, &item.Artist.Name, &item.Album, &item.Track.UUID, &item.Track.Name, &item.Source, &item.User.UUID, &item.User.Name) + if err != nil { + log.Printf("Failed to fetch scrobbles: %+v", err) + return scrobbleReq, errors.New("Failed to fetch scrobbles") + } + count++ + scrobbleReq.Items = append(scrobbleReq.Items, item) + } + + err = rows.Err() + if err != nil { + log.Printf("Failed to fetch scrobbles: %+v", err) + return scrobbleReq, errors.New("Failed to fetch scrobbles") + } + + scrobbleReq.Meta.Count = count + scrobbleReq.Meta.Total = 50 + scrobbleReq.Meta.Page = 1 + + return scrobbleReq, nil +} diff --git a/internal/goscrobble/server.go b/internal/goscrobble/server.go index a7cfa5e4..9ebb0029 100644 --- a/internal/goscrobble/server.go +++ b/internal/goscrobble/server.go @@ -72,13 +72,17 @@ func HandleRequests(port string) { // No Auth v1.HandleFunc("/stats", limitMiddleware(handleStats, lightLimiter)).Methods("GET") + v1.HandleFunc("/recent", limitMiddleware(handleRecentScrobbles, lightLimiter)).Methods("GET") + v1.HandleFunc("/profile/{username}", limitMiddleware(getProfile, lightLimiter)).Methods("GET") v1.HandleFunc("/artists/top/{uuid}", limitMiddleware(getArtists, lightLimiter)).Methods("GET") v1.HandleFunc("/artists/{uuid}", limitMiddleware(getArtist, lightLimiter)).Methods("GET") + v1.HandleFunc("/artists/{uuid}/top", limitMiddleware(getTopUsersForArtist, lightLimiter)).Methods("GET") v1.HandleFunc("/albums/top/{uuid}", limitMiddleware(getArtists, lightLimiter)).Methods("GET") v1.HandleFunc("/albums/{uuid}", limitMiddleware(getAlbum, lightLimiter)).Methods("GET") + v1.HandleFunc("/albums/{uuid}/top", limitMiddleware(getTopUsersForAlbum, lightLimiter)).Methods("GET") v1.HandleFunc("/tracks/top/{uuid}", limitMiddleware(getTracks, lightLimiter)).Methods("GET") // User UUID - Top Tracks v1.HandleFunc("/tracks/{uuid}", limitMiddleware(getTrack, lightLimiter)).Methods("GET") // Track UUID @@ -417,6 +421,8 @@ func patchUser(w http.ResponseWriter, r *http.Request, claims CustomClaims, reqU } else if k == "token" { token := generateToken(32) userFull.updateUser("token", token, ip) + } else if k == "active" { + userFull.updateUser("active", "0", ip) } } @@ -685,6 +691,56 @@ func getTopUsersForTrack(w http.ResponseWriter, r *http.Request) { w.Write(json) } +// getTopUsersForAlbum - I suck at naming. Returns top users that have scrobbled this track. +func getTopUsersForAlbum(w http.ResponseWriter, r *http.Request) { + var uuid string + for k, v := range mux.Vars(r) { + if k == "uuid" { + uuid = v + } + } + + if uuid == "" { + throwOkError(w, "Invalid UUID") + return + } + + userList, err := getTopUsersForAlbumUUID(uuid, 10, 1) + if err != nil { + throwOkError(w, err.Error()) + return + } + + json, _ := json.Marshal(&userList) + w.WriteHeader(http.StatusOK) + w.Write(json) +} + +// getTopUsersForArtist - I suck at naming. Returns top users that have scrobbled this track. +func getTopUsersForArtist(w http.ResponseWriter, r *http.Request) { + var uuid string + for k, v := range mux.Vars(r) { + if k == "uuid" { + uuid = v + } + } + + if uuid == "" { + throwOkError(w, "Invalid UUID") + return + } + + userList, err := getTopUsersForArtistUUID(uuid, 10, 1) + if err != nil { + throwOkError(w, err.Error()) + return + } + + json, _ := json.Marshal(&userList) + w.WriteHeader(http.StatusOK) + w.Write(json) +} + // postSpotifyResponse - Oauth Response from Spotify func postSpotifyReponse(w http.ResponseWriter, r *http.Request) { err := connectSpotifyResponse(r) @@ -782,7 +838,7 @@ func getServerInfo(w http.ResponseWriter, r *http.Request) { } info := ServerInfo{ - Version: "0.1.1", + Version: "0.1.2", RegistrationEnabled: registrationEnabled, } @@ -790,3 +846,15 @@ func getServerInfo(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(js) } + +func handleRecentScrobbles(w http.ResponseWriter, r *http.Request) { + scrobbleList, err := getRecentScrobbles() + if err != nil { + throwOkError(w, err.Error()) + return + } + + json, _ := json.Marshal(&scrobbleList) + w.WriteHeader(http.StatusOK) + w.Write(json) +} diff --git a/internal/goscrobble/track.go b/internal/goscrobble/track.go index 88f0df05..3e942673 100644 --- a/internal/goscrobble/track.go +++ b/internal/goscrobble/track.go @@ -26,27 +26,11 @@ type TopTrack struct { Img string `json:"img"` Plays int `json:"plays"` } + type TopTracks struct { Tracks map[int]TopTrack `json:"tracks"` } -type TopUserTrackResponse struct { - Meta TopUserTrackResponseMeta `json:"meta"` - Items []TopUserTrackResponseItem `json:"items"` -} - -type TopUserTrackResponseMeta struct { - Count int `json:"count"` - Total int `json:"total"` - Page int `json:"page"` -} - -type TopUserTrackResponseItem struct { - UserUUID string `json:"user_uuid"` - Count int `json:"count"` - UserName string `json:"user_name"` -} - // insertTrack - This will return if it exists or create it based on MBID > Name func insertTrack(name string, legnth int, mbid string, spotifyId string, album string, artists []string, tx *sql.Tx) (Track, error) { track := Track{} @@ -313,12 +297,12 @@ func (track *Track) getAlbumsForTrack() error { } // getTopUsersForTrackUUID - Returns list of top users for a track -func getTopUsersForTrackUUID(trackUUID string, limit int, page int) (TopUserTrackResponse, error) { - response := TopUserTrackResponse{} +func getTopUsersForTrackUUID(trackUUID string, limit int, page int) (TopUserResponse, error) { + response := TopUserResponse{} var count int total, err := getDbCount( - "SELECT COUNT(*) FROM `scrobbles` WHERE `track` = UUID_TO_BIN(?, true) GROUP BY `track`, `user`", trackUUID) + "SELECT COUNT(*) FROM `scrobbles` WHERE `track` = UUID_TO_BIN(?, true)", trackUUID) if err != nil { log.Printf("Failed to fetch scrobble count: %+v", err) @@ -341,7 +325,7 @@ func getTopUsersForTrackUUID(trackUUID string, limit int, page int) (TopUserTrac defer rows.Close() for rows.Next() { - item := TopUserTrackResponseItem{} + item := TopUserResponseItem{} err := rows.Scan(&item.UserUUID, &item.UserName, &item.Count) if err != nil { log.Printf("Failed to fetch scrobbles: %+v", err) diff --git a/internal/goscrobble/user.go b/internal/goscrobble/user.go index 830e5d7b..94be1192 100644 --- a/internal/goscrobble/user.go +++ b/internal/goscrobble/user.go @@ -48,6 +48,23 @@ type UserResponse struct { NavidromeURL string `json:"navidrome_server"` } +type TopUserResponse struct { + Meta TopUserResponseMeta `json:"meta"` + Items []TopUserResponseItem `json:"items"` +} + +type TopUserResponseMeta struct { + Count int `json:"count"` + Total int `json:"total"` + Page int `json:"page"` +} + +type TopUserResponseItem struct { + UserUUID string `json:"user_uuid"` + Count int `json:"count"` + UserName string `json:"user_name"` +} + // createUser - Called from API func createUser(req *RequestRequest, ip net.IP) error { // Check if user already exists.. diff --git a/migrations/0_create_db.sql b/migrations/0_create_db.sql new file mode 100644 index 00000000..cbcfb9ce --- /dev/null +++ b/migrations/0_create_db.sql @@ -0,0 +1 @@ +CREATE DATABASE IF NOT EXISTS goscrobble; \ No newline at end of file