feat(logs): sanitize logfile on download (#767)

* initial commit

* handle tleech urls

* improved and simplified regex

* add sanitization status & loading anim for log dl

* removed unused imports

* improved regex

* fixed regex and added tests

* regex improvements and tests

* added unicode matching to saslRegex

* added missing baseurl

* swapped the css animator for a react component

the css version froze when served through a reverse proxy

* optimized regex compilation

---------

Co-authored-by: soup <soup@r4tio.cat>

internal/http/logs.go

@@ -2,10 +2,12 @@ package http
 
 import (
 	"io/fs"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -85,6 +87,50 @@ func (h logsHandler) files(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, response)
 }
 
+var ( // regexes for sanitizing log files
+	keyValueRegex = regexp.MustCompile(`(torrent_pass|passkey|authkey|secret_key|apikey)=([a-zA-Z0-9]+)`)
+	combinedRegex = regexp.MustCompile(`(https?://[^\s]+/((rss/download/[a-zA-Z0-9]+/)|torrent/download/((auto\.[a-zA-Z0-9]+\.|[a-zA-Z0-9]+\.))))([a-zA-Z0-9]+)`)
+	inviteRegex   = regexp.MustCompile(`(Voyager autobot [\p{L}0-9]+ |Satsuki enter #announce [\p{L}0-9]+ |Millie announce |DBBot announce |ENDOR !invite [\p{L}0-9]+ |Vertigo ENTER #GGn-Announce [\p{L}0-9]+ |midgards announce |HeBoT !invite |NBOT !invite |Muffit bot #nbl-announce [\p{L}0-9]+ |hermes enter #announce [\p{L}0-9]+ |LiMEY_ !invite |PS-Info pass |PT-BOT invite |Hummingbird ENTER [\p{L}0-9]+ |Drone enter #red-announce [\p{L}0-9]+ |SceneHD \.invite |erica letmeinannounce [\p{L}0-9]+ |Synd1c4t3 invite |UHDBot invite |Sauron bot #ant-announce [\p{L}0-9]+ |RevoTT !invite [\p{L}0-9]+ |Cerberus identify [\p{L}0-9]+ )([\p{L}0-9]+)`)
+	nickservRegex = regexp.MustCompile(`(NickServ IDENTIFY )([\p{L}0-9!#%&*+/:;<=>?@^_` + "`" + `{|}~]+)`)
+	saslRegex     = regexp.MustCompile(`(--> AUTHENTICATE )([\p{L}0-9!#%&*+/:;<=>?@^_` + "`" + `{|}~]+)`)
+)
+
+func SanitizeLogFile(filePath string) (string, error) {
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return "", err
+	}
+
+	sanitizedData := string(data)
+
+	// torrent_pass, passkey, authkey, secret_key, apikey, rsskey
+	sanitizedData = keyValueRegex.ReplaceAllString(sanitizedData, "${1}=REDACTED")
+	sanitizedData = combinedRegex.ReplaceAllString(sanitizedData, "${1}REDACTED")
+
+	// irc related
+	sanitizedData = inviteRegex.ReplaceAllString(sanitizedData, "${1}REDACTED")
+	sanitizedData = nickservRegex.ReplaceAllString(sanitizedData, "${1}REDACTED")
+	sanitizedData = saslRegex.ReplaceAllString(sanitizedData, "${1}REDACTED")
+
+	tmpFile, err := ioutil.TempFile("", "sanitized-log-*.log")
+	if err != nil {
+		return "", err
+	}
+
+	_, err = tmpFile.WriteString(sanitizedData)
+	if err != nil {
+		tmpFile.Close()
+		return "", err
+	}
+
+	err = tmpFile.Close()
+	if err != nil {
+		return "", err
+	}
+
+	return tmpFile.Name(), nil
+}
+
 func (h logsHandler) downloadFile(w http.ResponseWriter, r *http.Request) {
 	if h.cfg.Config.LogPath == "" {
 		render.Status(r, http.StatusNotFound)
@@ -120,12 +166,24 @@ func (h logsHandler) downloadFile(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	filePath := filepath.Join(logsDir, logFile)
+
+	// Sanitize the log file
+	sanitizedFilePath, err := SanitizeLogFile(filePath)
+	if err != nil {
+		render.Status(r, http.StatusInternalServerError)
+		render.JSON(w, r, errorResponse{
+			Message: err.Error(),
+			Status:  http.StatusInternalServerError,
+		})
+		return
+	}
+	defer os.Remove(sanitizedFilePath)
+
 	w.Header().Set("Content-Disposition", "attachment; filename="+strconv.Quote(logFile))
 	w.Header().Set("Content-Type", "application/octet-stream")
 
-	filePath := filepath.Join(logsDir, logFile)
+	http.ServeFile(w, r, sanitizedFilePath)
-
-	http.ServeFile(w, r, filePath)
 }
 
 type logFile struct {

internal/http/logs_sanitize_test.go

@@ -0,0 +1,169 @@
+package http
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestSanitizeLogFile(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "https://beyond-hd.me/torrent/download/auto.t0rrent1d.rssk3y",
+			expected: "https://beyond-hd.me/torrent/download/auto.t0rrent1d.REDACTED",
+		},
+		{
+			input:    "https://aither.cc/torrent/download/t0rrent1d.rssk3y",
+			expected: "https://aither.cc/torrent/download/t0rrent1d.REDACTED",
+		},
+		{
+			input:    "https://www.torrentleech.org/rss/download/t0rrent1d/rssk3y/Dark+Places+1974+1080p+BluRay+x264-GAZER.torrent",
+			expected: "https://www.torrentleech.org/rss/download/t0rrent1d/REDACTED/Dark+Places+1974+1080p+BluRay+x264-GAZER.torrent",
+		},
+		{
+			input:    "https://alpharatio.cc/torrents.php?action=download&id=t0rrent1d&authkey=4uthk3y&torrent_pass=t0rrentp4ss",
+			expected: "https://alpharatio.cc/torrents.php?action=download&id=t0rrent1d&authkey=REDACTED&torrent_pass=REDACTED",
+		},
+		{
+			input:    "Voyager autobot us3rn4me 1RCK3Y",
+			expected: "Voyager autobot us3rn4me REDACTED",
+		},
+		{
+			input:    "Satsuki enter #announce us3rn4me 1RCK3Y",
+			expected: "Satsuki enter #announce us3rn4me REDACTED",
+		},
+		{
+			input:    "Millie announce 1RCK3Y",
+			expected: "Millie announce REDACTED",
+		},
+		{
+			input:    "DBBot announce 1RCK3Y",
+			expected: "DBBot announce REDACTED",
+		},
+		{
+			input:    "ENDOR !invite us3rnøme 1RCK3Y",
+			expected: "ENDOR !invite us3rnøme REDACTED",
+		},
+		{
+			input:    "Vertigo ENTER #GGn-Announce us3rn4me 1RCK3Y",
+			expected: "Vertigo ENTER #GGn-Announce us3rn4me REDACTED",
+		},
+		{
+			input:    "midgards announce 1RCK3Y",
+			expected: "midgards announce REDACTED",
+		},
+		{
+			input:    "HeBoT !invite 1RCK3Y",
+			expected: "HeBoT !invite REDACTED",
+		},
+		{
+			input:    "NBOT !invite 1RCK3Y",
+			expected: "NBOT !invite REDACTED",
+		},
+		{
+			input:    "Muffit bot #nbl-announce us3rn4me 1RCK3Y",
+			expected: "Muffit bot #nbl-announce us3rn4me REDACTED",
+		},
+		{
+			input:    "hermes enter #announce us3rn4me 1RCK3Y",
+			expected: "hermes enter #announce us3rn4me REDACTED",
+		},
+		{
+			input:    "LiMEY_ !invite 1RCK3Y us3rn4me",
+			expected: "LiMEY_ !invite REDACTED us3rn4me",
+		},
+		{
+			input:    "PS-Info pass 1RCK3Y",
+			expected: "PS-Info pass REDACTED",
+		},
+		{
+			input:    "PT-BOT invite 1RCK3Y",
+			expected: "PT-BOT invite REDACTED",
+		},
+		{
+			input:    "Hummingbird ENTER us3rn4me 1RCK3Y #ptp-announce-dev",
+			expected: "Hummingbird ENTER us3rn4me REDACTED #ptp-announce-dev",
+		},
+		{
+			input:    "Drone enter #red-announce us3rn4me 1RCK3Y",
+			expected: "Drone enter #red-announce us3rn4me REDACTED",
+		},
+		{
+			input:    "SceneHD .invite 1RCK3Y #announce",
+			expected: "SceneHD .invite REDACTED #announce",
+		},
+		{
+			input:    "erica letmeinannounce us3rn4me 1RCK3Y",
+			expected: "erica letmeinannounce us3rn4me REDACTED",
+		},
+		{
+			input:    "Synd1c4t3 invite 1RCK3Y",
+			expected: "Synd1c4t3 invite REDACTED",
+		},
+		{
+			input:    "UHDBot invite 1RCK3Y",
+			expected: "UHDBot invite REDACTED",
+		},
+		{
+			input:    "Sauron bot #ant-announce us3rn4me 1RCK3Y",
+			expected: "Sauron bot #ant-announce us3rn4me REDACTED",
+		},
+		{
+			input:    "RevoTT !invite us3rn4me P4SSK3Y",
+			expected: "RevoTT !invite us3rn4me REDACTED",
+		},
+		{
+			input:    "Cerberus identify us3rn4me P1D",
+			expected: "Cerberus identify us3rn4me REDACTED",
+		},
+		{
+			input:    "NickServ IDENTIFY dasøl13sa#!",
+			expected: "NickServ IDENTIFY REDACTED",
+		},
+		{
+			input:    "--> AUTHENTICATE poasd!232kljøasdj!%",
+			expected: "--> AUTHENTICATE REDACTED",
+		},
+	}
+
+	for _, testCase := range testCases {
+		// Create a temporary file with sample log data
+		tmpFile, err := ioutil.TempFile("", "test-log-*.log")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.Remove(tmpFile.Name())
+
+		// Write sample log data to the temporary file
+		_, err = tmpFile.WriteString(testCase.input)
+		if err != nil {
+			tmpFile.Close()
+			t.Fatal(err)
+		}
+		err = tmpFile.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Call SanitizeLogFile on the temporary file
+		sanitizedTmpFilePath, err := SanitizeLogFile(tmpFile.Name())
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.Remove(sanitizedTmpFilePath)
+
+		// Read the content of the sanitized temporary file
+		sanitizedData, err := ioutil.ReadFile(sanitizedTmpFilePath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Check if the sanitized data matches the expected content
+		if string(sanitizedData) != testCase.expected {
+			t.Errorf("Sanitized data does not match expected data for input: %s\nExpected:\n%s\nActual:\n%s", testCase.input, testCase.expected, sanitizedData)
+		}
+	}
+}

web/src/screens/Logs.tsx

@@ -12,8 +12,9 @@ import {
   DocumentArrowDownIcon
 } from "@heroicons/react/24/outline";
 import { useQuery } from "react-query";
-import { Link } from "react-router-dom";
 import { Menu, Transition } from "@headlessui/react";
+import { baseUrl } from "../utils";
+
 
 type LogEvent = {
   time: string;
@@ -207,7 +208,55 @@ interface LogFilesItemProps {
   file: LogFile;
 }
 
+const Dots = () => {
+  const [step, setStep] = useState(1);
+
+  useEffect(() => {
+    const interval = setInterval(() => {
+      setStep((prevStep) => (prevStep % 3) + 1);
+    }, 300);
+
+    return () => clearInterval(interval);
+  }, []);
+
+  return (
+    <div className="flex">
+      <div
+        className={`h-2 w-2 bg-blue-500 rounded-full mx-1 ${
+          step === 1 ? "opacity-100" : "opacity-30"
+        }`}
+      />
+      <div
+        className={`h-2 w-2 bg-blue-500 rounded-full mx-1 ${
+          step === 2 ? "opacity-100" : "opacity-30"
+        }`}
+      />
+      <div
+        className={`h-2 w-2 bg-blue-500 rounded-full mx-1 ${
+          step === 3 ? "opacity-100" : "opacity-30"
+        }`}
+      />
+    </div>
+  );
+};
+
 const LogFilesItem = ({ file }: LogFilesItemProps) => {
+  const [isDownloading, setIsDownloading] = useState(false);
+
+  const handleDownload = async () => {
+    setIsDownloading(true);
+    const response = await fetch(`${baseUrl()}api/logs/files/${file.filename}`);
+    const blob = await response.blob();
+    const url = URL.createObjectURL(blob);
+    const link = document.createElement("a");
+    link.href = url;
+    link.download = file.filename;
+    link.click();
+    URL.revokeObjectURL(url);
+    setIsDownloading(false);
+  };
+  
+
   return (
 
     <li className="text-gray-500 dark:text-gray-400">
@@ -224,20 +273,29 @@ const LogFilesItem = ({ file }: LogFilesItemProps) => {
         <div className="col-span-4 flex items-center text-sm font-medium text-gray-900 dark:text-gray-200" title={file.updated_at}>
           {simplifyDate(file.updated_at)}
         </div>
-
         <div className="col-span-1 hidden sm:flex items-center justify-center text-sm font-medium text-gray-900 dark:text-white">
-          <Link
+          <div className="logFilesItem">
-            className={classNames(
+            <button
-              "text-gray-900 dark:text-gray-300",
+              className={classNames(
-              "font-medium group flex rounded-md items-center px-2 py-2 text-sm"
+                "text-gray-900 dark:text-gray-300",
-            )}
+                "font-medium group flex rounded-md items-center px-2 py-2 text-sm"
-            title="Download file"
+              )}
-            to={`/api/logs/files/${file.filename}`}
+              title="Download file"
-            target="_blank"
+              onClick={handleDownload}
-            download={true}
+            >
-          >
+              {!isDownloading ? (
-            <DocumentArrowDownIcon className="text-blue-500 w-5 h-5" aria-hidden="true" />
+                <DocumentArrowDownIcon
-          </Link>
+                  className="text-blue-500 w-5 h-5 iconHeight"
+                  aria-hidden="true"
+                />
+              ) : (
+                <div className="h-5 flex items-center">
+                  <span className="sanitizing-text">Sanitizing log</span>
+                  <Dots />
+                </div>
+              )}
+            </button>
+          </div>
         </div>
       </div>
     </li>