feat(definitions): add expectations for test lines (#1257)

* factor out test helpers

* refactor, add tests for animebytes

* revert test refactor

* better name

* change format, migrate some examples

* migrated remaining test cases

* add comment about `Test` vs `Tests`

* refactor

* reorder expectations to match vars

* generate

* turn on strict unmarshalling, remove old `Test` from schema

* start modifying actual definitions

* done with the As

* Bs

* C, D

* E, F

* G, H, I, ... L

* M, N

* O, P

* R

* bonus error. without this, pattern/vars disagreement can panic.

* S

* T, U

* X.. Now we know our ABCs next time won't you sing with meeeee

* fix another test

* another driveby change

* be less strict parsing custom definitions

* fix(definitions): load custom definitions

---------

Co-authored-by: ze0s <ze0s@riseup.net>

internal/indexer/parse.go

@@ -0,0 +1,76 @@
+package indexer
+
+import (
+	"errors"
+	"regexp"
+
+	"github.com/rs/zerolog"
+)
+
+type Logger interface {
+	Debug() *zerolog.Event
+}
+
+func regExMatch(pattern string, value string) ([]string, error) {
+	rxp, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	matches := rxp.FindStringSubmatch(value)
+	if matches == nil {
+		return nil, nil
+	}
+
+	return matches[1:], nil
+}
+
+func ParseLine(logger Logger, pattern string, vars []string, tmpVars map[string]string, line string, ignore bool) (bool, error) {
+	if len(vars) > 0 {
+		return parseExtract(logger, pattern, vars, tmpVars, line)
+	}
+
+	return parseMatchRegexp(pattern, tmpVars, line, ignore)
+}
+
+func parseExtract(logger Logger, pattern string, vars []string, tmpVars map[string]string, line string) (bool, error) {
+	rxp, err := regExMatch(pattern, line)
+	if err != nil {
+		logger.Debug().Msgf("did not match expected line: %v", line)
+	}
+
+	if rxp == nil {
+		return false, nil
+	}
+
+	for i, v := range vars {
+		if i+1 > len(rxp) {
+			return false, errors.New("too few matches returned for rxp")
+		}
+
+		tmpVars[v] = rxp[i]
+	}
+	return true, nil
+}
+
+func parseMatchRegexp(pattern string, tmpVars map[string]string, line string, ignore bool) (bool, error) {
+	var re = regexp.MustCompile(`(?mi)` + pattern)
+
+	groupNames := re.SubexpNames()
+	for _, match := range re.FindAllStringSubmatch(line, -1) {
+		for groupIdx, group := range match {
+			// if line should be ignored then lets return
+			if ignore {
+				return true, nil
+			}
+
+			name := groupNames[groupIdx]
+			if name == "" {
+				name = "raw"
+			}
+			tmpVars[name] = group
+		}
+	}
+
+	return true, nil
+}