Rework to use API instead of CSVs

This commit is contained in:
Daniel Mason 2022-01-02 12:01:59 +13:00
parent b73a6b7cae
commit 4937ec7253
13 changed files with 290 additions and 675 deletions

151
internal/nzcovidbot/api.go Normal file
View file

@ -0,0 +1,151 @@
package nzcovidbot
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"sort"
"time"
)
const API_ENDPOINT = "https://api.integration.covid19.health.nz/locations/v1/current-locations-of-interest"
var newLocations ApiResponse
type ApiResponse struct {
Items []ApiItem `json:"items"`
}
type ApiItem struct {
EventID string `json:"eventId"`
EventName string `json:"eventName"`
StartDateTime time.Time `json:"startDateTime"`
EndDateTime time.Time `json:"endDateTime"`
PublicAdvice string `json:"publicAdvice"`
VisibleInWebform bool `json:"visibleInWebform"`
PublishedAt time.Time `json:"publishedAt"`
// UpdatedAt time.Time `json:"updatedAt"` // Nullable
ExposureType string `json:"exposureType"`
Location struct {
Latitude string `json:"latitude"`
Longitude string `json:"longitude"`
Suburb string `json:"suburb"`
City string `json:"city"`
Address string `json:"address"`
} `json:"location"`
}
// fetchAPILocations - Return struct of API response
func fetchAPILocations() (ApiResponse, error) {
var apiResponse ApiResponse
// Build HTTP Client and create request
client := &http.Client{}
req, err := http.NewRequest("GET", API_ENDPOINT, nil)
if err != nil {
return apiResponse, err
}
// Set user-agent info
req.Header.Set("User-Agent", "NZCovidBot/1.0 (https://m2.nz)")
// Fire off the request
resp, err := client.Do(req)
if err != nil {
return apiResponse, err
}
defer resp.Body.Close()
// Read body response
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return apiResponse, err
}
// Unmarshal JSON into Go struct
err = json.Unmarshal(body, &apiResponse)
return apiResponse, err
}
// getNewAPILocations - Gets all locations and triggers posts
func getNewAPILocations() {
// Set lastUpdate time at the start of the request so we don't miss any locations
// posted right after we poll
newPollTime := time.Now()
// Pull latest data
locations, err := fetchAPILocations()
if err != nil {
log.Printf("Error fetching API Locations %s", err)
return
}
// Re-init our apiRepsonse so we don't hold onto old locations!
newItems := make([]ApiItem, 0)
// Iterate over the data and only find new locations
for _, item := range locations.Items {
if item.PublishedAt.Unix() > lastUpdated.Unix() {
// Clone the item to put in our own lil slice
copy := item
newItems = append(newItems, copy)
}
}
// Make sure to clear out the previous list and append new data
newLocations = ApiResponse{}
newLocations.Items = newItems
// Order by StartDate
sort.Slice(newLocations.Items, func(i, j int) bool {
return newLocations.Items[i].StartDateTime.Before(newLocations.Items[j].StartDateTime)
})
// If new items, post it!
if len(newLocations.Items) > 0 {
postTheUpdates()
}
updateLastUpdated(newPollTime)
}
// updateLastUpdated - Creates/Updates lastUpdated.txt
func updateLastUpdated(newUpdated time.Time) {
// Make sure to update the global var for next poll
lastUpdated = newUpdated
// Open file in truncate/append mode
f, err := os.OpenFile("lastUpdated.txt", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
if err != nil {
log.Println(err)
return
}
// Write data
data := []byte(fmt.Sprintf("%d", lastUpdated.Unix()))
_, err = f.Write(data)
if err != nil {
log.Println(err)
return
}
// Close file so we can reopen next time
if err := f.Close(); err != nil {
log.Println(err)
}
}
// getDateString - Returns Date + StartTime + EndTime
func (item ApiItem) getDateString() string {
st := item.StartDateTime
et := item.EndDateTime
dateFormat := "Mon 2 Jan, 03:04PM"
timeFormat := "03:04PM"
return st.Format(dateFormat) + " - " + et.Format(timeFormat)
}

View file

@ -1,315 +0,0 @@
package nzcovidbot
import (
"encoding/csv"
"fmt"
"io"
"io/ioutil"
"log"
"net/url"
"os"
"regexp"
"sort"
"strings"
"time"
"github.com/ashwanthkumar/slack-go-webhook"
)
// Slice of updated located
type UpdatedLocations struct {
Locations []UpdatedRow
}
// Updated data
type UpdatedRow struct {
FromDate time.Time `json:"FromDate"` // Start date
EndDate time.Time `json:"EndDate"` // End date
LocationName string `json:"LocationName"` // Location Name
LocationAddress string `json:"LocationAddress"` // Location Address
DiscordData string `json:"-"` // Formatted Row data
TwitterData string `json:"-"` // Formatted Row data
SlackData slack.Attachment `json:"-"` // Formatted Row data
}
// Struct of updated locations
var updatedLocations UpdatedLocations
// cache of [exposureID]row of row data
var rowCache map[string]UpdatedRow
// parseCsvRow Build into struct for output later
func parseCsvRow(data string) {
c, st, et, err := parseRawRowData(data)
if err != nil {
return
}
if len(c) < 5 {
log.Printf("Invalid line. Skipping")
return
}
if rowHasChanged(c[4], st, et, c[2], c[3]) {
newRow := UpdatedRow{
FromDate: st,
EndDate: et,
LocationName: c[2],
LocationAddress: c[3],
DiscordData: formatCsvDiscordRow(c),
TwitterData: formatCsvTwitterRow(c),
SlackData: formatCsvSlackRow(c),
}
// Update row cache! [exposureId]UpdatedRow
rowCache[c[4]] = newRow
// Append row data
updatedLocations.Locations = append(updatedLocations.Locations, newRow)
}
}
// rowHasChanged - Determine if row has actually changed based on raw data
func rowHasChanged(exposureId string, startTime time.Time, endTime time.Time, locationName string, locationAddress string) bool {
val, exists := rowCache[exposureId]
if !exists {
log.Printf("exposureId %s is new. Adding to cache", exposureId)
return true
}
if val.FromDate.Unix() != startTime.Unix() {
log.Printf("StartDate Change for %s from %s to %s", exposureId, val.FromDate.String(), startTime.String())
return true
}
if val.EndDate.Unix() != endTime.Unix() {
log.Printf("EndDate Change for %s from %s to %s", exposureId, val.EndDate.String(), endTime.String())
return true
}
if !strings.EqualFold(val.LocationName, locationName) {
log.Printf("LocationName Change for %s from %s to %s", exposureId, val.LocationName, locationName)
return true
}
// if !strings.EqualFold(val.LocationAddress, locationAddress) {
// log.Printf("LocationAddress Change for %s from %s to %s", exposureId, val.LocationAddress, locationAddress)
// return true
// }
return false
}
// loadRepoIntoCache - reads all CSV data and parses the rows into our cache
func loadRepoIntoCache(repoLocation string) {
// Init our cache!
rowCache = make(map[string]UpdatedRow)
// Load cache file. ELSE load files.
folders, err := ioutil.ReadDir(repoLocation + "/locations-of-interest")
if err != nil {
log.Fatal(err)
}
// /august-2021
for _, f := range folders {
if f.IsDir() {
files, err := ioutil.ReadDir(repoLocation + "/locations-of-interest/" + f.Name())
if err != nil {
log.Fatal(err)
}
// august-2021/locations-of-interest.csv
for _, x := range files {
fullLocation := repoLocation + "/locations-of-interest/" + f.Name() + "/" + x.Name()
if strings.HasSuffix(fullLocation, ".csv") {
loadRowsIntoCache(fullLocation)
}
}
}
}
log.Printf("Successfully populated cache with %d entries", len(rowCache))
}
func loadRowsIntoCache(filePath string) {
// Open the file
csvfile, err := os.Open(filePath)
if err != nil {
log.Fatal(err)
}
defer csvfile.Close()
// Parse the file
r := csv.NewReader(csvfile)
// Iterate through the records
i := 0
for {
// Read each record from csv
row, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
continue
}
// Skip header row
if i == 0 {
i++
continue
}
// Parse into our required format
c := make([]string, 0)
c = append(c, row...)
st, et, err := parseRowTimes(c[4], c[5])
if err != nil {
continue
}
// Build object
newRow := UpdatedRow{
FromDate: st,
EndDate: et,
LocationName: c[1],
LocationAddress: c[2],
}
// Add to cache
rowCache[row[0]] = newRow
}
}
func orderRowDataByDate() {
sort.Slice(updatedLocations.Locations, func(i, j int) bool {
return updatedLocations.Locations[i].FromDate.Before(updatedLocations.Locations[j].FromDate)
})
}
// formatCsvDiscordRow Format the string to a tidy string for the interwebs
func formatCsvDiscordRow(c []string) string {
return fmt.Sprintf("**%s** %s on _%s_ - _%s_", c[2], c[3], c[0], c[1])
}
// formatCsvTwitterRow Format the string to a tidy string for the interwebs
func formatCsvTwitterRow(c []string) string {
return fmt.Sprintf("New Location: %s\n%s\n%s - %s\n#NZCovidTracker #NZCovid", c[2], c[3], c[0], c[1])
}
// formatCsvSlackRow Format the string to a tidy string for the interwebs
func formatCsvSlackRow(c []string) slack.Attachment {
url := getMapsLinkFromAddress(c[2], c[3])
name := stripDateFromName(c[2])
dateRange := fmt.Sprintf("%s - %s", c[0], c[1])
attachment := slack.Attachment{
Title: &name,
TitleLink: &url,
Text: &dateRange,
}
return attachment
}
// getMapsLinkFromAddress hyperlink gmaps
func getMapsLinkFromAddress(name string, address string) string {
return fmt.Sprintf("https://www.google.com/maps/search/?api=1&query=%s", url.QueryEscape(name+", "+address))
}
// stripDateFromName if theres a date at the end - remove it!
func stripDateFromName(name string) string {
re := regexp.MustCompile(`\d{1,2}/\d{1,2}/\d{2,4}`)
submatchall := re.FindAllString(name, -1)
for _, element := range submatchall {
name = strings.Replace(name, element, "", 1)
break
}
return strings.TrimSpace(name)
}
// Returns []string of parsed data.. starttime, endtime, name, address, ID
func parseRawRowData(data string) ([]string, time.Time, time.Time, error) {
output := make([]string, 0)
r := csv.NewReader(strings.NewReader(data))
r.Comma = ','
fields, err := r.Read()
if err != nil {
fmt.Println(err)
return output, time.Now(), time.Now(), err
}
c := make([]string, 0)
c = append(c, fields...)
if len(c) < 5 {
// Add helper in case someone somehow does something wrong
return output, time.Now(), time.Now(), err
}
st, et, err := parseRowTimes(c[4], c[5])
starttime := st.Format("Monday 2 Jan, 3:04PM")
endtime := et.Format("3:04PM")
return append(output, starttime, endtime, c[1], c[2], c[0]), st, et, err
}
func parseRowTimes(startString string, endString string) (time.Time, time.Time, error) {
st, err := time.Parse("2/01/2006, 3:04 pm", startString)
if err != nil {
log.Print(err)
st, err = time.Parse("2006-01-02 15:04:05", startString)
if err != nil {
log.Print(err)
return time.Now(), time.Now(), err
}
}
et, err := time.Parse("2/01/2006, 3:04 pm", endString)
if err != nil {
log.Print(err)
et, err = time.Parse("2006-01-02 15:04:05", endString)
if err != nil {
log.Print(err)
return time.Now(), time.Now(), err
}
}
return st, et, nil
}
func getPostableDiscordData() []string {
groups := make([]string, 0)
if len(updatedLocations.Locations) == 0 {
return groups
}
rows := make([]string, 0)
for _, location := range updatedLocations.Locations {
rows = append(rows, location.DiscordData)
if len(rows) > 20 {
groups = append(groups, strings.Join(rows, "\n"))
rows = make([]string, 0)
}
}
return append(groups, strings.Join(rows, "\n"))
}
func getPostableSlackData() []slack.Attachment {
rows := make([]slack.Attachment, 0)
if len(updatedLocations.Locations) == 0 {
return rows
}
for _, location := range updatedLocations.Locations {
rows = append(rows, location.SlackData)
}
return rows
}

View file

@ -1,8 +1,10 @@
package nzcovidbot
import (
"fmt"
"log"
"strings"
"time"
"github.com/DisgoOrg/disgohook"
"github.com/DisgoOrg/disgohook/api"
@ -11,29 +13,66 @@ import (
// Slice of discord webhooks
var DiscordWebhooks []string
func postToDiscord(webhookString string, msg string) {
if webhookString == "" {
func postToDiscord() {
postableDiscordData := getPostableDiscordData()
if len(postableDiscordData) == 0 {
return
}
tokenParts := strings.Split(webhookString, "/")
len := len(tokenParts)
webhook, err := disgohook.NewWebhookClientByToken(nil, nil, tokenParts[len-2]+"/"+tokenParts[len-1])
if err != nil {
log.Print(err)
return
}
for _, discordWebhook := range DiscordWebhooks {
for _, postableData := range postableDiscordData {
if discordWebhook != "" {
tokenParts := strings.Split(discordWebhook, "/")
len := len(tokenParts)
_, err = webhook.SendEmbeds(api.NewEmbedBuilder().
SetDescription(msg).
Build(),
)
if err != nil {
log.Print(err)
return
}
// Build discord request
webhook, err := disgohook.NewWebhookClientByToken(nil, nil, tokenParts[len-2]+"/"+tokenParts[len-1])
if err != nil {
log.Print(err)
return
}
if err != nil {
log.Print(err)
// Send discord message
_, err = webhook.SendEmbeds(api.NewEmbedBuilder().
SetDescription(postableData).
Build(),
)
if err != nil {
log.Print(err)
}
time.Sleep(500 * time.Millisecond)
}
}
}
}
// getPostableDiscordData - Returns slices containing 20~ locations each
// to send as separate messages
func getPostableDiscordData() []string {
// Create our slice of groups
groups := make([]string, 0)
if len(newLocations.Items) == 0 {
return groups
}
rows := make([]string, 0)
for _, item := range newLocations.Items {
rows = append(rows, getDiscordRow(item))
if len(rows) > 20 {
groups = append(groups, strings.Join(rows, "\n"))
rows = make([]string, 0)
}
}
return append(groups, strings.Join(rows, "\n"))
}
// formatCsvDiscordRow Format the string to a tidy string for the interwebs
func getDiscordRow(item ApiItem) string {
return fmt.Sprintf("**%s** %s on _%s_",
item.EventName, item.Location.Address, item.getDateString())
}

View file

@ -1,144 +0,0 @@
package nzcovidbot
import (
"log"
"os"
"strings"
"github.com/go-git/go-git/v5"
"github.com/waigani/diffparser"
)
// Repo URL
var gitRepo *git.Repository
// Location to store tmp data
var tmpDirectory = "./tmp"
// loadRepo Load repo into gitRepo var
func loadRepo(repository string) {
r, err := git.PlainOpen(tmpDirectory + "/repo")
if err != nil {
if err.Error() == "repository does not exist" {
r = cloneRepo(repository)
} else {
log.Fatal(err)
}
}
// Preload cache data of current rows
loadRepoIntoCache(tmpDirectory + "/repo")
gitRepo = r
}
// cloneRepo Clone git repo
func cloneRepo(repository string) *git.Repository {
if _, err := os.Stat(tmpDirectory); os.IsNotExist(err) {
err = os.Mkdir(tmpDirectory, 0755)
if err != nil {
log.Fatal(err)
}
}
r, err := git.PlainClone(tmpDirectory+"/repo", false, &git.CloneOptions{
URL: repository,
Progress: os.Stdout,
})
if err != nil {
log.Fatal(err)
}
log.Println("Succesfully cloned repo")
return r
}
func checkForUpdates() {
// Fetch updates from remote
pullOptions := git.PullOptions{
RemoteName: "origin",
}
// Get current commit hash PRE PULL
currentHead, err := gitRepo.Head()
if err != nil {
log.Printf("Err getting head: %s", err)
return
}
currentHash := currentHead.Hash()
// Pull latest changes if exist
worktree, err := gitRepo.Worktree()
if err != nil {
log.Printf("Err getting worktree: %s", err)
}
err = worktree.Pull(&pullOptions)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
log.Println("No updates")
return
} else {
log.Printf("Err pulling: %s", err)
return
}
}
// Get current commit hash POST PULL
head, err := gitRepo.Head()
if err != nil {
log.Printf("Err getting new head: %s", err)
return
}
// Get latest commit
latestCommit, err := gitRepo.CommitObject(head.Hash())
if err != nil {
log.Printf("Err getting latest commit: %s", err)
return
}
// Get current commit
currentCommit, err := gitRepo.CommitObject(currentHash)
if err != nil {
log.Printf("Err getting new commit: %s", err)
return
}
// Get patch of changes
patch, err := currentCommit.Patch(latestCommit)
if err != nil {
log.Printf("Err getting change patch: %s", err)
return
}
// Parse git diff
diff, err := diffparser.Parse(patch.String())
if err != nil {
log.Printf("Err parsing diff: %s", err)
return
}
// Loop through file changes
for _, file := range diff.Files {
if strings.HasSuffix(file.NewName, ".csv") {
if strings.HasPrefix(file.NewName, "locations-of-interest/") {
for _, hunk := range file.Hunks {
newRange := hunk.WholeRange
for _, line := range newRange.Lines {
if strings.Contains(line.Content, "Start,End,Advice") {
continue
}
if line.Mode == diffparser.ADDED {
parseCsvRow(line.Content)
}
}
}
}
}
}
// SEND IT o/---->
postTheUpdates()
}

View file

@ -2,14 +2,21 @@ package nzcovidbot
import (
"fmt"
"io/ioutil"
"log"
"os"
"strconv"
"time"
)
var Repository string
// Time of last succesful poll
var lastUpdated time.Time
// Main func
func Lesgoooo() {
// Setup repo stuff
loadRepo(Repository)
// Set last updated poll time!
lastUpdated = getLastPollTime()
log.Printf("Using last updated time: %s", lastUpdated.Local())
// Create chan to end timer
endTicker := make(chan bool)
@ -17,8 +24,8 @@ func Lesgoooo() {
// Timer to run every minute
minuteTicker := time.NewTicker(time.Duration(60) * time.Second)
// Initial check on load
go checkForUpdates()
// Initial poll check on load
go getNewAPILocations()
for {
select {
@ -27,34 +34,39 @@ func Lesgoooo() {
return
case <-minuteTicker.C:
// Check for updates
go checkForUpdates()
go getNewAPILocations()
}
}
}
// getLastPollTime - If run previously, get last TS, otherwise Now()
func getLastPollTime() time.Time {
// Set default of *now* if never run so we don't spam everything
lastUpdated = time.Now()
// Load up last-polled date if set
file, err := os.Open("lastUpdated.txt")
if err == nil {
b, err := ioutil.ReadAll(file)
if err != nil {
log.Printf("Unable to read lastUpdated.txt: %s", err)
}
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
log.Printf("Unable to read lastUpdated.txt: %s", err)
}
lastUpdated = time.Unix(i, 0)
}
return lastUpdated
}
func postTheUpdates() {
// Lets reshuffle our structured data a bit (Exposure Date ASC)
orderRowDataByDate()
// Twitter
go postToTwitter()
// Slack
go postToSlack()
// Discord
postableDiscordData := getPostableDiscordData()
if len(postableDiscordData) == 0 {
return
}
for _, discordWebhook := range DiscordWebhooks {
for _, postableData := range postableDiscordData {
go postToDiscord(discordWebhook, postableData)
time.Sleep(500 * time.Millisecond)
}
}
// Clear out posted data!
updatedLocations = UpdatedLocations{}
go postToDiscord()
}

View file

@ -2,6 +2,7 @@ package nzcovidbot
import (
"fmt"
"net/url"
"strings"
"github.com/ashwanthkumar/slack-go-webhook"
@ -10,6 +11,7 @@ import (
// Slack webhook#channel
var SlackWebhooks []string
// Send slack request
func postToSlack() {
if len(SlackWebhooks) == 0 {
return
@ -40,3 +42,36 @@ func postToSlack() {
}
}
}
// Adds new rows to a slice for slack
func getPostableSlackData() []slack.Attachment {
rows := make([]slack.Attachment, 0)
if len(newLocations.Items) == 0 {
return rows
}
for _, item := range newLocations.Items {
rows = append(rows, getSlackRow(item))
}
return rows
}
// getSlackRow - Get slack attachment row
func getSlackRow(item ApiItem) slack.Attachment {
url := getMapsLinkFromAddress(item.EventName, item.Location.Address)
dateRange := item.getDateString()
attachment := slack.Attachment{
Title: &item.EventName,
TitleLink: &url,
Text: &dateRange,
}
return attachment
}
// getMapsLinkFromAddress hyperlink gmaps
func getMapsLinkFromAddress(name string, address string) string {
return fmt.Sprintf("https://www.google.com/maps/search/?api=1&query=%s", url.QueryEscape(name+", "+address))
}

View file

@ -1,46 +0,0 @@
package nzcovidbot
import (
"log"
"time"
"github.com/dghubble/go-twitter/twitter"
"github.com/dghubble/oauth1"
)
type TwitterCredentials struct {
ConsumerKey string
ConsumerSecret string
AccessToken string
AccessTokenSecret string
}
var TwitterCreds TwitterCredentials
func postToTwitter() {
if TwitterCreds.AccessTokenSecret == "" {
return
}
if len(updatedLocations.Locations) == 0 {
return
}
config := oauth1.NewConfig(TwitterCreds.ConsumerKey, TwitterCreds.ConsumerSecret)
token := oauth1.NewToken(TwitterCreds.AccessToken, TwitterCreds.AccessTokenSecret)
httpClient := config.Client(oauth1.NoContext, token)
// Twitter client
client := twitter.NewClient(httpClient)
// Send a Tweet
for _, row := range updatedLocations.Locations {
_, _, err := client.Statuses.Update(row.TwitterData, nil)
if err != nil {
log.Print(err)
}
// Lets not ratelimit ourselves :upsidedownsmiley:
time.Sleep(1 * time.Second)
}
}