From a87f8e21fe45c88ed97b0352a5ac5a50720f1a13 Mon Sep 17 00:00:00 2001 From: idanoo Date: Tue, 13 Sep 2022 22:44:53 +1200 Subject: [PATCH] Working internet speeds --- flatfinder.service | 13 ++ go.mod | 4 - go.sum | 13 -- internal/flatfinder/chorus.go | 287 +++++++++++++++++++++++++++------ internal/flatfinder/discord.go | 5 +- internal/flatfinder/utils.go | 7 +- 6 files changed, 259 insertions(+), 70 deletions(-) create mode 100644 flatfinder.service diff --git a/flatfinder.service b/flatfinder.service new file mode 100644 index 0000000..cfd6cdc --- /dev/null +++ b/flatfinder.service @@ -0,0 +1,13 @@ +[Unit] +Description=FlatFinder +After=network.target + +[Service] +ExecStart=/root/flatfinder +WorkingDirectory=/root +Type=simple +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/go.mod b/go.mod index 130c160..d1c2f61 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module flatfinder go 1.19 require ( - github.com/bwmarrin/discordgo v0.26.1 github.com/disgoorg/disgo v0.13.19 github.com/disgoorg/snowflake/v2 v2.0.0 github.com/joho/godotenv v1.4.0 @@ -11,8 +10,5 @@ require ( require ( github.com/disgoorg/log v1.2.0 // indirect - github.com/gorilla/websocket v1.5.0 // indirect github.com/sasha-s/go-csync v0.0.0-20210812194225-61421b77c44b // indirect - golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect - golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect ) diff --git a/go.sum b/go.sum index 2628b59..4e7a8a2 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/bwmarrin/discordgo v0.26.1 h1:AIrM+g3cl+iYBr4yBxCBp9tD9jR3K7upEjl0d89FRkE= -github.com/bwmarrin/discordgo v0.26.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/disgoorg/disgo v0.13.19 h1:evbkWRQ9fU3dIrRJnl+jFTt55cKAeeEwnB/Q3dqgz20= github.com/disgoorg/disgo v0.13.19/go.mod h1:Cyip4bCYHD3rHgDhBPT9cLo81e9AMbDe8ocM50UNRM4= @@ -7,21 +5,10 @@ github.com/disgoorg/log v1.2.0 h1:sqlXnu/ZKAlIlHV9IO+dbMto7/hCQ474vlIdMWk8QKo= github.com/disgoorg/log v1.2.0/go.mod h1:3x1KDG6DI1CE2pDwi3qlwT3wlXpeHW/5rVay+1qDqOo= github.com/disgoorg/snowflake/v2 v2.0.0 h1:+xvyyDddXmXLHmiG8SZiQ3sdZdZPbUR22fSHoqwkrOA= github.com/disgoorg/snowflake/v2 v2.0.0/go.mod h1:SPU9c2CNn5DSyb86QcKtdZgix9osEtKrHLW4rMhfLCs= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/sasha-s/go-csync v0.0.0-20210812194225-61421b77c44b h1:qYTY2tN72LhgDj2rtWG+LI6TXFl2ygFQQ4YezfVaGQE= github.com/sasha-s/go-csync v0.0.0-20210812194225-61421b77c44b/go.mod h1:/pA7k3zsXKdjjAiUhB5CjuKib9KJGCaLvZwtxGC8U0s= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/internal/flatfinder/chorus.go b/internal/flatfinder/chorus.go index dc325bc..1dbdd21 100644 --- a/internal/flatfinder/chorus.go +++ b/internal/flatfinder/chorus.go @@ -1,6 +1,8 @@ package flatfinder import ( + "encoding/json" + "errors" "fmt" "io" "log" @@ -8,92 +10,281 @@ import ( "net/url" ) -type ChorusAddressLookupResponse struct { +type ChorusAddressSearchResponse struct { + Results []struct { + Aid string `json:"aid"` + Label string `json:"label"` + Links []struct { + Rel string `json:"rel"` + Href string `json:"href"` + Method string `json:"method"` + } `json:"links"` + } `json:"results"` } -func chorusAddressLookup(address string) string { - log.Printf("Querying address: %s", address) +type ChorusUniqueIdResponse struct { + FormattedAddress struct { + Line1 string `json:"line1"` + Line2 interface{} `json:"line2"` + Line3 string `json:"line3"` + Line4 string `json:"line4"` + } `json:"formattedAddress"` + StructuredAddress struct { + LevelNumber interface{} `json:"levelNumber"` + LevelType interface{} `json:"levelType"` + StreetNumber int `json:"streetNumber"` + SituationNumber int `json:"situationNumber"` + Suffix interface{} `json:"suffix"` + Unit interface{} `json:"unit"` + UnitType interface{} `json:"unitType"` + StreetName string `json:"streetName"` + RoadType string `json:"roadType"` + RoadAbv string `json:"roadAbv"` + RoadSuffix interface{} `json:"roadSuffix"` + Suburb string `json:"suburb"` + RuralDelivery interface{} `json:"ruralDelivery"` + Town string `json:"town"` + Postcode string `json:"postcode"` + BoxNumber interface{} `json:"boxNumber"` + BoxLobby interface{} `json:"boxLobby"` + BoxType interface{} `json:"boxType"` + Region string `json:"region"` + Country string `json:"country"` + IsPrimary string `json:"isPrimary"` + } `json:"structuredAddress"` + Location struct { + NztmX float64 `json:"nztmX"` + NztmY float64 `json:"nztmY"` + NzmgX float64 `json:"nzmgX"` + NzmgY float64 `json:"nzmgY"` + Wgs84Lat float64 `json:"wgs84Lat"` + Wgs84Lon float64 `json:"wgs84Lon"` + } `json:"location"` + References struct { + Aid string `json:"aid"` + Dpid interface{} `json:"dpid"` + Tui int `json:"tui"` + Tlc int `json:"tlc"` + Plsam int `json:"plsam"` + } `json:"references"` + Related []interface{} `json:"related"` + Links []struct { + Rel string `json:"rel"` + Href string `json:"href"` + Method string `json:"method"` + } `json:"links"` +} + +type ChorusAddressLookupResponse struct { + RegionRsp string `json:"region_rsp"` + SubregionRsp string `json:"subregion_rsp"` + AreaHyperfibre string `json:"area_hyperfibre"` + AlternativeFibreProvider string `json:"alternative_fibre_provider"` + AreaFibreSupplier string `json:"area_fibre_supplier"` + PointOfInterconnect string `json:"point_of_interconnect"` + ProductZoneType string `json:"product_zone_type"` + ActiveServices []struct { + Service string `json:"service"` + SpeedMbps int `json:"speed_mbps"` + SpeedUlMbps int `json:"speed_ul_mbps"` + } `json:"active_services"` + AvailableServices []struct { + Service string `json:"service"` + ServiceIndicator string `json:"service_indicator"` + Capable string `json:"capable"` + SpeedMbps float64 `json:"speed_mbps"` + InstallLeadTimeDays string `json:"install_lead_time_days,omitempty"` + InstallLeadTimeWeeks string `json:"install_lead_time_weeks,omitempty"` + SpeedUlMbps float64 `json:"speed_ul_mbps,omitempty"` + } `json:"available_services"` + FutureServices []interface{} `json:"future_services"` + Fibre struct { + BuildRequired string `json:"build_required"` + ConsentRequired string `json:"consent_required"` + ConsentStatus string `json:"consent_status"` + DesignRequired string `json:"design_required"` + DwellingType string `json:"dwelling_type"` + FibreInADayCapable string `json:"fibre_in_a_day_capable"` + Greenfields string `json:"greenfields"` + IntactOnt string `json:"intact_ont"` + MduBuildStatus string `json:"mdu_build_status"` + MduClass string `json:"mdu_class"` + MduDesignStatus interface{} `json:"mdu_design_status"` + PermitDelayLikely string `json:"permit_delay_likely"` + RightOfWay string `json:"right_of_way"` + } `json:"fibre"` + Copper struct { + PremiseWiringRecommended string `json:"premise_wiring_recommended"` + } `json:"copper"` +} + +// getAvailableSpeeds - Checks if VDSL/FIBRE is available +func getAvailableSpeeds(address string) (string, string) { + aid, err := chorusAddressLookup(address) + if err != nil { + log.Print(err) + return "UNK", "UNK" + } + // log.Printf("Using AID: %s", aid) + + tlc, err := chorusGetUnqiueID(aid) + if err != nil { + log.Print(err) + return "UNK", "UNK" + } + // log.Printf("Using TLC: %d", tlc) + + chorusURL := fmt.Sprintf("https://www.chorus.co.nz/api/bbc/bcc/%d", tlc) + client := http.Client{} + req, err := http.NewRequest("GET", chorusURL, nil) + if err != nil { + log.Print(err) + return "UNK", "UNK" + } + + // Do the request + resp, err := client.Do(req) + if err != nil { + log.Print(err) + return "UNK", "UNK" + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + log.Print(err) + return "UNK", "UNK" + } + + // Decode JSON + var chorusResult ChorusAddressLookupResponse + err = json.Unmarshal(bodyBytes, &chorusResult) + if err != nil { + log.Print(err) + return "UNK", "UNK" + } + + hasFibre := "No" + if chorusResult.Fibre.BuildRequired == "N" { + hasFibre = "Yes" + } + + maxSpeed := 0.0 + for _, available := range chorusResult.AvailableServices { + if available.SpeedMbps > maxSpeed && available.Capable == "YES" { + maxSpeed = available.SpeedMbps + } + } + + current := "None" + for _, active := range chorusResult.ActiveServices { + current = fmt.Sprintf("%s (%d Mbps)", active.Service, active.SpeedMbps) + } + + return fmt.Sprintf("%s (%.0f Mbps)", hasFibre, maxSpeed), current + } else { + log.Print("Invalid response from API: " + resp.Status) + } + + return "UNK", "UNK" +} + +// chorusAddressLookup - Try get the AID for the address +func chorusAddressLookup(address string) (string, error) { lookupURL := fmt.Sprintf( "https://api.chorus.co.nz/addresslookup/v1/addresses/?fuzzy=true&q=%s", url.QueryEscape(address), ) - //curl 'https://api.chorus.co.nz/addresslookup/v1/addresses/?fuzzy=true&q=35%20Rosalind' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0' -H 'Accept: application/json, text/plain, */*' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'X-Chorus-Client-Id: 82d4b4a8050c4d5e97c5f06120ef9c04' -H 'X-Chorus-Client-Secret: 8899c64746474Cf18849c6B721b5Db51' -H 'X-Transaction-Id: ca5ef871-9b3f-4af4-958d-ee9c51094e08' -H 'Origin: https://www.chorus.co.nz' + // Build HTTP request client := http.Client{} req, err := http.NewRequest("GET", lookupURL, nil) if err != nil { - log.Print(err) - return "UNK" + return "", err } // Magic numbers - May need to dynamically receive these req.Header.Set("X-Chorus-Client-Id", "82d4b4a8050c4d5e97c5f06120ef9c04") req.Header.Set("X-Chorus-Client-Secret", "8899c64746474Cf18849c6B721b5Db51") req.Header.Set("X-Transaction-Id", "ca5ef871-9b3f-4af4-958d-ee9c51094e08") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "https://tinker.nz/idanoo/flat-finder") // Do the request resp, err := client.Do(req) if err != nil { - log.Print(err) - return "UNK" + return "", err } defer resp.Body.Close() + // They return a 203 if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNonAuthoritativeInfo { bodyBytes, err := io.ReadAll(resp.Body) if err != nil { - log.Print(err) - return "UNK" + return "", err } - log.Println(string(bodyBytes)) - return "N/A" - } else { - log.Print("Invalid response from API: " + resp.Status) + // Decode JSON + var chorusResult ChorusAddressSearchResponse + err = json.Unmarshal(bodyBytes, &chorusResult) + if err != nil { + return "", err + } + + // If we have a result, return the first one + for _, result := range chorusResult.Results { + return result.Aid, nil + } + + return "", errors.New("No results found for address: " + address) } - return "UNK" + return "", errors.New("Invalid response from API: " + resp.Status) } -// getAvailableSpeeds - Checks if VDSL/FIBRE is available -func getAvailableSpeeds(address string) string { - return chorusAddressLookup(address) +// chorusGetUniqueID - Return ID needed to get avail services +func chorusGetUnqiueID(aid string) (int64, error) { + lookupURL := fmt.Sprintf( + "https://api.chorus.co.nz/addresslookup/v1/addresses/aid:%s", + aid, + ) - // // Build HTTP request - // client := http.Client{} - // req, err := http.NewRequest("GET", chorusURL, nil) - // if err != nil { - // log.Print(err) - // return "UNK" - // } + // Build HTTP request + client := http.Client{} + req, err := http.NewRequest("GET", lookupURL, nil) + if err != nil { + return 0, err + } - // req.Header.Set("Content-TypeContent-Type", "application/json") - // req.Header.Set("User-Agent", "https://tinker.nz/idanoo/flat-finder") + // Magic numbers - May need to dynamically receive these + req.Header.Set("X-Chorus-Client-Id", "82d4b4a8050c4d5e97c5f06120ef9c04") + req.Header.Set("X-Chorus-Client-Secret", "8899c64746474Cf18849c6B721b5Db51") + req.Header.Set("X-Transaction-Id", "ca5ef871-9b3f-4af4-958d-ee9c51094e08") + req.Header.Set("Content-Type", "application/json") - // // Do the request - // resp, err := client.Do(req) - // if err != nil { - // log.Print(err) - // return "UNK" - // } - // defer resp.Body.Close() + // Do the request + resp, err := client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() - // if resp.StatusCode == http.StatusOK { - // bodyBytes, err := io.ReadAll(resp.Body) - // if err != nil { - // log.Print(err) - // return "UNK" - // } + // They return a 203 + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNonAuthoritativeInfo { + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return 0, err + } - // log.Println(string(bodyBytes)) + // Decode JSON + var chorusResult ChorusUniqueIdResponse + err = json.Unmarshal(bodyBytes, &chorusResult) + if err != nil { + return 0, err + } - // return "N/A" - // } else { - // log.Print("Invalid response from API: " + resp.Status) - // } + return int64(chorusResult.References.Tlc), nil + } - return "UNK" + return 0, errors.New("Invalid response from API: " + resp.Status) } diff --git a/internal/flatfinder/discord.go b/internal/flatfinder/discord.go index 21d4421..bb3cf79 100644 --- a/internal/flatfinder/discord.go +++ b/internal/flatfinder/discord.go @@ -37,7 +37,7 @@ func (c *LocalConfig) initDiscord() { func (c *LocalConfig) sendEmbeddedMessage(listing TradeMeListing) { log.Printf("New listing: %s", listing.Title) - hasFibre := getAvailableSpeeds( + hasFibre, currentConn := getAvailableSpeeds( fmt.Sprintf( "%s, %s, %s", strings.TrimSpace(listing.Address), @@ -62,7 +62,8 @@ func (c *LocalConfig) sendEmbeddedMessage(listing TradeMeListing) { true, ). AddField("Bedrooms", fmt.Sprintf("%d", listing.Bedrooms), true). - AddField("Has Fibre", hasFibre, true) + AddField("Fibre Avail", hasFibre, false). + AddField("Current Connection", currentConn, false) // Only add address if token set if c.GoogleApiToken != "" && c.GoogleLocation1 != "" { diff --git a/internal/flatfinder/utils.go b/internal/flatfinder/utils.go index d82144c..fc09f20 100644 --- a/internal/flatfinder/utils.go +++ b/internal/flatfinder/utils.go @@ -34,10 +34,11 @@ func (c *LocalConfig) loadConfig() { // Load it into global err = json.Unmarshal(data, c) if err != nil { - log.Fatal(err) + maps := make(map[int64]bool) + c.PostedProperties = maps + } else { + log.Printf("Loaded %d previously posted property IDs", len(c.PostedProperties)) } - - log.Printf("Loaded %d previously posted property IDs", len(c.PostedProperties)) } else { // Create empty map for first run maps := make(map[int64]bool)