fedilogue/poll/instance.go

234 lines
5.4 KiB
Go
Raw Normal View History

package main
import (
"github.com/microcosm-cc/bluemonday"
"encoding/json"
"crypto/sha1"
"io/ioutil"
"net/http"
"strings"
"html"
"time"
"fmt"
)
/*
func DeferPollRun(instancereport InstanceReport, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) {
delay := 10
if instancereport.status == RUNNING && instancereport.numposts <= 10 {
delay = 10
} else if instancereport.status == RUNNING && instancereport.numposts > 10 {
delay = 15
} else if instancereport.status == 429 {
delay = 30
} else {
fmt.Println("error, status code is ------------->: ", instancereport.status)
os.Exit(1)
}
time.Sleep(time.Second * time.Duration(delay))
2020-11-16 00:10:33 -05:00
StartInstancePoll(instancereport, reportPostChan, instanceReportChan)
}
*/
func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
fmt.Println("Arrived at PollMastodonPleroma for " + endpoint)
// Make this a global variable
p := bluemonday.NewPolicy()
newposts := make([]ReportPost, 0)
min_id := ""
http_client := http.Client{Timeout: 5 * time.Second}
for {
api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&min_id=" + min_id
fmt.Println(api_timeline)
numposts := 0
// newinstances := make([]string, 0)
resp, err := http_client.Get(api_timeline)
if err != nil {
ri_mutex.Lock()
var m = runninginstances[endpoint]
m.Status = CLIENT_ISSUE
runninginstances[endpoint] = m
ri_mutex.Unlock()
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
ri_mutex.Lock()
var m = runninginstances[endpoint]
m.Status = BAD_RESPONSE
runninginstances[endpoint] = m
ri_mutex.Unlock()
return
}
err = json.Unmarshal(body, &newposts)
if err != nil {
ri_mutex.Lock()
var m = runninginstances[endpoint]
m.Status = BAD_RESPONSE
runninginstances[endpoint] = m
ri_mutex.Unlock()
return
}
resp.Body.Close()
for _, newpost := range newposts {
if newpost.Account.Acct == "" {
continue
}
posthash := sha1.New()
at_sign := strings.Index(newpost.Account.Acct, "@")
if at_sign == -1 {
at_sign = len(newpost.Account.Acct)
newpost.Account.Acct += "@" + endpoint
}
// Calculate the post hash
fmt.Fprint(posthash, newpost.Url)
fmt.Fprint(posthash, newpost.normalized)
fmt.Fprint(posthash, newpost.Account.Acct)
fmt.Fprint(posthash, newpost.Account.Display_name)
newpost.posthash = posthash.Sum(nil)
newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content)))
reportPostChan <- newpost
// Check min_id
if newpost.Id > min_id {
min_id = newpost.Id
}
numposts = numposts + 1
newinstance := newpost.Account.Acct[at_sign+1:]
ri_mutex.Lock()
_, exists := runninginstances[newinstance]
if exists == false {
m := RunningInstance{}
runninginstances[newinstance] = m
go StartInstance(newinstance, reportPostChan)
}
ri_mutex.Unlock()
}
time.Sleep(time.Second * 10)
}
}
// Change this to return a proper "err"
func GetNodeInfo(endpoint string) (NodeInfo) {
var nodeinfo NodeInfo
api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json"
http_client := http.Client{Timeout: 5 * time.Second}
resp, err := http_client.Get(api_nodeinfo)
if err != nil {
fmt.Println("Make a legit error here")
return NodeInfo{}
}
2020-11-16 00:10:33 -05:00
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &nodeinfo)
if err != nil {
fmt.Println("Make a legit error here")
fmt.Println("Unmarshal 2");
return NodeInfo{}
}
return nodeinfo
}
func StartInstance(endpoint string, reportPostChan chan ReportPost) {
// This might not be necessary...
// ri_mutex.Lock()
// _, exists := runninginstances[endpoint]
// fmt.Println("The exists is", exists)
// ri_mutex.Unlock()
// if exists == true {
// return
// }
nodeinfo := GetNodeInfo(endpoint)
if nodeinfo.Software.Name == "" {
ri_mutex.Lock()
var m = runninginstances[endpoint]
m.Software = ""
runninginstances[endpoint] = m
ri_mutex.Unlock()
return
}
if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" {
go PollMastodonPleroma(endpoint, reportPostChan)
}
}
/*
func NewInstance(endpoint string, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) {
var nodeinfo NodeInfo
if endpoint == "" {
return
}
// No repeats
for _, runninginstance := range runninginstances {
if runninginstance.Endpoint == endpoint {
return
}
}
// Check node type
GetNodeInfo(endpoint, &nodeinfo)
if nodeinfo.Software.Name == "" {
go func() {
var q InstanceReport
q.endpoint = endpoint
q.status = BAD_NODEINFO
instanceReportChan <- q
}()
}
newinstance := RunningInstance{endpoint, "", "", NEW_INSTANCE, "Queued"}
runninginstances = append(runninginstances, newinstance)
if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" {
var newinstancereport InstanceReport
newinstancereport.endpoint = endpoint
newinstancereport.status = 0
newinstancereport.min_id = ""
newinstancereport.numposts = 0
go StartInstancePoll(newinstancereport, reportPostChan, instanceReportChan)
}
}
*/
/*
func SuspendInstance(suspendinstance InstanceReport) {
for i, runninginstance := range runninginstances {
if runninginstance.Endpoint == suspendinstance.endpoint {
(runninginstances)[i].Status = suspendinstance.status
(runninginstances)[i].LastRun = time.Now().Format("2006.01.02-15:04:05")
return
}
}
}
*/