package main import ( "github.com/microcosm-cc/bluemonday" "encoding/json" "crypto/sha1" "io/ioutil" "net/http" "strings" "html" "time" "fmt" ) /* func DeferPollRun(instancereport InstanceReport, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) { delay := 10 if instancereport.status == RUNNING && instancereport.numposts <= 10 { delay = 10 } else if instancereport.status == RUNNING && instancereport.numposts > 10 { delay = 15 } else if instancereport.status == 429 { delay = 30 } else { fmt.Println("error, status code is ------------->: ", instancereport.status) os.Exit(1) } time.Sleep(time.Second * time.Duration(delay)) StartInstancePoll(instancereport, reportPostChan, instanceReportChan) } */ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { fmt.Println("Arrived at PollMastodonPleroma for " + endpoint) // Make this a global variable p := bluemonday.NewPolicy() newposts := make([]ReportPost, 0) min_id := "" http_client := http.Client{Timeout: 5 * time.Second} for { api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&min_id=" + min_id fmt.Println(api_timeline) numposts := 0 // newinstances := make([]string, 0) resp, err := http_client.Get(api_timeline) if err != nil { ri_mutex.Lock() var m = runninginstances[endpoint] m.Status = CLIENT_ISSUE runninginstances[endpoint] = m ri_mutex.Unlock() return } body, err := ioutil.ReadAll(resp.Body) if err != nil { ri_mutex.Lock() var m = runninginstances[endpoint] m.Status = BAD_RESPONSE runninginstances[endpoint] = m ri_mutex.Unlock() return } err = json.Unmarshal(body, &newposts) if err != nil { ri_mutex.Lock() var m = runninginstances[endpoint] m.Status = BAD_RESPONSE runninginstances[endpoint] = m ri_mutex.Unlock() return } resp.Body.Close() for _, newpost := range newposts { if newpost.Account.Acct == "" { continue } posthash := sha1.New() at_sign := strings.Index(newpost.Account.Acct, "@") if at_sign == -1 { at_sign = len(newpost.Account.Acct) newpost.Account.Acct += "@" + endpoint } // Calculate the post hash fmt.Fprint(posthash, newpost.Url) fmt.Fprint(posthash, newpost.normalized) fmt.Fprint(posthash, newpost.Account.Acct) fmt.Fprint(posthash, newpost.Account.Display_name) newpost.posthash = posthash.Sum(nil) newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content))) reportPostChan <- newpost // Check min_id if newpost.Id > min_id { min_id = newpost.Id } numposts = numposts + 1 newinstance := newpost.Account.Acct[at_sign+1:] ri_mutex.Lock() _, exists := runninginstances[newinstance] if exists == false { m := RunningInstance{} runninginstances[newinstance] = m go StartInstance(newinstance, reportPostChan) } ri_mutex.Unlock() } time.Sleep(time.Second * 10) } } // Change this to return a proper "err" func GetNodeInfo(endpoint string) (NodeInfo) { var nodeinfo NodeInfo api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json" http_client := http.Client{Timeout: 5 * time.Second} resp, err := http_client.Get(api_nodeinfo) if err != nil { fmt.Println("Make a legit error here") return NodeInfo{} } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) err = json.Unmarshal(body, &nodeinfo) if err != nil { fmt.Println("Make a legit error here") fmt.Println("Unmarshal 2"); return NodeInfo{} } return nodeinfo } func StartInstance(endpoint string, reportPostChan chan ReportPost) { // This might not be necessary... // ri_mutex.Lock() // _, exists := runninginstances[endpoint] // fmt.Println("The exists is", exists) // ri_mutex.Unlock() // if exists == true { // return // } nodeinfo := GetNodeInfo(endpoint) if nodeinfo.Software.Name == "" { ri_mutex.Lock() var m = runninginstances[endpoint] m.Software = "" runninginstances[endpoint] = m ri_mutex.Unlock() return } if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" { go PollMastodonPleroma(endpoint, reportPostChan) } } /* func NewInstance(endpoint string, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) { var nodeinfo NodeInfo if endpoint == "" { return } // No repeats for _, runninginstance := range runninginstances { if runninginstance.Endpoint == endpoint { return } } // Check node type GetNodeInfo(endpoint, &nodeinfo) if nodeinfo.Software.Name == "" { go func() { var q InstanceReport q.endpoint = endpoint q.status = BAD_NODEINFO instanceReportChan <- q }() } newinstance := RunningInstance{endpoint, "", "", NEW_INSTANCE, "Queued"} runninginstances = append(runninginstances, newinstance) if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" { var newinstancereport InstanceReport newinstancereport.endpoint = endpoint newinstancereport.status = 0 newinstancereport.min_id = "" newinstancereport.numposts = 0 go StartInstancePoll(newinstancereport, reportPostChan, instanceReportChan) } } */ /* func SuspendInstance(suspendinstance InstanceReport) { for i, runninginstance := range runninginstances { if runninginstance.Endpoint == suspendinstance.endpoint { (runninginstances)[i].Status = suspendinstance.status (runninginstances)[i].LastRun = time.Now().Format("2006.01.02-15:04:05") return } } } */