package main import ( "github.com/microcosm-cc/bluemonday" "encoding/json" "crypto/sha1" "io/ioutil" "net/http" "strings" "html" "time" "fmt" ) var p *bluemonday.Policy func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { // Make this a global variable //p := bluemonday.NewPolicy() newposts := make([]ReportPost, 0) min_id := "" http_client := http.Client{Timeout: 5 * time.Second} for { ri_mutex.Lock() m := runninginstances[endpoint] ri_mutex.Unlock() api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&min_id=" + min_id resp, err := http_client.Get(api_timeline) m.LastRun = time.Now().Format("2006.01.02-15:04:05") if err != nil { ri_mutex.Lock() m.Status = CLIENT_ISSUE runninginstances[endpoint] = m ri_mutex.Unlock() return } if resp.StatusCode == UNAUTHORIZED || resp.StatusCode == UNPROCESSABLE_ENTITY { _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = UNAUTHORIZED runninginstances[endpoint] = m ri_mutex.Unlock() return } else if resp.StatusCode == TOOMANYREQUESTS { // Apparently you just need to do this to throw away the body _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = TOOMANYREQUESTS runninginstances[endpoint] = m ri_mutex.Unlock() time.Sleep(time.Second * 30) continue } err = json.NewDecoder(resp.Body).Decode(&newposts) if err != nil { fmt.Println("Error Message 1:", resp.StatusCode, err, endpoint, resp.Status) ri_mutex.Lock() m.Status = BAD_RESPONSE runninginstances[endpoint] = m ri_mutex.Unlock() return } resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = RUNNING runninginstances[endpoint] = m ri_mutex.Unlock() for _, newpost := range newposts { if newpost.Account.Acct == "" { continue } posthash := sha1.New() at_sign := strings.Index(newpost.Account.Acct, "@") if at_sign == -1 { at_sign = len(newpost.Account.Acct) newpost.Account.Acct += "@" + endpoint } // Calculate the post hash fmt.Fprint(posthash, newpost.Url) fmt.Fprint(posthash, newpost.normalized) fmt.Fprint(posthash, newpost.Account.Acct) fmt.Fprint(posthash, newpost.Account.Display_name) newpost.posthash = posthash.Sum(nil) newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content))) // Validate time _, err := time.Parse("2006-01-02T15:04:05.999Z", newpost.Created_at) if err != nil { fmt.Println("---------> FIXED TIME", newpost.Created_at) newpost.Created_at = time.Now().Format("2006.01.02-15:04:05") } reportPostChan <- newpost // Check min_id if newpost.Id > min_id { min_id = newpost.Id } newinstance := newpost.Account.Acct[at_sign+1:] ri_mutex.Lock() _, exists := runninginstances[newinstance] if exists == false { m := RunningInstance{} runninginstances[newinstance] = m go StartInstance(newinstance, reportPostChan) } ri_mutex.Unlock() } time.Sleep(time.Second * 10) } } // Change this to return a proper "err" func GetNodeInfo(endpoint string) (NodeInfo) { var nodeinfo NodeInfo /* Not everyone implements this */ api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json" http_client := http.Client{Timeout: 10 * time.Second} resp_node, err := http_client.Get(api_nodeinfo) if err != nil { fmt.Println("Make a legit error here", err) return NodeInfo{} } defer resp_node.Body.Close() /* Some Pleroma and Mastodon instances hide their /nodeinfo/2.0.json So this attempts to check the index URL for references to Mastodon or Pleroma. */ if resp_node.StatusCode == 200 { err = json.NewDecoder(resp_node.Body).Decode(&nodeinfo) if err != nil { fmt.Println("Error Message 2:", resp_node.StatusCode, err, endpoint, resp_node.Status, api_nodeinfo) return NodeInfo{} } } if resp_node.StatusCode == 404 { indexurl := "https://" + endpoint + "/" resp_index, err := http_client.Get(indexurl) defer resp_index.Body.Close() if err != nil { fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo) return NodeInfo{} } indexbin, err := ioutil.ReadAll(resp_index.Body) if err != nil { fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo) return NodeInfo{} } indexstr := string(indexbin) if strings.Contains(indexstr, "Pleroma") { nodeinfo.Software.Name = "pleroma" fmt.Println("It is Pleroma: " + endpoint) } else if strings.Contains(indexstr, "Mastodon") { nodeinfo.Software.Name = "mastodon" fmt.Println("It is Mastodon: " + endpoint) } else { return NodeInfo{} } } /* err = json.NewDecoder(resp.Body).Decode(&nodeinfo) if err != nil { fmt.Println("Error Message 2:", resp.StatusCode, err, endpoint, resp.Status, api_nodeinfo) return NodeInfo{} } */ return nodeinfo } func StartInstance(endpoint string, reportPostChan chan ReportPost) { nodeinfo := GetNodeInfo(endpoint) if nodeinfo.Software.Name == "" { var m = runninginstances[endpoint] m.Software = "" m.LastRun = time.Now().Format("2006.01.02-15:04:05") m.Status = UNSUPPORTED_INSTANCE ri_mutex.Lock() runninginstances[endpoint] = m ri_mutex.Unlock() return } if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" { go PollMastodonPleroma(endpoint, reportPostChan) } } /* func SuspendInstance(suspendinstance InstanceReport) { for i, runninginstance := range runninginstances { if runninginstance.Endpoint == suspendinstance.endpoint { (runninginstances)[i].Status = suspendinstance.status (runninginstances)[i].LastRun = time.Now().Format("2006.01.02-15:04:05") return } } } */