package main import ( "github.com/microcosm-cc/bluemonday" "encoding/json" "crypto/sha1" "io/ioutil" "net/http" "strings" "html" "time" "fmt" "os" ) func DeferPollRun(instancereport InstanceReport, runninginstances *[]RunningInstance, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) { delay := 10 if instancereport.status == RUNNING && instancereport.numposts <= 10 { delay = 10 } else if instancereport.status == RUNNING && instancereport.numposts > 10 { delay = 15 } else if instancereport.status == 429 { delay = 30 } else { fmt.Println("error, status code is ------------->: ", instancereport.status) os.Exit(1) } time.Sleep(time.Second * time.Duration(delay)) go StartInstancePoll(instancereport, reportPostChan, instanceReportChan) } func StartInstancePoll(instancereport InstanceReport, reportPostChan chan ReportPost, instanceReportChan chan InstanceReport) { p := bluemonday.NewPolicy() newposts := make([]ReportPost, 0) // Only placing this here to later have the option of using // an HTTP client via a SOCKS5 Tor proxy if strings.Contains(instancereport.endpoint, ".onion") { instanceReportChan <- InstanceReport{instancereport.endpoint, ONION_PROTOCOL, "", 0} return } api_timeline := "https://" + instancereport.endpoint + "/api/v1/timelines/public?limit=40&min_id=" + instancereport.min_id resp, err := http.Get(api_timeline) if err != nil { defer resp.Body.Close() instanceReportChan <- InstanceReport{instancereport.endpoint, CLIENT_ISSUE, "", 0} return } body, err := ioutil.ReadAll(resp.Body) if err != nil { instanceReportChan <- InstanceReport{instancereport.endpoint, BAD_RESPONSE, "", 0} return } err = json.Unmarshal(body, &newposts) if err != nil { instanceReportChan <- InstanceReport{instancereport.endpoint, BAD_RESPONSE, "", 0} return } newinstances := make([]string, 0) min_id := "" numposts := 0 for _, newpost := range newposts { if newpost.Account.Acct == "" { continue } sendpost := newpost posthash := sha1.New() at_sign := strings.Index(sendpost.Account.Acct, "@") if at_sign == -1 { at_sign = len(sendpost.Account.Acct) sendpost.Account.Acct += "@" + instancereport.endpoint } // Calculate the post hash fmt.Fprint(posthash, sendpost.Url) fmt.Fprint(posthash, sendpost.normalized) fmt.Fprint(posthash, sendpost.Account.Acct) fmt.Fprint(posthash, sendpost.Account.Display_name) sendpost.posthash = posthash.Sum(nil) sendpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(sendpost.Content))) reportPostChan <- sendpost // Check min_id if sendpost.Id > min_id { min_id = sendpost.Id } numposts = numposts + 1 newinstance := sendpost.Account.Acct[at_sign+1:] newinstances = AppendIfMissing(newinstances, newinstance) } for _, newinstance := range newinstances { var q InstanceReport q.endpoint = newinstance q.status = NEW_INSTANCE q.min_id = "" q.numposts = 0 instanceReportChan <- q } instanceReportChan <- InstanceReport{instancereport.endpoint, resp.StatusCode, min_id, numposts} } // Change this to return a proper "err" func GetNodeInfo(endpoint string, nodeinfo *NodeInfo) { api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json" http_client := http.Client{Timeout: 5 * time.Second} resp, err := http_client.Get(api_nodeinfo) if err != nil { defer resp.Body.Close() return } body, err := ioutil.ReadAll(resp.Body) err = json.Unmarshal(body, &nodeinfo) if err != nil { // fmt.Println("Unmarshal 2"); return } } func NewInstance(endpoint string, runninginstances *[]RunningInstance, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) { var nodeinfo NodeInfo if endpoint == "" { return } // No repeats for _, runninginstance := range *runninginstances { if runninginstance.Endpoint == endpoint { return } } // Check node type GetNodeInfo(endpoint, &nodeinfo) if nodeinfo.Software.Name == "" { go func() { var q InstanceReport q.endpoint = endpoint q.status = BAD_NODEINFO instanceReportChan <- q }() } newinstance := RunningInstance{endpoint, "", "", NEW_INSTANCE, "Queued"} *runninginstances = append(*runninginstances, newinstance) if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" { var newinstancereport InstanceReport newinstancereport.endpoint = endpoint newinstancereport.status = 0 newinstancereport.min_id = "" newinstancereport.numposts = 0 go StartInstancePoll(newinstancereport, reportPostChan, instanceReportChan) } } func SuspendInstance(suspendinstance InstanceReport, runninginstances *[]RunningInstance) { for i, runninginstance := range *runninginstances { if runninginstance.Endpoint == suspendinstance.endpoint { (*runninginstances)[i].Status = suspendinstance.status (*runninginstances)[i].LastRun = time.Now().Format("2006.01.02-15:04:05") return } } }