package main import ( "github.com/microcosm-cc/bluemonday" "encoding/json" "crypto/sha1" "io/ioutil" "net/http" "strings" "html" "time" "fmt" "log" ) var p *bluemonday.Policy func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { newposts := make([]ReportPost, 0) min_id := "" http_client := http.Client{} for { ri_mutex.Lock() m := runninginstances[endpoint] ri_mutex.Unlock() api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&min_id=" + min_id resp, err := http_client.Get(api_timeline) m.LastRun = time.Now().Format("2006.01.02-15:04:05") if err != nil { ri_mutex.Lock() m.Status = CLIENT_ISSUE runninginstances[endpoint] = m ri_mutex.Unlock() log.Fatal("Failure here", err.Error()) return } if resp.StatusCode == UNAUTHORIZED || resp.StatusCode == UNPROCESSABLE_ENTITY { // Apparently you just need to do this to throw away the body _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = UNAUTHORIZED runninginstances[endpoint] = m ri_mutex.Unlock() return } else if resp.StatusCode == TOOMANYREQUESTS { // Apparently you just need to do this to throw away the body _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = TOOMANYREQUESTS runninginstances[endpoint] = m ri_mutex.Unlock() time.Sleep(time.Second * 30) continue } else if resp.StatusCode == FORBIDDEN { // Apparently you just need to do this to throw away the body _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = FORBIDDEN runninginstances[endpoint] = m ri_mutex.Unlock() return } else if resp.StatusCode == INTERNAL_ERROR { // Apparently you just need to do this to throw away the body _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = FORBIDDEN runninginstances[endpoint] = m ri_mutex.Unlock() time.Sleep(time.Second * 3600) continue } else if resp.StatusCode == NOT_FOUND { // 404 // Apparently you just need to do this to throw away the body _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = FORBIDDEN runninginstances[endpoint] = m ri_mutex.Unlock() time.Sleep(time.Second * 3600) continue } err = json.NewDecoder(resp.Body).Decode(&newposts) if err != nil { fmt.Println("-----------------") fmt.Println("Error Message 1:", resp.StatusCode, err, err.Error, endpoint, resp.Status) fmt.Println("-----------------") q, err := ioutil.ReadAll(resp.Body) fmt.Println(string(q)) fmt.Println(q) fmt.Println(err) resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = BAD_RESPONSE runninginstances[endpoint] = m ri_mutex.Unlock() log.Fatal("Exiting") continue } resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = RUNNING runninginstances[endpoint] = m ri_mutex.Unlock() for _, newpost := range newposts { if newpost.Account.Acct == "" { continue } posthash := sha1.New() at_sign := strings.Index(newpost.Account.Acct, "@") if at_sign == -1 { at_sign = len(newpost.Account.Acct) newpost.Account.Acct += "@" + endpoint } // Calculate the post hash fmt.Fprint(posthash, newpost.Url) fmt.Fprint(posthash, newpost.normalized) fmt.Fprint(posthash, newpost.Account.Acct) fmt.Fprint(posthash, newpost.Account.Display_name) newpost.posthash = posthash.Sum(nil) newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content))) // Validate time //t, err := time.Parse("2006-01-02T15:04:05.999Z", newpost.Created_at) t, err := time.Parse(time.RFC3339, newpost.Created_at) if err != nil { log.Print("Time was: " + newpost.Created_at) newpost.Created_at = time.Now().Format(time.RFC3339) log.Print("Set to : " + newpost.Created_at) } if t.Unix() < 0 { log.Print("Time was: " + newpost.Created_at) newpost.Created_at = time.Now().Format(time.RFC3339) log.Print("Set to : " + newpost.Created_at) } //t, err = time.Parse("2006-01-02T15:04:05.999Z", newpost.Account.Created_at) t, err = time.Parse(time.RFC3339, newpost.Created_at) if err != nil { log.Print("Time was: " + newpost.Account.Created_at) newpost.Account.Created_at = time.Now().Format(time.RFC3339) log.Print("Set to : " + newpost.Account.Created_at) } if t.Unix() < 0 { log.Print("Time was: " + newpost.Account.Created_at) newpost.Account.Created_at = time.Now().Format(time.RFC3339) log.Print("Set to : " + newpost.Account.Created_at) } reportPostChan <- newpost // Check min_id if newpost.Id > min_id { min_id = newpost.Id } newinstance := newpost.Account.Acct[at_sign+1:] ri_mutex.Lock() _, exists := runninginstances[newinstance] if exists == false { m := RunningInstance{} runninginstances[newinstance] = m go StartInstance(newinstance, reportPostChan) } ri_mutex.Unlock() } time.Sleep(time.Second * 10) } } // Change this to return a proper "err" func GetNodeInfo(endpoint string) (NodeInfo) { /* Checking order * Mastodon/Pleroma * Um..nothing else yet */ pleromastodon_nodeinfo_url := "https://" + endpoint + "/nodeinfo/2.0.json" //http_client := http.Client{Timeout: 10 * time.Second} http_client := http.Client{} pleromastodon_api_resp, err := http_client.Get(pleromastodon_nodeinfo_url) if err != nil { return NodeInfo{} } else { defer pleromastodon_api_resp.Body.Close() } if pleromastodon_api_resp.StatusCode == 200 { var nodeinfo NodeInfo err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo) if err != nil { log.Println("Was not unmarshalled, continuing...") } return nodeinfo } // Check the front page index_url := "https://" + endpoint + "/" resp_index, err := http_client.Get(index_url) if err != nil { log.Fatal("Error Message 2b:", resp_index.StatusCode, err, endpoint, resp_index.Status) } defer resp_index.Body.Close() indexbin, err := ioutil.ReadAll(resp_index.Body) if err != nil { log.Fatal("Error Message 2c:", resp_index.StatusCode, err, endpoint, resp_index.Status) } indexstr := string(indexbin) nodeinfo := NodeInfo{} if strings.Contains(indexstr, "Pleroma") || strings.Contains(indexstr, "Soapbox") { log.Print("Manual view: Pleroma" + endpoint) nodeinfo.Software.Name = "pleroma" nodeinfo.Software.Version = "guess" } else if strings.Contains(indexstr, "Mastodon") { log.Print("Manual view: Mastodon" + endpoint) nodeinfo.Software.Name = "mastodon" nodeinfo.Software.Version = "guess" } else if strings.Contains(indexstr, "Gab") { log.Print("Manual view: Gab" + endpoint) nodeinfo.Software.Name = "gab" nodeinfo.Software.Version = "guess" } return nodeinfo } func StartInstance(endpoint string, reportPostChan chan ReportPost) { nodeinfo := GetNodeInfo(endpoint) if nodeinfo.Software.Name == "" { var m = runninginstances[endpoint] m.Software = "" m.LastRun = time.Now().Format(time.RFC3339) m.Status = UNSUPPORTED_INSTANCE ri_mutex.Lock() runninginstances[endpoint] = m ri_mutex.Unlock() return } if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" { log.Print("Starting " + endpoint + " as Mastodon/Pleroma instance") go PollMastodonPleroma(endpoint, reportPostChan) } }