package main import ( "encoding/json" "io/ioutil" "log" "net/http" "time" ) type ImageData struct { Type string `"json:type"` Url string `"json:url"` } type PublicKeyData struct { Id string `"json:id"` Owner string `"json:owner"` PublicKeyPem string `"json:publicKeyPem"` } type UserInfo struct { Id string `"json:id"` Type string `"json:type"` Following string `"json:following"` Followers string `"json:followers"` Inbox string `"json:inbox"` Outbox string `"json:outbox"` Featured string `"json:featured"` PreferredUsername string `"json:preferredUsername"` PublicKey PublicKeyData `"json:publicKeyPem"` Name string `"json:name"` Summary string `"json:summary"` Url string `"json:Url"` Icon ImageData `"json:icon"` Image ImageData `"json:image"` // ManuallyApprovesFollowers string `"json:manuallyApprovesFollowers"` // Discoverable bool `"json:discoverable"` } type PostInfo struct { Id string `"json:id"` Type string `"json:type"` Published string `"json:published"` Url string `"json:Url"` Content string `"json:content"` } func PollMastodonPleroma(endpoint string, o *RunningInstance) { newposts := make([]ReportPost, 0) min_id := "" parsing_error := 0 unprocess_error := 0 use_auth := false var last_refresh int64 var client_id string var client_secret string var oauthData OAuth var err error for _, extaccount := range settings.Externalaccounts { if extaccount.Endpoint == endpoint { use_auth = true o := GetRunner(endpoint) err = get_client(endpoint, &o) if err != nil { log.Print("Unable to register client: ", err) return } oauthData, err = oauth_login(endpoint, &o, extaccount.Username, extaccount.Password) if err != nil { log.Print("Unable to login: ", err) return } last_refresh = time.Now().Unix() } } for { ri_mutex.Lock() m := runninginstances[endpoint] ri_mutex.Unlock() api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&since_id=" + min_id req, err := http.NewRequest("GET", api_timeline, nil) if err != nil { log.Print("Unable to create new request") return } if use_auth == true { if time.Now().Unix() > last_refresh+oauthData.Expires_in { oauthData, err = oauth_refresh(endpoint, client_id, client_secret, oauthData.Refresh_token) if err != nil { log.Print("Unable to refresh: ", err) return } last_refresh = time.Now().Unix() } req.Header.Add("Authorization", oauthData.Access_token) } m.LastRun = time.Now().Format(time.RFC3339) resp, err := o.client.Do(req) if err != nil { m.Status = CLIENT_ISSUE ri_mutex.Lock() runninginstances[endpoint] = m ri_mutex.Unlock() return } if resp.StatusCode == TOOMANYREQUESTS { // Short Delay, 30 seconds log.Print("Delaying "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay") _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done m.Status = resp.StatusCode ri_mutex.Lock() runninginstances[endpoint] = m ri_mutex.Unlock() if unprocess_error > 5 { log.Print("Exiting for " + endpoint) } unprocess_error = unprocess_error + 1 time.Sleep(time.Second * 30) continue } else if resp.StatusCode == INTERNAL_ERROR { // Longer delay, 1 hour log.Print("Suspending "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay") _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done m.Status = 765 ri_mutex.Lock() runninginstances[endpoint] = m ri_mutex.Unlock() time.Sleep(time.Second * 3600) continue } else if resp.StatusCode != 200 { // Crash log.Print("Terminating "+endpoint+", gave status ", resp.StatusCode) _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done m.Status = resp.StatusCode ri_mutex.Lock() runninginstances[endpoint] = m ri_mutex.Unlock() return } err = json.NewDecoder(resp.Body).Decode(&newposts) if err != nil { if parsing_error > 5 { m.Status = BAD_RESPONSE ri_mutex.Lock() runninginstances[endpoint] = m ri_mutex.Unlock() log.Print("Giving up on " + endpoint) return } parsing_error = parsing_error + 1 time.Sleep(time.Second * 30) } resp.Body.Close() // Release as soon as done m.Status = RUNNING ri_mutex.Lock() runninginstances[endpoint] = m ri_mutex.Unlock() for _, newpost := range newposts { go check_post(newpost.Uri) matchset := re.FindStringSubmatch(newpost.Uri) if matchset != nil { newinstance := matchset[1] // Check min_id if newpost.Id > min_id { min_id = newpost.Id } // Only done if we are crawling go CheckInstance(newinstance, endpoint) } } time.Sleep(time.Second * 10) } }