diff --git a/poll/headers.go b/poll/headers.go index 6c9e920..aa111cb 100644 --- a/poll/headers.go +++ b/poll/headers.go @@ -4,8 +4,11 @@ const ( NEW_INSTANCE = 0 RUNNING = 200 UNAUTHORIZED = 401 + FORBIDDEN = 403 + NOT_FOUND = 404 UNPROCESSABLE_ENTITY = 422 TOOMANYREQUESTS = 429 + INTERNAL_ERROR = 500 CLIENT_ISSUE = 600 ONION_PROTOCOL = 601 BAD_RESPONSE = 602 diff --git a/poll/instance.go b/poll/instance.go index e60ffc7..6ef7bc4 100644 --- a/poll/instance.go +++ b/poll/instance.go @@ -20,7 +20,7 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { min_id := "" - http_client := http.Client{Timeout: 5 * time.Second} + http_client := http.Client{} for { ri_mutex.Lock() @@ -35,10 +35,12 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { m.Status = CLIENT_ISSUE runninginstances[endpoint] = m ri_mutex.Unlock() + log.Fatal("Failure here", err.Error()) return } if resp.StatusCode == UNAUTHORIZED || resp.StatusCode == UNPROCESSABLE_ENTITY { + // Apparently you just need to do this to throw away the body _, _ = ioutil.ReadAll(resp.Body) resp.Body.Close() // Release as soon as done ri_mutex.Lock() @@ -56,15 +58,53 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { ri_mutex.Unlock() time.Sleep(time.Second * 30) continue + } else if resp.StatusCode == FORBIDDEN { + // Apparently you just need to do this to throw away the body + _, _ = ioutil.ReadAll(resp.Body) + resp.Body.Close() // Release as soon as done + ri_mutex.Lock() + m.Status = FORBIDDEN + runninginstances[endpoint] = m + ri_mutex.Unlock() + return + } else if resp.StatusCode == INTERNAL_ERROR { + // Apparently you just need to do this to throw away the body + _, _ = ioutil.ReadAll(resp.Body) + resp.Body.Close() // Release as soon as done + ri_mutex.Lock() + m.Status = FORBIDDEN + runninginstances[endpoint] = m + ri_mutex.Unlock() + time.Sleep(time.Second * 3600) + continue + } else if resp.StatusCode == NOT_FOUND { // 404 + // Apparently you just need to do this to throw away the body + _, _ = ioutil.ReadAll(resp.Body) + resp.Body.Close() // Release as soon as done + ri_mutex.Lock() + m.Status = FORBIDDEN + runninginstances[endpoint] = m + ri_mutex.Unlock() + time.Sleep(time.Second * 3600) + continue } + err = json.NewDecoder(resp.Body).Decode(&newposts) if err != nil { - fmt.Println("Error Message 1:", resp.StatusCode, err, endpoint, resp.Status) + fmt.Println("-----------------") + fmt.Println("Error Message 1:", resp.StatusCode, err, err.Error, endpoint, resp.Status) + fmt.Println("-----------------") + q, err := ioutil.ReadAll(resp.Body) + fmt.Println(string(q)) + fmt.Println(q) + fmt.Println(err) + resp.Body.Close() // Release as soon as done ri_mutex.Lock() m.Status = BAD_RESPONSE runninginstances[endpoint] = m ri_mutex.Unlock() - return + log.Fatal("Exiting") + continue } resp.Body.Close() // Release as soon as done @@ -96,10 +136,30 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content))) // Validate time - _, err := time.Parse("2006-01-02T15:04:05.999Z", newpost.Created_at) + //t, err := time.Parse("2006-01-02T15:04:05.999Z", newpost.Created_at) + t, err := time.Parse(time.RFC3339, newpost.Created_at) if err != nil { - fmt.Println("---------> FIXED TIME", newpost.Created_at) - newpost.Created_at = time.Now().Format("2006.01.02-15:04:05") + log.Print("Time was: " + newpost.Created_at) + newpost.Created_at = time.Now().Format(time.RFC3339) + log.Print("Set to : " + newpost.Created_at) + } + if t.Unix() < 0 { + log.Print("Time was: " + newpost.Created_at) + newpost.Created_at = time.Now().Format(time.RFC3339) + log.Print("Set to : " + newpost.Created_at) + } + + //t, err = time.Parse("2006-01-02T15:04:05.999Z", newpost.Account.Created_at) + t, err = time.Parse(time.RFC3339, newpost.Created_at) + if err != nil { + log.Print("Time was: " + newpost.Account.Created_at) + newpost.Account.Created_at = time.Now().Format(time.RFC3339) + log.Print("Set to : " + newpost.Account.Created_at) + } + if t.Unix() < 0 { + log.Print("Time was: " + newpost.Account.Created_at) + newpost.Account.Created_at = time.Now().Format(time.RFC3339) + log.Print("Set to : " + newpost.Account.Created_at) } reportPostChan <- newpost @@ -125,51 +185,54 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { // Change this to return a proper "err" func GetNodeInfo(endpoint string) (NodeInfo) { - var nodeinfo NodeInfo - - /* Not everyone implements this */ - api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json" - http_client := http.Client{Timeout: 10 * time.Second} - resp_node, err := http_client.Get(api_nodeinfo) - if err != nil { - fmt.Println("Make a legit error here", err) - return NodeInfo{} - } - defer resp_node.Body.Close() - - /* - Some Pleroma and Mastodon instances hide their /nodeinfo/2.0.json - So this attempts to check the index URL for references to Mastodon - or Pleroma. + /* Checking order + * Mastodon/Pleroma + * Um..nothing else yet */ - if resp_node.StatusCode == 200 { - err = json.NewDecoder(resp_node.Body).Decode(&nodeinfo) - if err != nil { - fmt.Println("Error Message 2:", resp_node.StatusCode, err, endpoint, resp_node.Status, api_nodeinfo) - return NodeInfo{} - } + pleromastodon_nodeinfo_url := "https://" + endpoint + "/nodeinfo/2.0.json" + //http_client := http.Client{Timeout: 10 * time.Second} + http_client := http.Client{} + pleromastodon_api_resp, err := http_client.Get(pleromastodon_nodeinfo_url) + if err != nil { + return NodeInfo{} + } else { + defer pleromastodon_api_resp.Body.Close() } - if resp_node.StatusCode == 404 { - indexurl := "https://" + endpoint + "/" - resp_index, err := http_client.Get(indexurl) - defer resp_index.Body.Close() + + if pleromastodon_api_resp.StatusCode == 200 { + var nodeinfo NodeInfo + err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo) if err != nil { - fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo) - return NodeInfo{} - } - indexbin, err := ioutil.ReadAll(resp_index.Body) - if err != nil { - fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo) - return NodeInfo{} - } - indexstr := string(indexbin) - if strings.Contains(indexstr, "Pleroma") { - nodeinfo.Software.Name = "pleroma" - } else if strings.Contains(indexstr, "Mastodon") { - nodeinfo.Software.Name = "mastodon" - } else { - return NodeInfo{} + log.Println("Was not unmarshalled, continuing...") } + return nodeinfo + } + + // Check the front page + index_url := "https://" + endpoint + "/" + resp_index, err := http_client.Get(index_url) + if err != nil { + log.Fatal("Error Message 2b:", resp_index.StatusCode, err, endpoint, resp_index.Status) + } + defer resp_index.Body.Close() + indexbin, err := ioutil.ReadAll(resp_index.Body) + if err != nil { + log.Fatal("Error Message 2c:", resp_index.StatusCode, err, endpoint, resp_index.Status) + } + indexstr := string(indexbin) + nodeinfo := NodeInfo{} + if strings.Contains(indexstr, "Pleroma") || strings.Contains(indexstr, "Soapbox") { + log.Print("Manual view: Pleroma" + endpoint) + nodeinfo.Software.Name = "pleroma" + nodeinfo.Software.Version = "guess" + } else if strings.Contains(indexstr, "Mastodon") { + log.Print("Manual view: Mastodon" + endpoint) + nodeinfo.Software.Name = "mastodon" + nodeinfo.Software.Version = "guess" + } else if strings.Contains(indexstr, "Gab") { + log.Print("Manual view: Gab" + endpoint) + nodeinfo.Software.Name = "gab" + nodeinfo.Software.Version = "guess" } return nodeinfo @@ -180,7 +243,7 @@ func StartInstance(endpoint string, reportPostChan chan ReportPost) { if nodeinfo.Software.Name == "" { var m = runninginstances[endpoint] m.Software = "" - m.LastRun = time.Now().Format("2006.01.02-15:04:05") + m.LastRun = time.Now().Format(time.RFC3339) m.Status = UNSUPPORTED_INSTANCE ri_mutex.Lock() runninginstances[endpoint] = m