fedilogue/poll/instance.go

package main

import (
	"github.com/microcosm-cc/bluemonday"
	"encoding/json"
	"crypto/sha1"
	"io/ioutil"
	"net/http"
	"strings"
	"html"
	"time"
	"fmt"
)

var p *bluemonday.Policy

func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
	// Make this a global variable
	//p := bluemonday.NewPolicy()
	newposts := make([]ReportPost, 0)

	min_id := ""

	http_client := http.Client{Timeout: 5 * time.Second}

	for {
		ri_mutex.Lock()
		m := runninginstances[endpoint]
		ri_mutex.Unlock()

		api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&min_id=" + min_id
		resp, err := http_client.Get(api_timeline)
		m.LastRun = time.Now().Format("2006.01.02-15:04:05")
		if err != nil {
			ri_mutex.Lock()
			m.Status = CLIENT_ISSUE
			runninginstances[endpoint] = m
			ri_mutex.Unlock()
			return
		}

		if resp.StatusCode == UNAUTHORIZED || resp.StatusCode == UNPROCESSABLE_ENTITY {
			_, _ = ioutil.ReadAll(resp.Body)
			resp.Body.Close() // Release as soon as done
			ri_mutex.Lock()
			m.Status = UNAUTHORIZED
			runninginstances[endpoint] = m
			ri_mutex.Unlock()
			return
		} else if resp.StatusCode == TOOMANYREQUESTS {
			// Apparently you just need to do this to throw away the body
			_, _ = ioutil.ReadAll(resp.Body)
			resp.Body.Close() // Release as soon as done
			ri_mutex.Lock()
			m.Status = TOOMANYREQUESTS
			runninginstances[endpoint] = m
			ri_mutex.Unlock()
			time.Sleep(time.Second * 30)
			continue
		}
		err = json.NewDecoder(resp.Body).Decode(&newposts)
		if err != nil {
			fmt.Println("Error Message 1:", resp.StatusCode, err, endpoint, resp.Status)
			ri_mutex.Lock()
			m.Status = BAD_RESPONSE
			runninginstances[endpoint] = m
			ri_mutex.Unlock()
			return
		}
		resp.Body.Close() // Release as soon as done

		ri_mutex.Lock()
		m.Status = RUNNING
		runninginstances[endpoint] = m
		ri_mutex.Unlock()

		for _, newpost := range newposts {
			if newpost.Account.Acct == "" {
				continue
			}
			posthash := sha1.New()

			at_sign := strings.Index(newpost.Account.Acct, "@")

			if at_sign == -1 {
				at_sign = len(newpost.Account.Acct)
				newpost.Account.Acct += "@" + endpoint
			}

			// Calculate the post hash
			fmt.Fprint(posthash, newpost.Url)
			fmt.Fprint(posthash, newpost.normalized)
			fmt.Fprint(posthash, newpost.Account.Acct)
			fmt.Fprint(posthash, newpost.Account.Display_name)
			newpost.posthash = posthash.Sum(nil)

			newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content)))

			// Validate time
			_, err := time.Parse("2006-01-02T15:04:05.999Z", newpost.Created_at)
			if err != nil {
				fmt.Println("---------> FIXED TIME", newpost.Created_at)
				newpost.Created_at = time.Now().Format("2006.01.02-15:04:05")
			}

			reportPostChan <- newpost

			// Check min_id
			if newpost.Id > min_id {
				min_id = newpost.Id
			}
			newinstance := newpost.Account.Acct[at_sign+1:]
			ri_mutex.Lock()
			_, exists := runninginstances[newinstance]
			if exists == false {
				m := RunningInstance{}
				runninginstances[newinstance] = m
				go StartInstance(newinstance, reportPostChan)
			}

			ri_mutex.Unlock()
		}
		time.Sleep(time.Second * 10)
	}
}

// Change this to return a proper "err"
func GetNodeInfo(endpoint string) (NodeInfo) {
	var nodeinfo NodeInfo

	/* Not everyone implements this */
	api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json"
	http_client := http.Client{Timeout: 10 * time.Second}
	resp_node, err := http_client.Get(api_nodeinfo)
	if err != nil {
		fmt.Println("Make a legit error here", err)
		return NodeInfo{}
	}
	defer resp_node.Body.Close()

	/*
		Some Pleroma and Mastodon instances hide their /nodeinfo/2.0.json
		So this attempts to check the index URL for references to Mastodon
		or Pleroma.
	*/
	if resp_node.StatusCode == 200 {
		err = json.NewDecoder(resp_node.Body).Decode(&nodeinfo)
		if err != nil {
			fmt.Println("Error Message 2:", resp_node.StatusCode, err, endpoint, resp_node.Status, api_nodeinfo)
			return NodeInfo{}
		}
	}
	if resp_node.StatusCode == 404 {
		indexurl := "https://" + endpoint + "/"
		resp_index, err := http_client.Get(indexurl)
		defer resp_index.Body.Close()
		if err != nil {
			fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo)
			return NodeInfo{}
		}
		indexbin, err := ioutil.ReadAll(resp_index.Body)
		if err != nil {
			fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo)
			return NodeInfo{}
		}
		indexstr := string(indexbin)
		if strings.Contains(indexstr, "Pleroma") {
			nodeinfo.Software.Name = "pleroma"
			fmt.Println("It is Pleroma: " + endpoint)
		} else if strings.Contains(indexstr, "Mastodon") {
			nodeinfo.Software.Name = "mastodon"
			fmt.Println("It is Mastodon: " + endpoint)
		} else {
			return NodeInfo{}
		}
	}

	/*
	err = json.NewDecoder(resp.Body).Decode(&nodeinfo)
	if err != nil {
		fmt.Println("Error Message 2:", resp.StatusCode, err, endpoint, resp.Status, api_nodeinfo)
		return NodeInfo{}
	}
	*/

	return nodeinfo
}

func StartInstance(endpoint string, reportPostChan chan ReportPost) {
	nodeinfo := GetNodeInfo(endpoint)
	if nodeinfo.Software.Name == "" {
		var m = runninginstances[endpoint]
		m.Software = ""
		m.LastRun = time.Now().Format("2006.01.02-15:04:05")
		m.Status = UNSUPPORTED_INSTANCE
		ri_mutex.Lock()
		runninginstances[endpoint] = m
		ri_mutex.Unlock()
		return
	}

	if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" {
		go PollMastodonPleroma(endpoint, reportPostChan)
	}

}

/*
func SuspendInstance(suspendinstance InstanceReport) {
	for i, runninginstance := range runninginstances {
		if runninginstance.Endpoint == suspendinstance.endpoint {
			(runninginstances)[i].Status = suspendinstance.status
			(runninginstances)[i].LastRun = time.Now().Format("2006.01.02-15:04:05")
			return
		}
	}
}
*/