fedilogue/fedilogue/instance.go

package main

import (
	"gitlab.com/khanzf/fedilogue/shared"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"math/rand"
	"context"
	"strings"
	"time"
	"net"
	"fmt"
)

var staggeredStartChan chan bool

func DoTries(o *RunningInstance, req *http.Request) (*http.Response, error) {
	var resp *http.Response
	var err error

	for tries := 0; tries < 10; tries++ {
		resp, err = o.client.Do(req)
		if err != nil {
			// URL.Scheme, Host, Path Opaque
			logWarn("Failure connecting to " + req.URL.Scheme + "://" + req.URL.Host + req.URL.Path + ", attempt ", tries + 1, ", sleeping for 5 minutes: ", err)
			time.Sleep(time.Minute * 5)
			continue
		}
		break
	}
	return resp, err
}

func BuildClient(endpoint string) http.Client {
	// Test: TestBuildClient, TestBuildClientProxy
	/* The seemingly unused 'endpoint' variable is for proxying based on endpoint, ie for Tor */
	tr := &http.Transport{
		MaxIdleConns: 2,
		IdleConnTimeout: 3600 * time.Second,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).DialContext,
	}
	client := http.Client{Transport: tr}

	if len(settings.Proxies) >= 1 {
		rand.Seed(time.Now().Unix())
		ridx := rand.Intn(len(settings.Proxies))
		proxyuri := fmt.Sprintf("socks5://%s:%d", settings.Proxies[ridx].Host, settings.Proxies[ridx].Port)
		proxy, err := url.Parse(proxyuri)
		if err != nil {
			logFatal.Fatal("Error: ", err)
		}
		tr.Proxy = http.ProxyURL(proxy)
	}

	return client
}

func GetRunner(endpoint string) (RunningInstance, bool) {
	// Tests: TestGetRunnerNonExist, TestGetRunnerExists
	ri_mutex.Lock()
	o, exists := runninginstances[endpoint]

	if exists == false {
		o = RunningInstance{}
		o.client = BuildClient(endpoint)
		o.Status = KEEPALIVE
		o.recentactivities = shared.NewUniqueFifo(10)
		o.recentactors = shared.NewUniqueFifo(10)
		runninginstances[endpoint] = o
	}
	ri_mutex.Unlock()

	return o, exists
}

func UpdateRunner(endpoint string, o RunningInstance) {
	// Tests: None necessary
	ri_mutex.Lock()
	runninginstances[endpoint] = o
	ri_mutex.Unlock()
}

func GetInstanceInfo(endpoint string, o RunningInstance) RunningInstance {
	/* Checking order
	 * Mastodon/Pleroma
	 * Um..nothing else yet
	 */
	var nodeinfo NodeInfo
	pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"

	req, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil)
	req.Header.Set("User-Agent", "Tusky")

	pleromastodon_api_resp, err := DoTries(&o, req)
	if err != nil {
		o.Software = "Unsupported"
		return o
	} else {
		defer pleromastodon_api_resp.Body.Close()
	}

	if pleromastodon_api_resp.StatusCode == 200 {
		err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo)
		if err == nil {
			o.Software = nodeinfo.Software.Name
			o.Version = nodeinfo.Software.Version
			o.LastRun = time.Now().Format(time.RFC3339)
			defer pleromastodon_api_resp.Body.Close()
			return o
		}
	}

	// Check the front page
	index_uri := "https://" + endpoint + "/"
	req, _ = http.NewRequest("GET", index_uri, nil)
	req.Header.Set("User-Agent", "Tusky")
	resp_index, err := DoTries(&o, req)
	o.LastRun = time.Now().Format(time.RFC3339)
	if err != nil {
		o.Status = UNSUPPORTED_INSTANCE
		o.Software = "Unsupported"
		logWarn("Unable to connect to " + endpoint + ", giving up")
		return o
	}
	defer resp_index.Body.Close()

	indexbin, err := ioutil.ReadAll(resp_index.Body)
	if err != nil {
		o.Status = UNSUPPORTED_INSTANCE
		o.Software = "Unsupported"
		logWarn("Unable to read index of " + endpoint + ", giving up")
		return o
	}
	indexstr := string(indexbin)

	if strings.Contains(indexstr, "Pleroma") || strings.Contains(indexstr, "Soapbox") {
		o.Software = "pleroma"
		o.Version = "guess"
	} else if strings.Contains(indexstr, "Mastodon") {
		o.Software = "mastodon"
		o.Version = "guess"
	} else if strings.Contains(indexstr, "Gab") {
		o.Software = "gab"
		o.Version = "guess"
	} else {
		o.Software = "Unsupported"
		o.Version = "Unknown"
	}

	return o
}

func LogInstance(endpoint string, o RunningInstance) bool {
	selectRet := pool.QueryRow(context.Background(), "SELECT FROM instances WHERE endpoint = $1", endpoint)
	err := selectRet.Scan()
	if err == nil {
		return true // Endpoint already in database, continuing
	}

	_, err = pool.Exec(context.Background(), "INSERT INTO instances (endpoint, autostart, state, software) VALUES($1, $2, $3, $4)", endpoint, true, "", o.Software)
	if err != nil {
		logWarn("Error inserting %s into `instances`: "+endpoint, err)
		return true
	}

	return false
}

func CheckInstance(newinstance string, callerEndpoint string) {
	if settings.Crawl == true && stringexists(newinstance, settings.Banned) == false {
		// Skip over this if its the same as the endpoint or empty
		if newinstance == callerEndpoint || newinstance == "" {
			return
		}

		var err error
		for attempt := 0; attempt > 5; attempt = attempt + 1 {
			_, err = net.LookupHost(newinstance)
			if err != nil {
				logDebug("Unable to resolve " + newinstance + " attempt ", attempt, "/5. Sleeping for 30 seconds")
				time.Sleep(time.Second * 30)
				continue
			}
			break
		}
		if err != nil {
			logWarn("Unable to resolve ", newinstance, " after 5 attempts, giving up: ", err)
			return
		}

		// Skip over this if its the same as the endpoint
		if newinstance == callerEndpoint {
			return
		}

		// Going forward, this might be merged into GetRunner
		ri_mutex.Lock()
		o, exists := runninginstances[newinstance]
		if exists == false || o.Status == KEEPALIVE {
			m := RunningInstance{}
			m.client = BuildClient(newinstance)
			m.recentactivities = shared.NewUniqueFifo(10)
			m.recentactors = shared.NewUniqueFifo(10)
			runninginstances[newinstance] = m
			go StartInstance(newinstance)
		}
		ri_mutex.Unlock()
	}
}

func staggeredStart() {
	for {
		_ :<- staggeredStartChan
		time.Sleep(500 * time.Millisecond)
	}
}

func StartInstance(endpoint string) {
	staggeredStartChan <- true
	logInfo("Starting " + endpoint)

	// Check if exists. If so, get the object. If not, create it
	o, _ := GetRunner(endpoint)

	o = GetInstanceInfo(endpoint, o)
	UpdateRunner(endpoint, o)
	LogInstance(endpoint, o)

	if o.Software == "pleroma" {
		logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
		o.CaptureType = "Poll"
		UpdateRunner(endpoint, o)
		PollMastodonPleroma(endpoint, &o)
	} else if o.Software == "mastodon" {
		logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
		o.CaptureType = "Stream"
		UpdateRunner(endpoint, o)
		StreamMastodon(endpoint, &o)
	} else {
		o.Status = 605
		UpdateRunner(endpoint, o)
		logConn("Unsupported endpoint " + endpoint)
	}
}