reduced the number of new connections to 1, reduce spamming servers

set keep-alive to 2 hours
probably a bunch of regressions but I don't have unit tests yet

headers.go

@@ -49,10 +49,13 @@ type AccountType struct {
 // Instance's new min_id value
 type RunningInstance struct {
 	Software    string `json:"software"`
+	Version     string `json:"version"`
 	Status      int    `json:"status"`
 	LastRun     string `json:"lastrun"`
 	CaptureType string `json:"capturetype"`
 	client      http.Client
+	client_id	string
+	client_secret	string
 }
 
 type NodeInfoSoftware struct {

instance.go

@@ -14,92 +14,115 @@ import (
 var p *bluemonday.Policy
 var spaceReg *regexp.Regexp
 
-// Change this to return a proper "err"
+func GetRunner(endpoint string) (RunningInstance) {
-func GetNodeInfo(endpoint string) (http.Client, NodeInfo) {
+	ri_mutex.Lock()
+	o, exists := runninginstances[endpoint]
+	if exists == false {
+		o := RunningInstance{}
+
+		tr := &http.Transport {MaxIdleConns: 10, IdleConnTimeout: 7200 * time.Second}
+
+		o.client = http.Client{Transport: tr}
+		o.Status = KEEPALIVE
+		runninginstances[endpoint] = o
+	}
+	ri_mutex.Unlock()
+
+	return o
+}
+
+
+func UpdateRunner(endpoint string, o RunningInstance) {
+	ri_mutex.Lock()
+	runninginstances[endpoint] = o
+	ri_mutex.Unlock()
+}
+
+
+func GetNodeInfo(endpoint string, o RunningInstance) (RunningInstance) {
 	/* Checking order
 	 * Mastodon/Pleroma
 	 * Um..nothing else yet
 	 */
+	var nodeinfo NodeInfo
 	pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"
-	http_client := http.Client{}
+
-	pleromastodon_api_resp, err := http_client.Get(pleromastodon_nodeinfo_uri)
+	req, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil)
+
+	pleromastodon_api_resp, err := o.client.Do(req)
 	if err != nil {
-		return http_client, NodeInfo{}
+		return o
 	} else {
 		defer pleromastodon_api_resp.Body.Close()
 	}
 
 	if pleromastodon_api_resp.StatusCode == 200 {
-		var nodeinfo NodeInfo
 		err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo)
 		if err == nil {
+			o.Software = nodeinfo.Software.Name
+			o.LastRun = time.Now().Format(time.RFC3339)
 			defer pleromastodon_api_resp.Body.Close()
-			return http_client, nodeinfo
+			return o
 		}
 	}
 
 	// Check the front page
 	index_uri := "https://" + endpoint + "/"
-	resp_index, err := http_client.Get(index_uri)
+	req, _ = http.NewRequest("GET", index_uri, nil)
+	resp_index, err := o.client.Do(req)
+	o.LastRun = time.Now().Format(time.RFC3339)
 	if err != nil {
+		o.Status = UNSUPPORTED_INSTANCE
 		log.Print("Unable to connect to " + endpoint + ", giving up")
-		return http_client, NodeInfo{}
+		return o
 	}
 	defer resp_index.Body.Close()
+
 	indexbin, err := ioutil.ReadAll(resp_index.Body)
 	if err != nil {
+		o.Status = UNSUPPORTED_INSTANCE
 		log.Print("Unable to read index of " + endpoint + ", giving up")
-		return http_client, NodeInfo{}
+		return o
 	}
 	indexstr := string(indexbin)
-	nodeinfo := NodeInfo{}
+
 	if strings.Contains(indexstr, "Pleroma") || strings.Contains(indexstr, "Soapbox") {
 		log.Print("Manual view: Pleroma" + endpoint)
-		nodeinfo.Software.Name = "pleroma"
+		o.Software = "pleroma"
-		nodeinfo.Software.Version = "guess"
+		o.Version = "guess"
 	} else if strings.Contains(indexstr, "Mastodon") {
 		log.Print("Manual view: Mastodon" + endpoint)
-		nodeinfo.Software.Name = "mastodon"
+		o.Software = "mastodon"
-		nodeinfo.Software.Version = "guess"
+		o.Version = "guess"
 	} else if strings.Contains(indexstr, "Gab") {
 		log.Print("Manual view: Gab" + endpoint)
-		nodeinfo.Software.Name = "gab"
+		o.Software = "gab"
-		nodeinfo.Software.Version = "guess"
+		o.Version = "guess"
 	}
 
-	return http_client, nodeinfo
+	return o
 }
 
 func StartInstance(endpoint string) {
-	http_client, nodeinfo := GetNodeInfo(endpoint)
+	// Check if exists. If so, get the object. If not, create it
-	ri_mutex.Lock()
+	o := GetRunner(endpoint)
-	m := runninginstances[endpoint]
+
-	ri_mutex.Unlock()
+	o = GetNodeInfo(endpoint, o)
-	if nodeinfo.Software.Name == "" {
+	UpdateRunner(endpoint, o)
-		m.Software = ""
+
-		m.LastRun = time.Now().Format(time.RFC3339)
+	if o.Software == "" {
-		m.Status = UNSUPPORTED_INSTANCE
-		ri_mutex.Lock()
-		runninginstances[endpoint] = m
-		ri_mutex.Unlock()
 		return
 	}
 
-	m.client = http_client
+	if o.Software == "pleroma" {
-	if nodeinfo.Software.Name == "pleroma" {
+		log.Print("Starting " + endpoint + " as " + o.Software)
-		log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
+		o.CaptureType = "Poll"
-		m.CaptureType = "Poll"
+		UpdateRunner(endpoint, o)
-		ri_mutex.Lock()
+		PollMastodonPleroma(endpoint, &o)
-		runninginstances[endpoint] = m
+	} else if o.Software == "mastodon" {
-		ri_mutex.Unlock()
+		log.Print("Starting " + endpoint + " as " + o.Software)
-		PollMastodonPleroma(endpoint, http_client)
+		o.CaptureType = "Stream"
-	} else if nodeinfo.Software.Name == "mastodon" {
+		UpdateRunner(endpoint, o)
-		log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
+		StreamMastodon(endpoint, &o)
-		m.CaptureType = "Stream"
-		ri_mutex.Lock()
-		runninginstances[endpoint] = m
-		ri_mutex.Unlock()
-		StreamMastodon(endpoint)
 	}
-
 }

oauth.go

@@ -26,7 +26,7 @@ func (e *authError) Error() string {
 	return e.msg
 }
 
-func register_client(endpoint string, http_client *http.Client) (string, string, error) {
+func register_client(endpoint string, o *RunningInstance) (error) {
 	requestBodymap, _ := json.Marshal(map[string]string{
 		"client_name":   "Tusky", // Hard-coded in for now...
 		"scopes":        "read write follow push",
@@ -36,79 +36,101 @@ func register_client(endpoint string, http_client *http.Client) (string, string,
 
 	api_base_apps := "https://" + endpoint + "/api/v1/apps"
 
-	resp, err := http_client.Post(api_base_apps, "application/json", requestBodybytes)
+	resp, err := o.client.Post(api_base_apps, "application/json", requestBodybytes)
 	if err != nil {
 		log.Fatal("Unable to connect to "+api_base_apps+" ", err)
 	}
 
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		log.Print("Unable to read HTTP response: ", err)
+		log.Fatal("Unable to read HTTP response: ", err)
-		return "", "", err
+		o.client_id = ""
+		o.client_secret = ""
+		return err
 	}
 	defer resp.Body.Close()
 
 	bodymap := make(map[string]string)
 	err = json.Unmarshal(body, &bodymap)
 	if err != nil {
-		log.Print("Unable to Unmarshal response: ", err)
+		log.Fatal("Unable to Unmarshal response: ", err)
-		return "", "", err
+		o.client_id = ""
+		o.client_secret = ""
+		return err
 	}
 
 	client_file := "clients/" + endpoint
 
 	f, err := os.Create("clients/" + endpoint)
 	if err != nil {
-		log.Print("Unable to create "+client_file+": ", err)
+		log.Fatal("Unable to create "+client_file+": ", err)
-		return bodymap["client_id"], bodymap["client_secret"], nil
+		o.client_id = ""
+		o.client_secret = ""
+		return err
+		//return bodymap["client_id"], bodymap["client_secret"], nil
 	}
 	defer f.Close()
 
 	_, err = io.WriteString(f, bodymap["client_id"]+"\n")
 	if err != nil {
-		log.Print("Unable to write client_id line: ", err)
+		log.Fatal("Unable to write client_id line: ", err)
-		return bodymap["client_id"], bodymap["client_secret"], nil
+		o.client_id = bodymap["client_id"]
+		o.client_secret = bodymap["client_secret"]
+		return nil
 	}
 	_, err = io.WriteString(f, bodymap["client_secret"]+"\n")
 	if err != nil {
-		log.Print("Unable to write client_secret line: ", err)
+		log.Fatal("Unable to write client_secret line: ", err)
-		return bodymap["client_id"], bodymap["client_secret"], nil
+		o.client_id = bodymap["client_id"]
+		o.client_secret = bodymap["client_secret"]
+		return nil
 	}
 
-	return bodymap["client_id"], bodymap["client_secret"], nil
+	o.client_id = bodymap["client_id"]
+	o.client_secret = bodymap["client_secret"]
+	return nil
 }
 
-func get_client(endpoint string, http_client *http.Client) (string, string, error) {
+func get_client(endpoint string, o *RunningInstance) (error) {
+	var err error
 	client_file := "clients/" + endpoint
-	_, err := os.Stat(client_file)
+	_, err = os.Stat(client_file)
 	if os.IsNotExist(err) == false { // The file exists
 		f, err := os.Open(client_file)
 		if err != nil {
 			log.Print("Unable to open " + client_file + ", creating new client")
-			return register_client(endpoint, http_client)
+			return err
+//			return register_client(endpoint, o)
 		}
 		defer f.Close()
 
 		rd := bufio.NewReader(f)
 
-		client_id, _, err := rd.ReadLine()
+		client_id_bin, _, err := rd.ReadLine()
+		o.client_id = string(client_id_bin)
 		if err != nil {
 			log.Print("Unable to read client_id line of " + client_file + ", building new client")
-			return register_client(endpoint, http_client)
+			return err
+//			return register_client(endpoint, o)
 		}
-		client_secret, _, err := rd.ReadLine()
+		client_secret_bin, _, err := rd.ReadLine()
+		o.client_secret = string(client_secret_bin)
 		if err != nil {
 			log.Print("Unable to read client_secret line of " + client_file + ", building new client")
-			return register_client(endpoint, http_client)
+			return err
+//			return register_client(endpoint, o)
+//			return o
 		}
 
-		return string(client_id), string(client_secret), nil
+		return nil
 	} else {
-		return register_client(endpoint, http_client)
+		return register_client(endpoint, o)
-	}
 	}
 
-func oauth_login(endpoint string, username string, password string, client_id string, client_secret string) (OAuth, error) {
+	return nil
+}
+
+func oauth_login(endpoint string, o *RunningInstance, username string, password string) (OAuth, error) {
 	authMap, err := json.Marshal(map[string]string{
 		"username":      username,
 		"password":      password,
@@ -116,8 +138,8 @@ func oauth_login(endpoint string, username string, password string, client_id st
 		"grant_type":    "password",
 		"client_name":   "Tusky",
 		"scope":         "read write follow push",
-		"client_id":     client_id,
+		"client_id":     o.client_id,
-		"client_secret": client_secret,
+		"client_secret": o.client_secret,
 	})
 
 	if err != nil {

poll.go

@@ -50,7 +50,7 @@ type PostInfo struct {
 	Content   string `"json:content"`
 }
 
-func PollMastodonPleroma(endpoint string, http_client http.Client) {
+func PollMastodonPleroma(endpoint string, o *RunningInstance) {
 	newposts := make([]ReportPost, 0)
 
 	min_id := ""
@@ -68,13 +68,14 @@ func PollMastodonPleroma(endpoint string, http_client http.Client) {
 	for _, extaccount := range settings.Externalaccounts {
 		if extaccount.Endpoint == endpoint {
 			use_auth = true
-			client_id, client_secret, err = get_client(endpoint, &http_client)
+			o := GetRunner(endpoint)
+			err = get_client(endpoint, &o)
 			if err != nil {
 				log.Print("Unable to register client: ", err)
 				return
 			}
 
-			oauthData, err = oauth_login(endpoint, extaccount.Username, extaccount.Password, client_id, client_secret)
+			oauthData, err = oauth_login(endpoint, &o, extaccount.Username, extaccount.Password)
 			if err != nil {
 				log.Print("Unable to login: ", err)
 				return
@@ -109,7 +110,7 @@ func PollMastodonPleroma(endpoint string, http_client http.Client) {
 		}
 
 		m.LastRun = time.Now().Format(time.RFC3339)
-		resp, err := http_client.Do(req)
+		resp, err := o.client.Do(req)
 		if err != nil {
 			m.Status = CLIENT_ISSUE
 			ri_mutex.Lock()

stream.go

@@ -9,11 +9,11 @@ import (
 	"time"
 )
 
-func StreamMastodon(endpoint string) {
+func StreamMastodon(endpoint string, o *RunningInstance) {
 	http_client := http.Client{}
 
-	var client_id string
+	//var client_id string
-	var client_secret string
+	//var client_secret string
 	var oauthData OAuth
 	var err error
 
@@ -26,14 +26,14 @@ func StreamMastodon(endpoint string) {
 
 	for _, extaccount := range settings.Externalaccounts {
 		if extaccount.Endpoint == endpoint {
-			get_client(endpoint, &http_client)
+			get_client(endpoint, o)
 
-			client_id, client_secret, err = get_client(endpoint, &http_client)
+			err = get_client(endpoint, o)
 			if err != nil {
 				log.Fatal("Unable to register client: ", err)
 			}
 
-			oauthData, err = oauth_login(endpoint, extaccount.Username, extaccount.Password, client_id, client_secret)
+			oauthData, err = oauth_login(endpoint, o, extaccount.Username, extaccount.Password)
 			if err != nil {
 				log.Print("Unable to login: ", err)
 				return