few bugs

instance.go

@@ -74,10 +74,12 @@ func StartInstance(endpoint string) {
 	http_client, nodeinfo := GetNodeInfo(endpoint)
 	ri_mutex.Lock()
 	m := runninginstances[endpoint]
+	ri_mutex.Unlock()
 	if nodeinfo.Software.Name == "" {
 		m.Software = ""
 		m.LastRun = time.Now().Format(time.RFC3339)
 		m.Status = UNSUPPORTED_INSTANCE
+		ri_mutex.Lock()
 		runninginstances[endpoint] = m
 		ri_mutex.Unlock()
 		return
@@ -87,12 +89,14 @@ func StartInstance(endpoint string) {
 	if nodeinfo.Software.Name == "pleroma" {
 		log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
 		m.CaptureType = "Poll"
+		ri_mutex.Lock()
 		runninginstances[endpoint] = m
 		ri_mutex.Unlock()
 		PollMastodonPleroma(endpoint, http_client)
 	} else if nodeinfo.Software.Name == "mastodon" {
 		log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
 		m.CaptureType = "Stream"
+		ri_mutex.Lock()
 		runninginstances[endpoint] = m
 		ri_mutex.Unlock()
 		StreamMastodon(endpoint)

poll.go

@@ -70,7 +70,8 @@ func PollMastodonPleroma(endpoint string, http_client http.Client) {
 			use_auth = true
 			client_id, client_secret, err = get_client(endpoint, &http_client)
 			if err != nil {
-				log.Fatal("Unable to register client: ", err)
+				log.Print("Unable to register client: ", err)
+				return
 			}
 
 			oauthData, err = oauth_login(endpoint, extaccount.Username, extaccount.Password, client_id, client_secret)
@@ -114,7 +115,7 @@ func PollMastodonPleroma(endpoint string, http_client http.Client) {
 			ri_mutex.Lock()
 			runninginstances[endpoint] = m
 			ri_mutex.Unlock()
-			log.Fatal("Failure here", err.Error())
+			log.Print("Failure here", err.Error())
 			return
 		}
 

retrieve.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"github.com/jackc/pgx/pgxpool"
 	"time"
+	"io/ioutil"
 	"net/http"
 )
 
@@ -110,6 +111,10 @@ func check_post(uri string) (PostJson, error) {
 		return postjson, nil
 	}
 	endslash := strings.Index(uri[8:], "/")
+	if endslash == -1 {
+		return postjson, nil
+	}
+	log.Print("Was: " + uri, " ", endslash)
 	postjson.instance = uri[8:endslash+8]
 
 
@@ -118,17 +123,37 @@ func check_post(uri string) (PostJson, error) {
 	req.Header.Add("Accept", "application/ld+json")
 
 	resp, err := o.client.Do(req)
+	if err != nil {
+		return postjson, nil
+	}
 
+/*
 	err = json.NewDecoder(resp.Body).Decode(&postjson)
 	if err != nil {
 		return postjson, err
 	}
+*/
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return postjson, nil
+	}
+	err = json.Unmarshal(body, &postjson)
+	if err != nil {
+		return postjson, nil
+	}
 
 	if postjson.InReplyTo != "" && postjson.InReplyTo != uri {
 		log.Print("GOING INTO NEW POST: " + postjson.InReplyTo + " " + uri)
 		go check_post(postjson.InReplyTo)
 	}
 
+	// If AttributedTo is blank, this is likely an authentication failure
+	// For now, skip it...
+
+	if postjson.AttributedTo == "" {
+		return postjson, nil
+	}
 	check_user(postjson.AttributedTo) // This must be done BEFORE the `INSERT INTO posts` below
 
 	connrequest = ConnRequest{}
@@ -141,12 +166,12 @@ func check_post(uri string) (PostJson, error) {
 	_, err = myconn.Exec(context.Background(), "INSERT INTO posts (id, inreplyto, published, summary, content, normalized, attributedto, posthash, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)", postjson.ID, postjson.InReplyTo, postjson.Published, postjson.Summary, postjson.Content, postjson.normalized, postjson.AttributedTo, postjson.posthash, postjson.instance)
 	close(connrequest.b)
 	if err != nil {
-		log.Print("INSERT posts error: ", err)
+		log.Print("INSERT posts error of " + uri + ": ", err)
 		return postjson, err
 	}
 
 	for _, to := range postjson.To{
-		if to != "https://www.w3.org/ns/activitystreams#Public" {
+		if to != "https://www.w3.org/ns/activitystreams#Public" && to != "" {
 			go check_user(to)
 		}
 	}
@@ -171,7 +196,9 @@ func check_user(uri string) (UserJson, error) {
 	if err == nil {
 		return userjson, nil
 	}
+	log.Print("The URI: " + uri)
 	endslash := strings.Index(uri[8:], "/")
+	log.Print("on " + uri + " ", endslash)
 	userjson.instance = uri[8:endslash+8]
 
 	o := GetHTTPSession(userjson.instance)

stream.go

@@ -46,7 +46,8 @@ func StreamMastodon(endpoint string) {
 
 	resp, err := http_client.Do(req)
 	if err != nil {
-		log.Fatal("Error occured for " + api_timeline)
+		log.Print("Unable to stream " + api_timeline + ": ", err)
+		return
 	}
 	defer resp.Body.Close()