using the same http client, adding it to running instances

some web work
lost track of what's going on and I'm not accountable to anyone...so screw it

engine/db.go

@@ -41,12 +41,12 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
 	}
 
 	// Insert new post if new
-	_, err = conn.Exec(context.Background(), "INSERT INTO posts (uri, content, created_at, normalized, account_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, accountid, reportpost.posthash)
+	_, err = conn.Exec(context.Background(), "INSERT INTO posts (uri, content, created_at, normalized, account_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Uri, reportpost.Content, reportpost.Created_at, reportpost.normalized, accountid, reportpost.posthash)
 	if err != nil { // For now I want to know why this failed.
 		log.Print("Second ", err)
 		log.Print("--------------------------")
 		log.Print("Reported error: ", err)
-		log.Print("Url: ", reportpost.Url)
+		log.Print("Uri: ", reportpost.Uri)
 		log.Print("Content: ", reportpost.Content)
 		log.Print("Created_at: ", reportpost.Created_at)
 		log.Print("normalized: ", reportpost.normalized)

engine/headers.go

@@ -1,5 +1,9 @@
 package main
 
+import (
+	"net/http"
+)
+
 const (
 	NEW_INSTANCE =			0
 	RUNNING =			200
@@ -22,7 +26,7 @@ type ReportPost struct {
 
 	// Retrieved values
 	Id			string `json:"id"`
-	Url			string `json:"uri"`
+	Uri			string `json:"uri"`
 	Account			AccountType
 	Content			string `json:"content"`
 	Created_at		string `json:"created_at"`
@@ -38,7 +42,7 @@ type AccountType struct {
 	Bot			bool	`json:"bot"`
 	Created_at		string	`json:"created_at"`
 	Display_name		string	`json:"display_name"`
-	Url			string	`json:"uri"`
+	Url			string	`json:"url"`
 }
 
 // Instance's new min_id value
@@ -47,6 +51,7 @@ type RunningInstance struct {
 	Status			int	`json:"status"`
 	LastRun			string `json:"lastrun"`
 	CaptureType		string `json:"capturetype"`
+	client			*http.Client
 }
 
 type NodeInfoSoftware struct {
@@ -68,3 +73,20 @@ type ResponseBack struct {
 	Message			string `json:"Message"`
 	RunningInstances	map[string]RunningInstance `json:"RunningInstances"`
 }
+
+type Userinfo struct {
+	Id		string	`"json:id"`
+	Type	string	`"json:type"`
+	Following	string	`"json:following"`
+	Followers	string	`"json:followers"`
+	Inbox	string		`"json:inbox"`
+	Outbox	string	`"json:outbox"`
+	Featured	string	`"json:featured"`
+	PreferredUsername	string	`"json:preferredUsername"`
+	Name	string	`"json:name"`
+	Summary	string	`"json:summary"`
+	Url	string	`"json:Url"`
+	ManuallyApprovesFollowers	string	`"json:manuallyApprovesFollowers"`
+	Discoverable	string	`"json:discoverable"`
+}
+

engine/instance.go

@@ -15,17 +15,16 @@ var p *bluemonday.Policy
 var spaceReg *regexp.Regexp
 
 // Change this to return a proper "err"
-func GetNodeInfo(endpoint string) (NodeInfo) {
+func GetNodeInfo(endpoint string) (http.Client, NodeInfo) {
 	/* Checking order
 	 * Mastodon/Pleroma
 	 * Um..nothing else yet
 	*/
 	pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"
-	//http_client := http.Client{Timeout: 10 * time.Second}
 	http_client := http.Client{}
 	pleromastodon_api_resp, err := http_client.Get(pleromastodon_nodeinfo_uri)
 	if err != nil {
-		return NodeInfo{}
+		return http_client, NodeInfo{}
 	} else {
 		defer pleromastodon_api_resp.Body.Close()
 	}
@@ -35,7 +34,7 @@ func GetNodeInfo(endpoint string) (NodeInfo) {
 		err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo)
 		if err == nil {
 			defer pleromastodon_api_resp.Body.Close()
-			return nodeinfo
+			return http_client, nodeinfo
 		}
 	}
 
@@ -44,13 +43,13 @@ func GetNodeInfo(endpoint string) (NodeInfo) {
 	resp_index, err := http_client.Get(index_uri)
 	if err != nil {
 		log.Print("Unable to connect to " + endpoint + ", giving up")
-		return NodeInfo{}
+		return http_client, NodeInfo{}
 	}
 	defer resp_index.Body.Close()
 	indexbin, err := ioutil.ReadAll(resp_index.Body)
 	if err != nil {
 		log.Print("Unable to read index of " + endpoint + ", giving up")
-		return NodeInfo{}
+		return http_client, NodeInfo{}
 	}
 	indexstr := string(indexbin)
 	nodeinfo := NodeInfo{}
@@ -68,30 +67,29 @@ func GetNodeInfo(endpoint string) (NodeInfo) {
 		nodeinfo.Software.Version = "guess"
 	}
 
-	return nodeinfo
+	return http_client, nodeinfo
 }
 
 func StartInstance(endpoint string, reportPostChan chan ReportPost) {
-	nodeinfo := GetNodeInfo(endpoint)
+	http_client, nodeinfo := GetNodeInfo(endpoint)
-	if nodeinfo.Software.Name == "" {
+	ri_mutex.Lock()
 	m := runninginstances[endpoint]
+	if nodeinfo.Software.Name == "" {
 		m.Software = ""
 		m.LastRun = time.Now().Format(time.RFC3339)
 		m.Status = UNSUPPORTED_INSTANCE
-		ri_mutex.Lock()
 		runninginstances[endpoint] = m
 		ri_mutex.Unlock()
 		return
 	}
 
-	ri_mutex.Lock()
+	m.client = &http_client
-	m := runninginstances[endpoint]
 	if nodeinfo.Software.Name == "pleroma" {
 		log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
 		m.CaptureType = "Poll"
 		runninginstances[endpoint] = m
 		ri_mutex.Unlock()
-		PollMastodonPleroma(endpoint, reportPostChan)
+		PollMastodonPleroma(endpoint, reportPostChan, http_client)
 	} else if nodeinfo.Software.Name == "mastodon" {
 		log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
 		m.CaptureType = "Stream"

engine/poll.go

@@ -12,12 +12,12 @@ import (
 	"log"
 )
 
-func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
+func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost, http_client http.Client) {
 	newposts := make([]ReportPost, 0)
 
 	min_id := ""
 
-	http_client := http.Client{}
+//	http_client := http.Client{}
 	parsing_error := 0
 	unprocess_error := 0
 	use_auth := false
@@ -46,7 +46,6 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
 		}
 	}
 
-
 	for {
 		ri_mutex.Lock()
 		m := runninginstances[endpoint]
@@ -151,7 +150,7 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
 			}
 
 			// Calculate the post hash
-			fmt.Fprint(posthash, newpost.Url)
+			fmt.Fprint(posthash, newpost.Uri)
 			fmt.Fprint(posthash, newpost.normalized)
 			fmt.Fprint(posthash, newpost.Account.Acct)
 			fmt.Fprint(posthash, newpost.Account.Display_name)

engine/stream.go

@@ -106,7 +106,7 @@ func StreamMastodon(endpoint string, reportPostChan chan ReportPost) {
 		}
 
 		// Calculate the post hash
-		fmt.Fprint(posthash, newpost.Url)
+		fmt.Fprint(posthash, newpost.Uri)
 		fmt.Fprint(posthash, newpost.normalized)
 		fmt.Fprint(posthash, newpost.Account.Acct)
 		fmt.Fprint(posthash, newpost.Account.Display_name)

engine/web.go

@@ -6,6 +6,9 @@ import (
 	"net/http"
 	"encoding/json"
 	"io/ioutil"
+	"html"
+	"time"
+	"strings"
 	"os"
 )
 
@@ -91,7 +94,8 @@ func webfinger(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func inbox(w http.ResponseWriter, r *http.Request) {
+func inboxHandler(reportPostChan chan ReportPost) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
 		fmt.Println("PATH --> ", r.URL.Path)
 
 		body, err := ioutil.ReadAll(r.Body)
@@ -103,7 +107,7 @@ func inbox(w http.ResponseWriter, r *http.Request) {
 		var findtype FindType
 		err = json.Unmarshal(body, &findtype)
 
-	fmt.Println(findtype.Type)
+		fmt.Println(string(body))
 
 		switch findtype.Type {
 		case "Create":
@@ -114,6 +118,22 @@ func inbox(w http.ResponseWriter, r *http.Request) {
 				fmt.Println("Ignore the post here")
 			}
 			fmt.Println("Content:      ", createobject.Content)
+
+			var newpost ReportPost
+			newpost.Uri = findtype.Id
+			newpost.Content = createobject.Content
+			newpost.Created_at = findtype.Published
+			newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content)))
+			newpost.normalized = strings.ReplaceAll(newpost.normalized, "\t", " ")
+			newpost.normalized = spaceReg.ReplaceAllString(newpost.normalized, " ")
+			newpost.Account.Acct = "aaa@bbb"
+			newpost.Account.Avatar = "ttt"
+			newpost.Account.Bot = false
+			newpost.Account.Created_at = time.Now().Format(time.RFC3339)
+			newpost.Account.Display_name = "Fqrhqn"
+			newpost.Account.Url = "qwerty"
+			reportPostChan <- newpost
+
 		case "Like":
 		case "Announcement":
 		case "Delete":
@@ -130,6 +150,7 @@ func inbox(w http.ResponseWriter, r *http.Request) {
 		fmt.Println("Type:         ", findtype.Type)
 
 		fmt.Println("Object:       ", string(findtype.Object))
+	}
 }
 
 func users_fedilogue_followers(w http.ResponseWriter, r *http.Request) {
@@ -218,7 +239,7 @@ func errorHandler(w http.ResponseWriter, r *http.Request) {
 func webmain(reportPostChan chan ReportPost) {
 	http.HandleFunc("/.well-known/webfinger", webfinger)
 	http.HandleFunc("/.well-known/host-meta", hostmeta)
-	http.HandleFunc("/inbox", inbox)
+	http.HandleFunc("/inbox", inboxHandler(reportPostChan))
 	http.HandleFunc("/users/fedilogue", users_fedilogue)
 	http.HandleFunc("/users/fedilogue/followers", users_fedilogue_followers)
 	http.HandleFunc("/users/fedilogue/following", users_fedilogue_following)