working through post and account verification

engine/headers.go

@@ -51,7 +51,7 @@ type RunningInstance struct {
 	Status			int	`json:"status"`
 	LastRun			string `json:"lastrun"`
 	CaptureType		string `json:"capturetype"`
-	client			*http.Client
+	client			http.Client
 }
 
 type NodeInfoSoftware struct {

engine/instance.go

@@ -83,7 +83,7 @@ func StartInstance(endpoint string, reportPostChan chan ReportPost) {
 		return
 	}
 
-	m.client = &http_client
+	m.client = http_client
 	if nodeinfo.Software.Name == "pleroma" {
 		log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
 		m.CaptureType = "Poll"
@@ -95,6 +95,7 @@ func StartInstance(endpoint string, reportPostChan chan ReportPost) {
 		m.CaptureType = "Stream"
 		runninginstances[endpoint] = m
 		ri_mutex.Unlock()
+		PollMastodonPleroma(endpoint, reportPostChan, http_client)
 		StreamMastodon(endpoint, reportPostChan)
 	}
 

engine/poll.go

@@ -12,12 +12,88 @@ import (
 	"log"
 )
 
+type UserInfo struct {
+	Id		string	`"json:id"`
+	Type	string	`"json:type"`
+	Following	string	`"json:following"`
+	Followers	string	`"json:followers"`
+	Inbox	string		`"json:inbox"`
+	Outbox	string	`"json:outbox"`
+	Featured	string	`"json:featured"`
+	PreferredUsername	string	`"json:preferredUsername"`
+	Name	string	`"json:name"`
+	Summary	string	`"json:summary"`
+	Url	string	`"json:Url"`
+//	ManuallyApprovesFollowers	string	`"json:manuallyApprovesFollowers"`
+//	Discoverable	bool	`"json:discoverable"`
+}
+
+type PostInfo struct {
+	Id		string	`"json:id"`
+	Type	string	`"json:type"`
+	Published	string	`"json:published"`
+	Url	string	`"json:Url"`
+	Content	string	`"json:content"`
+}
+
+func fetch_user_info(http_client http.Client, uri string) (UserInfo, error) {
+	var userinfo UserInfo
+
+//	http_client := http.Client{}
+	req, err := http.NewRequest(http.MethodGet, uri, nil)
+	if err != nil {
+		return UserInfo{}, err
+	}
+
+	req.Header.Set("Accept", "application/ld+json")
+
+	resp, err := http_client.Do(req)
+	if err != nil {
+		return UserInfo{}, err
+	}
+	defer resp.Body.Close()
+
+//	content, err := ioutil.ReadAll(resp.Body)
+	err = json.NewDecoder(resp.Body).Decode(&userinfo)
+	if err != nil {
+		return UserInfo{}, err
+	}
+
+	return userinfo, nil
+}
+
+
+func fetch_post(http_client http.Client, uri string) (PostInfo, error) {
+	var postinfo PostInfo
+
+//	http_client := http.Client{}
+	req, err := http.NewRequest(http.MethodGet, uri, nil)
+	if err != nil {
+		return PostInfo{}, err
+	}
+
+	req.Header.Set("Accept", "application/ld+json")
+
+	resp, err := http_client.Do(req)
+	if err != nil {
+		return PostInfo{}, err
+	}
+	defer resp.Body.Close()
+
+//	content, err := ioutil.ReadAll(resp.Body)
+	err = json.NewDecoder(resp.Body).Decode(&postinfo)
+	if err != nil {
+		return PostInfo{}, err
+	}
+
+	return postinfo, nil
+}
+
 func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost, http_client http.Client) {
 	newposts := make([]ReportPost, 0)
 
 	min_id := ""
 
-//	http_client := http.Client{}
 	parsing_error := 0
 	unprocess_error := 0
 	use_auth := false
@@ -140,9 +216,45 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost, http_c
 			if newpost.Account.Acct == "" {
 				continue
 			}
-			posthash := sha1.New()
-
 			at_sign := strings.Index(newpost.Account.Acct, "@")
+			newinstance := newpost.Account.Acct[at_sign+1:]
+			// Trust the post if it comes from the same source
+			if newinstance == endpoint {
+//				fmt.Println("Do not need to verify" + newpost.Account.Acct[at_sign+1:])
+			} else {
+				ri_mutex.Lock()
+				o, exist := runninginstances[newinstance]
+				ri_mutex.Unlock()
+				if exist == false {
+					log.Print("Didn't exist so adding...")
+					o := RunningInstance{}
+					new_client := http.Client{}
+					o.client = new_client
+					ri_mutex.Lock()
+					runninginstances[newinstance] = o
+					log.Print(runninginstances)
+					ri_mutex.Unlock()
+				} else {
+					log.Print("Exists! ", o)
+				}
+
+				realuser, err := fetch_user_info(o.client, newpost.Account.Url)
+				if err != nil {
+					continue
+				}
+				realpost, err := fetch_post(o.client, newpost.Uri)
+				if err != nil {
+					continue
+				}
+
+				// Minor verification for now...
+				newpost.Account.Display_name = realuser.Name
+				newpost.Content = realpost.Content
+				newpost.Created_at = realpost.Published
+				log.Print("Updated! for " + newinstance)
+			}
+
+			posthash := sha1.New()
 
 			if at_sign == -1 {
 				at_sign = len(newpost.Account.Acct)
@@ -186,10 +298,13 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost, http_c
 
 			// Only done if we are crawling
 			if settings.Crawl == true && StringExists(endpoint, settings.Banned) == false {
-				newinstance := newpost.Account.Acct[at_sign+1:]
+				// Skip over this if its the same as the endpoint
+				if newinstance == endpoint {
+					continue
+				}
 				ri_mutex.Lock()
-				_, exists := runninginstances[newinstance]
+				o, exists := runninginstances[newinstance]
-				if exists == false {
+				if exists == false  || o.Status == NEW_INSTANCE {
 					m := RunningInstance{}
 					runninginstances[newinstance] = m
 					go StartInstance(newinstance, reportPostChan)