From a50916eb365e58db4bd0a1dcc567e61e74386fcc Mon Sep 17 00:00:00 2001 From: Farhan Khan Date: Wed, 16 Dec 2020 02:41:45 +0000 Subject: [PATCH] using the same http client, adding it to running instances some web work lost track of what's going on and I'm not accountable to anyone...so screw it --- engine/db.go | 4 +- engine/headers.go | 26 ++++++++++++- engine/instance.go | 24 ++++++------ engine/poll.go | 7 ++-- engine/stream.go | 2 +- engine/web.go | 93 ++++++++++++++++++++++++++++------------------ 6 files changed, 98 insertions(+), 58 deletions(-) diff --git a/engine/db.go b/engine/db.go index 6c30aef..27989fd 100644 --- a/engine/db.go +++ b/engine/db.go @@ -41,12 +41,12 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) { } // Insert new post if new - _, err = conn.Exec(context.Background(), "INSERT INTO posts (uri, content, created_at, normalized, account_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, accountid, reportpost.posthash) + _, err = conn.Exec(context.Background(), "INSERT INTO posts (uri, content, created_at, normalized, account_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Uri, reportpost.Content, reportpost.Created_at, reportpost.normalized, accountid, reportpost.posthash) if err != nil { // For now I want to know why this failed. log.Print("Second ", err) log.Print("--------------------------") log.Print("Reported error: ", err) - log.Print("Url: ", reportpost.Url) + log.Print("Uri: ", reportpost.Uri) log.Print("Content: ", reportpost.Content) log.Print("Created_at: ", reportpost.Created_at) log.Print("normalized: ", reportpost.normalized) diff --git a/engine/headers.go b/engine/headers.go index 437f21f..2aa57c0 100644 --- a/engine/headers.go +++ b/engine/headers.go @@ -1,5 +1,9 @@ package main +import ( + "net/http" +) + const ( NEW_INSTANCE = 0 RUNNING = 200 @@ -22,7 +26,7 @@ type ReportPost struct { // Retrieved values Id string `json:"id"` - Url string `json:"uri"` + Uri string `json:"uri"` Account AccountType Content string `json:"content"` Created_at string `json:"created_at"` @@ -38,7 +42,7 @@ type AccountType struct { Bot bool `json:"bot"` Created_at string `json:"created_at"` Display_name string `json:"display_name"` - Url string `json:"uri"` + Url string `json:"url"` } // Instance's new min_id value @@ -47,6 +51,7 @@ type RunningInstance struct { Status int `json:"status"` LastRun string `json:"lastrun"` CaptureType string `json:"capturetype"` + client *http.Client } type NodeInfoSoftware struct { @@ -68,3 +73,20 @@ type ResponseBack struct { Message string `json:"Message"` RunningInstances map[string]RunningInstance `json:"RunningInstances"` } + +type Userinfo struct { + Id string `"json:id"` + Type string `"json:type"` + Following string `"json:following"` + Followers string `"json:followers"` + Inbox string `"json:inbox"` + Outbox string `"json:outbox"` + Featured string `"json:featured"` + PreferredUsername string `"json:preferredUsername"` + Name string `"json:name"` + Summary string `"json:summary"` + Url string `"json:Url"` + ManuallyApprovesFollowers string `"json:manuallyApprovesFollowers"` + Discoverable string `"json:discoverable"` +} + diff --git a/engine/instance.go b/engine/instance.go index 6fb773e..34afce0 100644 --- a/engine/instance.go +++ b/engine/instance.go @@ -15,17 +15,16 @@ var p *bluemonday.Policy var spaceReg *regexp.Regexp // Change this to return a proper "err" -func GetNodeInfo(endpoint string) (NodeInfo) { +func GetNodeInfo(endpoint string) (http.Client, NodeInfo) { /* Checking order * Mastodon/Pleroma * Um..nothing else yet */ pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json" - //http_client := http.Client{Timeout: 10 * time.Second} http_client := http.Client{} pleromastodon_api_resp, err := http_client.Get(pleromastodon_nodeinfo_uri) if err != nil { - return NodeInfo{} + return http_client, NodeInfo{} } else { defer pleromastodon_api_resp.Body.Close() } @@ -35,7 +34,7 @@ func GetNodeInfo(endpoint string) (NodeInfo) { err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo) if err == nil { defer pleromastodon_api_resp.Body.Close() - return nodeinfo + return http_client, nodeinfo } } @@ -44,13 +43,13 @@ func GetNodeInfo(endpoint string) (NodeInfo) { resp_index, err := http_client.Get(index_uri) if err != nil { log.Print("Unable to connect to " + endpoint + ", giving up") - return NodeInfo{} + return http_client, NodeInfo{} } defer resp_index.Body.Close() indexbin, err := ioutil.ReadAll(resp_index.Body) if err != nil { log.Print("Unable to read index of " + endpoint + ", giving up") - return NodeInfo{} + return http_client, NodeInfo{} } indexstr := string(indexbin) nodeinfo := NodeInfo{} @@ -68,30 +67,29 @@ func GetNodeInfo(endpoint string) (NodeInfo) { nodeinfo.Software.Version = "guess" } - return nodeinfo + return http_client, nodeinfo } func StartInstance(endpoint string, reportPostChan chan ReportPost) { - nodeinfo := GetNodeInfo(endpoint) + http_client, nodeinfo := GetNodeInfo(endpoint) + ri_mutex.Lock() + m := runninginstances[endpoint] if nodeinfo.Software.Name == "" { - m := runninginstances[endpoint] m.Software = "" m.LastRun = time.Now().Format(time.RFC3339) m.Status = UNSUPPORTED_INSTANCE - ri_mutex.Lock() runninginstances[endpoint] = m ri_mutex.Unlock() return } - ri_mutex.Lock() - m := runninginstances[endpoint] + m.client = &http_client if nodeinfo.Software.Name == "pleroma" { log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name) m.CaptureType = "Poll" runninginstances[endpoint] = m ri_mutex.Unlock() - PollMastodonPleroma(endpoint, reportPostChan) + PollMastodonPleroma(endpoint, reportPostChan, http_client) } else if nodeinfo.Software.Name == "mastodon" { log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name) m.CaptureType = "Stream" diff --git a/engine/poll.go b/engine/poll.go index 69c4af9..a3a845e 100644 --- a/engine/poll.go +++ b/engine/poll.go @@ -12,12 +12,12 @@ import ( "log" ) -func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { +func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost, http_client http.Client) { newposts := make([]ReportPost, 0) min_id := "" - http_client := http.Client{} +// http_client := http.Client{} parsing_error := 0 unprocess_error := 0 use_auth := false @@ -46,7 +46,6 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { } } - for { ri_mutex.Lock() m := runninginstances[endpoint] @@ -151,7 +150,7 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) { } // Calculate the post hash - fmt.Fprint(posthash, newpost.Url) + fmt.Fprint(posthash, newpost.Uri) fmt.Fprint(posthash, newpost.normalized) fmt.Fprint(posthash, newpost.Account.Acct) fmt.Fprint(posthash, newpost.Account.Display_name) diff --git a/engine/stream.go b/engine/stream.go index 41f74b6..d9369e1 100644 --- a/engine/stream.go +++ b/engine/stream.go @@ -106,7 +106,7 @@ func StreamMastodon(endpoint string, reportPostChan chan ReportPost) { } // Calculate the post hash - fmt.Fprint(posthash, newpost.Url) + fmt.Fprint(posthash, newpost.Uri) fmt.Fprint(posthash, newpost.normalized) fmt.Fprint(posthash, newpost.Account.Acct) fmt.Fprint(posthash, newpost.Account.Display_name) diff --git a/engine/web.go b/engine/web.go index 2be0bbb..b95f0d7 100644 --- a/engine/web.go +++ b/engine/web.go @@ -6,6 +6,9 @@ import ( "net/http" "encoding/json" "io/ioutil" + "html" + "time" + "strings" "os" ) @@ -91,45 +94,63 @@ func webfinger(w http.ResponseWriter, r *http.Request) { } } -func inbox(w http.ResponseWriter, r *http.Request) { - fmt.Println("PATH --> ", r.URL.Path) +func inboxHandler(reportPostChan chan ReportPost) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + fmt.Println("PATH --> ", r.URL.Path) - body, err := ioutil.ReadAll(r.Body) - if err != nil { - fmt.Println(err) - return - } - - var findtype FindType - err = json.Unmarshal(body, &findtype) - - fmt.Println(findtype.Type) - - switch findtype.Type { - case "Create": - var createobject CreateObject - - err = json.Unmarshal(findtype.Object, &createobject) + body, err := ioutil.ReadAll(r.Body) if err != nil { - fmt.Println("Ignore the post here") + fmt.Println(err) + return } - fmt.Println("Content: ", createobject.Content) - case "Like": - case "Announcement": - case "Delete": - case "Undo": - default: - fmt.Println("Others --> " + findtype.Type) + + var findtype FindType + err = json.Unmarshal(body, &findtype) + + fmt.Println(string(body)) + + switch findtype.Type { + case "Create": + var createobject CreateObject + + err = json.Unmarshal(findtype.Object, &createobject) + if err != nil { + fmt.Println("Ignore the post here") + } + fmt.Println("Content: ", createobject.Content) + + var newpost ReportPost + newpost.Uri = findtype.Id + newpost.Content = createobject.Content + newpost.Created_at = findtype.Published + newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content))) + newpost.normalized = strings.ReplaceAll(newpost.normalized, "\t", " ") + newpost.normalized = spaceReg.ReplaceAllString(newpost.normalized, " ") + newpost.Account.Acct = "aaa@bbb" + newpost.Account.Avatar = "ttt" + newpost.Account.Bot = false + newpost.Account.Created_at = time.Now().Format(time.RFC3339) + newpost.Account.Display_name = "Fqrhqn" + newpost.Account.Url = "qwerty" + reportPostChan <- newpost + + case "Like": + case "Announcement": + case "Delete": + case "Undo": + default: + fmt.Println("Others --> " + findtype.Type) + } + + fmt.Println("Actor: ", findtype.Actor) + fmt.Println("Cc: ", findtype.Cc) + fmt.Println("Id: ", findtype.Id) + fmt.Println("Published: ", findtype.Published) + fmt.Println("To: ", findtype.To) + fmt.Println("Type: ", findtype.Type) + + fmt.Println("Object: ", string(findtype.Object)) } - - fmt.Println("Actor: ", findtype.Actor) - fmt.Println("Cc: ", findtype.Cc) - fmt.Println("Id: ", findtype.Id) - fmt.Println("Published: ", findtype.Published) - fmt.Println("To: ", findtype.To) - fmt.Println("Type: ", findtype.Type) - - fmt.Println("Object: ", string(findtype.Object)) } func users_fedilogue_followers(w http.ResponseWriter, r *http.Request) { @@ -218,7 +239,7 @@ func errorHandler(w http.ResponseWriter, r *http.Request) { func webmain(reportPostChan chan ReportPost) { http.HandleFunc("/.well-known/webfinger", webfinger) http.HandleFunc("/.well-known/host-meta", hostmeta) - http.HandleFunc("/inbox", inbox) + http.HandleFunc("/inbox", inboxHandler(reportPostChan)) http.HandleFunc("/users/fedilogue", users_fedilogue) http.HandleFunc("/users/fedilogue/followers", users_fedilogue_followers) http.HandleFunc("/users/fedilogue/following", users_fedilogue_following)