farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

using the same http client, adding it to running instances

Browse Source 
some web work
lost track of what's going on and I'm not accountable to anyone...so screw it
This commit is contained in:
farhan 2020-12-16 02:41:45 +00:00
parent 3d196ccf41
commit a50916eb36
6 changed files with 98 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
engine/db.go
View File

@ -41,12 +41,12 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
}
// Insert new post if new
_, err = conn.Exec(context.Background(), "INSERT INTO posts (uri, content, created_at, normalized, account_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, accountid, reportpost.posthash)
_, err = conn.Exec(context.Background(), "INSERT INTO posts (uri, content, created_at, normalized, account_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Uri, reportpost.Content, reportpost.Created_at, reportpost.normalized, accountid, reportpost.posthash)
if err != nil { // For now I want to know why this failed.
log.Print("Second ", err)
log.Print("--------------------------")
log.Print("Reported error: ", err)
log.Print("Url: ", reportpost.Url)
log.Print("Uri: ", reportpost.Uri)
log.Print("Content: ", reportpost.Content)
log.Print("Created_at: ", reportpost.Created_at)
log.Print("normalized: ", reportpost.normalized)

26
engine/headers.go
View File

@ -1,5 +1,9 @@
package main
import (
"net/http"
)
const (
NEW_INSTANCE = 0
RUNNING = 200
@ -22,7 +26,7 @@ type ReportPost struct {
// Retrieved values
Id string `json:"id"`
Url string `json:"uri"`
Uri string `json:"uri"`
Account AccountType
Content string `json:"content"`
Created_at string `json:"created_at"`
@ -38,7 +42,7 @@ type AccountType struct {
Bot bool `json:"bot"`
Created_at string `json:"created_at"`
Display_name string `json:"display_name"`
Url string `json:"uri"`
Url string `json:"url"`
}
// Instance's new min_id value
@ -47,6 +51,7 @@ type RunningInstance struct {
Status int `json:"status"`
LastRun string `json:"lastrun"`
CaptureType string `json:"capturetype"`
client *http.Client
}
type NodeInfoSoftware struct {
@ -68,3 +73,20 @@ type ResponseBack struct {
Message string `json:"Message"`
RunningInstances map[string]RunningInstance `json:"RunningInstances"`
}
type Userinfo struct {
Id string `"json:id"`
Type string `"json:type"`
Following string `"json:following"`
Followers string `"json:followers"`
Inbox string `"json:inbox"`
Outbox string `"json:outbox"`
Featured string `"json:featured"`
PreferredUsername string `"json:preferredUsername"`
Name string `"json:name"`
Summary string `"json:summary"`
Url string `"json:Url"`
ManuallyApprovesFollowers string `"json:manuallyApprovesFollowers"`
Discoverable string `"json:discoverable"`
}

24
engine/instance.go
View File

@ -15,17 +15,16 @@ var p *bluemonday.Policy
var spaceReg *regexp.Regexp
// Change this to return a proper "err"
func GetNodeInfo(endpoint string) (NodeInfo) {
func GetNodeInfo(endpoint string) (http.Client, NodeInfo) {
/* Checking order
* Mastodon/Pleroma
* Um..nothing else yet
*/
pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"
//http_client := http.Client{Timeout: 10 * time.Second}
http_client := http.Client{}
pleromastodon_api_resp, err := http_client.Get(pleromastodon_nodeinfo_uri)
if err != nil {
return NodeInfo{}
return http_client, NodeInfo{}
} else {
defer pleromastodon_api_resp.Body.Close()
}
@ -35,7 +34,7 @@ func GetNodeInfo(endpoint string) (NodeInfo) {
err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo)
if err == nil {
defer pleromastodon_api_resp.Body.Close()
return nodeinfo
return http_client, nodeinfo
}
}
@ -44,13 +43,13 @@ func GetNodeInfo(endpoint string) (NodeInfo) {
resp_index, err := http_client.Get(index_uri)
if err != nil {
log.Print("Unable to connect to " + endpoint + ", giving up")
return NodeInfo{}
return http_client, NodeInfo{}
}
defer resp_index.Body.Close()
indexbin, err := ioutil.ReadAll(resp_index.Body)
if err != nil {
log.Print("Unable to read index of " + endpoint + ", giving up")
return NodeInfo{}
return http_client, NodeInfo{}
}
indexstr := string(indexbin)
nodeinfo := NodeInfo{}
@ -68,30 +67,29 @@ func GetNodeInfo(endpoint string) (NodeInfo) {
nodeinfo.Software.Version = "guess"
}
return nodeinfo
return http_client, nodeinfo
}
func StartInstance(endpoint string, reportPostChan chan ReportPost) {
nodeinfo := GetNodeInfo(endpoint)
http_client, nodeinfo := GetNodeInfo(endpoint)
ri_mutex.Lock()
m := runninginstances[endpoint]
if nodeinfo.Software.Name == "" {
m := runninginstances[endpoint]
m.Software = ""
m.LastRun = time.Now().Format(time.RFC3339)
m.Status = UNSUPPORTED_INSTANCE
ri_mutex.Lock()
runninginstances[endpoint] = m
ri_mutex.Unlock()
return
}
ri_mutex.Lock()
m := runninginstances[endpoint]
m.client = &http_client
if nodeinfo.Software.Name == "pleroma" {
log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
m.CaptureType = "Poll"
runninginstances[endpoint] = m
ri_mutex.Unlock()
PollMastodonPleroma(endpoint, reportPostChan)
PollMastodonPleroma(endpoint, reportPostChan, http_client)
} else if nodeinfo.Software.Name == "mastodon" {
log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
m.CaptureType = "Stream"

7
engine/poll.go
View File

@ -12,12 +12,12 @@ import (
"log"
)
func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost, http_client http.Client) {
newposts := make([]ReportPost, 0)
min_id := ""
http_client := http.Client{}
// http_client := http.Client{}
parsing_error := 0
unprocess_error := 0
use_auth := false
@ -46,7 +46,6 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
}
}
for {
ri_mutex.Lock()
m := runninginstances[endpoint]
@ -151,7 +150,7 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
}
// Calculate the post hash
fmt.Fprint(posthash, newpost.Url)
fmt.Fprint(posthash, newpost.Uri)
fmt.Fprint(posthash, newpost.normalized)
fmt.Fprint(posthash, newpost.Account.Acct)
fmt.Fprint(posthash, newpost.Account.Display_name)

2
engine/stream.go
View File

@ -106,7 +106,7 @@ func StreamMastodon(endpoint string, reportPostChan chan ReportPost) {
}
// Calculate the post hash
fmt.Fprint(posthash, newpost.Url)
fmt.Fprint(posthash, newpost.Uri)
fmt.Fprint(posthash, newpost.normalized)
fmt.Fprint(posthash, newpost.Account.Acct)
fmt.Fprint(posthash, newpost.Account.Display_name)

93
engine/web.go
View File

@ -6,6 +6,9 @@ import (
"net/http"
"encoding/json"
"io/ioutil"
"html"
"time"
"strings"
"os"
)
@ -91,45 +94,63 @@ func webfinger(w http.ResponseWriter, r *http.Request) {
}
}
func inbox(w http.ResponseWriter, r *http.Request) {
fmt.Println("PATH --> ", r.URL.Path)
func inboxHandler(reportPostChan chan ReportPost) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
fmt.Println("PATH --> ", r.URL.Path)
body, err := ioutil.ReadAll(r.Body)
if err != nil {
fmt.Println(err)
return
}
var findtype FindType
err = json.Unmarshal(body, &findtype)
fmt.Println(findtype.Type)
switch findtype.Type {
case "Create":
var createobject CreateObject
err = json.Unmarshal(findtype.Object, &createobject)
body, err := ioutil.ReadAll(r.Body)
if err != nil {
fmt.Println("Ignore the post here")
fmt.Println(err)
return
}
fmt.Println("Content: ", createobject.Content)
case "Like":
case "Announcement":
case "Delete":
case "Undo":
default:
fmt.Println("Others --> " + findtype.Type)
var findtype FindType
err = json.Unmarshal(body, &findtype)
fmt.Println(string(body))
switch findtype.Type {
case "Create":
var createobject CreateObject
err = json.Unmarshal(findtype.Object, &createobject)
if err != nil {
fmt.Println("Ignore the post here")
}
fmt.Println("Content: ", createobject.Content)
var newpost ReportPost
newpost.Uri = findtype.Id
newpost.Content = createobject.Content
newpost.Created_at = findtype.Published
newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content)))
newpost.normalized = strings.ReplaceAll(newpost.normalized, "\t", " ")
newpost.normalized = spaceReg.ReplaceAllString(newpost.normalized, " ")
newpost.Account.Acct = "aaa@bbb"
newpost.Account.Avatar = "ttt"
newpost.Account.Bot = false
newpost.Account.Created_at = time.Now().Format(time.RFC3339)
newpost.Account.Display_name = "Fqrhqn"
newpost.Account.Url = "qwerty"
reportPostChan <- newpost
case "Like":
case "Announcement":
case "Delete":
case "Undo":
default:
fmt.Println("Others --> " + findtype.Type)
}
fmt.Println("Actor: ", findtype.Actor)
fmt.Println("Cc: ", findtype.Cc)
fmt.Println("Id: ", findtype.Id)
fmt.Println("Published: ", findtype.Published)
fmt.Println("To: ", findtype.To)
fmt.Println("Type: ", findtype.Type)
fmt.Println("Object: ", string(findtype.Object))
}
fmt.Println("Actor: ", findtype.Actor)
fmt.Println("Cc: ", findtype.Cc)
fmt.Println("Id: ", findtype.Id)
fmt.Println("Published: ", findtype.Published)
fmt.Println("To: ", findtype.To)
fmt.Println("Type: ", findtype.Type)
fmt.Println("Object: ", string(findtype.Object))
}
func users_fedilogue_followers(w http.ResponseWriter, r *http.Request) {
@ -218,7 +239,7 @@ func errorHandler(w http.ResponseWriter, r *http.Request) {
func webmain(reportPostChan chan ReportPost) {
http.HandleFunc("/.well-known/webfinger", webfinger)
http.HandleFunc("/.well-known/host-meta", hostmeta)
http.HandleFunc("/inbox", inbox)
http.HandleFunc("/inbox", inboxHandler(reportPostChan))
http.HandleFunc("/users/fedilogue", users_fedilogue)
http.HandleFunc("/users/fedilogue/followers", users_fedilogue_followers)
http.HandleFunc("/users/fedilogue/following", users_fedilogue_following)