farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

decoupling retrieval of actor from activity in database

Browse Source
This commit is contained in:
farhan 2021-02-10 22:30:34 +00:00
parent 6efd723aac
commit 126aac8c81
1 changed files with 40 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
retrieve.go
View File

@ -3,7 +3,7 @@ package main
import (
"context"
"encoding/json"
"errors"
//"errors"
"html"
"io/ioutil"
"net/http"
@ -69,20 +69,22 @@ type PostJson struct {
instance string
}
func check_activity(uri string) (PostJson, error) {
func check_activity(uri string) {
var activityjson PostJson
// Ignore banned
for _, banned := range settings.Banned {
if strings.Index(uri, "https://"+banned+"/") == 0 {
return activityjson, errors.New("Banned instance")
//return activityjson, errors.New("Banned instance")
return
}
}
// Ignore invalid URIs
endslash := strings.Index(uri[8:], "/")
if endslash == -1 {
return activityjson, errors.New("Invalid URI " + uri)
//return activityjson, errors.New("Invalid URI " + uri)
return
}
activityjson.instance = uri[8 : endslash+8]
@ -92,7 +94,8 @@ func check_activity(uri string) (PostJson, error) {
o.recentactivities.mu.Lock()
if o.recentactivities.Add(uri) == true {
o.recentactivities.mu.Unlock()
return activityjson, errors.New("Recently requested within local cache")
//return activityjson, errors.New("Recently requested within local cache")
return
}
o.recentactivities.mu.Unlock()
@ -103,7 +106,8 @@ func check_activity(uri string) (PostJson, error) {
err := selectRet.Scan(&activityjson.id, &jsonmap)
if err == nil {
/////////// BETTER RETURN VALUES!!!!!
return activityjson, nil
//return activityjson, nil
return
}
req, _ := http.NewRequest("GET", uri, nil)
@ -112,18 +116,21 @@ func check_activity(uri string) (PostJson, error) {
resp, err := DoTries(&o, req)
if err != nil {
return activityjson, errors.New("Connection error to " + uri)
//return activityjson, errors.New("Connection error to " + uri)
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return activityjson, errors.New("Read error on " + uri)
//return activityjson, errors.New("Read error on " + uri)
return
}
resp.Body.Close()
jsondocument = string(body)
err = json.Unmarshal(body, &activityjson)
if err != nil {
return activityjson, err
return
//return activityjson, err
}
if activityjson.InReplyTo != "" && activityjson.InReplyTo != uri {
@ -135,13 +142,11 @@ func check_activity(uri string) (PostJson, error) {
// If AttributedTo is blank, this is likely an authentication failure
// For now, skip it...
if activityjson.AttributedTo == "" {
return activityjson, errors.New("Invalid AttributedTo value on " + uri)
//return activityjson, errors.New("Invalid AttributedTo value on " + uri)
return
}
_, err = check_actor(activityjson.AttributedTo) // This must be done BEFORE the `INSERT INTO activities'` below
if err != nil {
return activityjson, err
}
go check_actor(activityjson.AttributedTo) // This must be done BEFORE the `INSERT INTO activities'` below
activityjson.normalized = removeHTMLReg.ReplaceAllString(activityjson.Content, " ")
activityjson.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(activityjson.normalized)))
@ -151,7 +156,8 @@ func check_activity(uri string) (PostJson, error) {
_, err = pool.Exec(context.Background(), "INSERT INTO activities (document, normalized, instance) VALUES($1, $2, $3)", jsondocument, activityjson.normalized, activityjson.instance)
if err != nil {
logWarn.Printf("Error inserting %s into `activities`: %s", uri, err)
return activityjson, err
//return activityjson, err
return
}
for _, to := range activityjson.To {
@ -164,19 +170,21 @@ func check_activity(uri string) (PostJson, error) {
}
}
return activityjson, nil
//return activityjson, nil
}
func check_actor(uri string) (ActorJson, error) {
func check_actor(uri string) {
var actorjson ActorJson
endslash := strings.Index(uri[8:], "/")
if endslash == -1 {
return actorjson, errors.New("Invalid user: " + uri)
// return actorjson, errors.New("Invalid user: " + uri)
return
}
actorjson.instance = uri[8 : endslash+8]
for _, banned := range settings.Banned {
if strings.Index(uri, "https://"+banned+"/") == 0 {
return actorjson, errors.New("Banned instance")
// return actorjson, errors.New("Banned instance")
return
}
}
@ -185,7 +193,8 @@ func check_actor(uri string) (ActorJson, error) {
o.recentactors.mu.Lock()
if o.recentactors.Add(uri) == true {
o.recentactors.mu.Unlock()
return actorjson, errors.New("Recently requested actor within local cache")
return
// return actorjson, errors.New("Recently requested actor within local cache")
}
o.recentactors.mu.Unlock()
@ -194,7 +203,8 @@ func check_actor(uri string) (ActorJson, error) {
err := selectRet.Scan(&actorjson.id, &jsonmap, &actorjson.instance)
if err == nil {
///////// BETTER RETURN VALUES ////////
return actorjson, nil
// return actorjson, nil
return
}
req, _ := http.NewRequest("GET", uri, nil)
@ -208,7 +218,8 @@ func check_actor(uri string) (ActorJson, error) {
if err != nil {
if tries > 10 {
logErr.Print("Unable to connect to "+uri+" attempt 10/10, giving up.")
return actorjson, err
// return actorjson, err
return
}
logWarn.Print("Unable to connect to "+uri+", attempt ",tries+1,"+/10 sleeping for 30 seconds.")
time.Sleep(time.Second * 30)
@ -220,7 +231,8 @@ func check_actor(uri string) (ActorJson, error) {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return actorjson, errors.New("Read error on " + uri)
// return actorjson, errors.New("Read error on " + uri)
return
}
resp.Body.Close()
@ -228,14 +240,16 @@ func check_actor(uri string) (ActorJson, error) {
err = json.Unmarshal(body, &actorjson)
if err != nil {
return actorjson, err
// return actorjson, err
return
}
_, err = pool.Exec(context.Background(), "INSERT INTO actors (document, instance) VALUES($1, $2)", jsondocument, actorjson.instance)
if err != nil {
logWarn.Printf("Error inserting %s into `actors`: %s", uri, err)
return actorjson, err
// return actorjson, err
return
}
return actorjson, nil
// return actorjson, nil
}