diff --git a/retrieve.go b/retrieve.go index e5607ea..a7b810f 100644 --- a/retrieve.go +++ b/retrieve.go @@ -3,7 +3,7 @@ package main import ( "context" "encoding/json" - "errors" + //"errors" "html" "io/ioutil" "net/http" @@ -69,20 +69,22 @@ type PostJson struct { instance string } -func check_activity(uri string) (PostJson, error) { +func check_activity(uri string) { var activityjson PostJson // Ignore banned for _, banned := range settings.Banned { if strings.Index(uri, "https://"+banned+"/") == 0 { - return activityjson, errors.New("Banned instance") + //return activityjson, errors.New("Banned instance") + return } } // Ignore invalid URIs endslash := strings.Index(uri[8:], "/") if endslash == -1 { - return activityjson, errors.New("Invalid URI " + uri) + //return activityjson, errors.New("Invalid URI " + uri) + return } activityjson.instance = uri[8 : endslash+8] @@ -92,7 +94,8 @@ func check_activity(uri string) (PostJson, error) { o.recentactivities.mu.Lock() if o.recentactivities.Add(uri) == true { o.recentactivities.mu.Unlock() - return activityjson, errors.New("Recently requested within local cache") + //return activityjson, errors.New("Recently requested within local cache") + return } o.recentactivities.mu.Unlock() @@ -103,7 +106,8 @@ func check_activity(uri string) (PostJson, error) { err := selectRet.Scan(&activityjson.id, &jsonmap) if err == nil { /////////// BETTER RETURN VALUES!!!!! - return activityjson, nil + //return activityjson, nil + return } req, _ := http.NewRequest("GET", uri, nil) @@ -112,18 +116,21 @@ func check_activity(uri string) (PostJson, error) { resp, err := DoTries(&o, req) if err != nil { - return activityjson, errors.New("Connection error to " + uri) + //return activityjson, errors.New("Connection error to " + uri) + return } body, err := ioutil.ReadAll(resp.Body) if err != nil { - return activityjson, errors.New("Read error on " + uri) + //return activityjson, errors.New("Read error on " + uri) + return } resp.Body.Close() jsondocument = string(body) err = json.Unmarshal(body, &activityjson) if err != nil { - return activityjson, err + return + //return activityjson, err } if activityjson.InReplyTo != "" && activityjson.InReplyTo != uri { @@ -135,13 +142,11 @@ func check_activity(uri string) (PostJson, error) { // If AttributedTo is blank, this is likely an authentication failure // For now, skip it... if activityjson.AttributedTo == "" { - return activityjson, errors.New("Invalid AttributedTo value on " + uri) + //return activityjson, errors.New("Invalid AttributedTo value on " + uri) + return } - _, err = check_actor(activityjson.AttributedTo) // This must be done BEFORE the `INSERT INTO activities'` below - if err != nil { - return activityjson, err - } + go check_actor(activityjson.AttributedTo) // This must be done BEFORE the `INSERT INTO activities'` below activityjson.normalized = removeHTMLReg.ReplaceAllString(activityjson.Content, " ") activityjson.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(activityjson.normalized))) @@ -151,7 +156,8 @@ func check_activity(uri string) (PostJson, error) { _, err = pool.Exec(context.Background(), "INSERT INTO activities (document, normalized, instance) VALUES($1, $2, $3)", jsondocument, activityjson.normalized, activityjson.instance) if err != nil { logWarn.Printf("Error inserting %s into `activities`: %s", uri, err) - return activityjson, err + //return activityjson, err + return } for _, to := range activityjson.To { @@ -164,19 +170,21 @@ func check_activity(uri string) (PostJson, error) { } } - return activityjson, nil + //return activityjson, nil } -func check_actor(uri string) (ActorJson, error) { +func check_actor(uri string) { var actorjson ActorJson endslash := strings.Index(uri[8:], "/") if endslash == -1 { - return actorjson, errors.New("Invalid user: " + uri) +// return actorjson, errors.New("Invalid user: " + uri) + return } actorjson.instance = uri[8 : endslash+8] for _, banned := range settings.Banned { if strings.Index(uri, "https://"+banned+"/") == 0 { - return actorjson, errors.New("Banned instance") +// return actorjson, errors.New("Banned instance") + return } } @@ -185,7 +193,8 @@ func check_actor(uri string) (ActorJson, error) { o.recentactors.mu.Lock() if o.recentactors.Add(uri) == true { o.recentactors.mu.Unlock() - return actorjson, errors.New("Recently requested actor within local cache") + return +// return actorjson, errors.New("Recently requested actor within local cache") } o.recentactors.mu.Unlock() @@ -194,7 +203,8 @@ func check_actor(uri string) (ActorJson, error) { err := selectRet.Scan(&actorjson.id, &jsonmap, &actorjson.instance) if err == nil { ///////// BETTER RETURN VALUES //////// - return actorjson, nil +// return actorjson, nil + return } req, _ := http.NewRequest("GET", uri, nil) @@ -208,7 +218,8 @@ func check_actor(uri string) (ActorJson, error) { if err != nil { if tries > 10 { logErr.Print("Unable to connect to "+uri+" attempt 10/10, giving up.") - return actorjson, err +// return actorjson, err + return } logWarn.Print("Unable to connect to "+uri+", attempt ",tries+1,"+/10 sleeping for 30 seconds.") time.Sleep(time.Second * 30) @@ -220,7 +231,8 @@ func check_actor(uri string) (ActorJson, error) { body, err := ioutil.ReadAll(resp.Body) if err != nil { - return actorjson, errors.New("Read error on " + uri) +// return actorjson, errors.New("Read error on " + uri) + return } resp.Body.Close() @@ -228,14 +240,16 @@ func check_actor(uri string) (ActorJson, error) { err = json.Unmarshal(body, &actorjson) if err != nil { - return actorjson, err +// return actorjson, err + return } _, err = pool.Exec(context.Background(), "INSERT INTO actors (document, instance) VALUES($1, $2)", jsondocument, actorjson.instance) if err != nil { logWarn.Printf("Error inserting %s into `actors`: %s", uri, err) - return actorjson, err +// return actorjson, err + return } - return actorjson, nil +// return actorjson, nil }