farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

renaming posts to activities and accounts to actors

Browse Source 
removed old posthash mechanism
This commit is contained in:
farhan 2021-02-01 12:52:42 +00:00
parent 102ddbe41c
commit 3662535b0d
6 changed files with 92 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
headers.go
View File

@ -27,7 +27,7 @@ type ObjectType struct {
} }
// Parsing Unmarshal JSON type // Parsing Unmarshal JSON type
type ReportPost struct { type ReportActivity struct {
// Retrieved values // Retrieved values
Id string `json:"id"` Id string `json:"id"`
@ -38,7 +38,6 @@ type ReportPost struct {
// Derived values // Derived values
normalized string normalized string
posthash []byte
} }
type AccountType struct { type AccountType struct {

14
poll.go
View File

@ -47,7 +47,7 @@ type PostInfo struct {
} }
func PollMastodonPleroma(endpoint string, o *RunningInstance) { func PollMastodonPleroma(endpoint string, o *RunningInstance) {
newposts := make([]ReportPost, 0) newactivities := make([]ReportActivity, 0)
min_id := "" min_id := ""
@ -147,7 +147,7 @@ func PollMastodonPleroma(endpoint string, o *RunningInstance) {
return return
} }
err = json.NewDecoder(resp.Body).Decode(&newposts) err = json.NewDecoder(resp.Body).Decode(&newactivities)
resp.Body.Close() // Release as soon as done resp.Body.Close() // Release as soon as done
if err != nil { if err != nil {
if parsing_error > 5 { if parsing_error > 5 {
@ -168,15 +168,15 @@ func PollMastodonPleroma(endpoint string, o *RunningInstance) {
runninginstances[endpoint] = m runninginstances[endpoint] = m
ri_mutex.Unlock() ri_mutex.Unlock()
for _, newpost := range newposts { for _, newactivity := range newactivities {
go check_post(newpost.Uri) go check_activity(newactivity.Uri)
matchset := re.FindStringSubmatch(newpost.Uri) matchset := re.FindStringSubmatch(newactivity.Uri)
if matchset != nil { if matchset != nil {
newinstance := matchset[1] newinstance := matchset[1]
// Check min_id // Check min_id
if newpost.Id > min_id { if newactivity.Id > min_id {
min_id = newpost.Id min_id = newactivity.Id
} }
go CheckInstance(newinstance, endpoint) go CheckInstance(newinstance, endpoint)

94
retrieve.go
View File

@ -29,7 +29,7 @@ type PublicKeyType struct {
PublicKeyPem string `json:"publicKeyPem"` PublicKeyPem string `json:"publicKeyPem"`
} }
type UserJson struct { type ActorJson struct {
ID string `json:"id"` ID string `json:"id"`
Type string `json:"type"` Type string `json:"type"`
Inbox string `json:"inbox"` Inbox string `json:"inbox"`
@ -52,7 +52,6 @@ type PostJson struct {
InReplyTo string `json:"inReplyTo"` InReplyTo string `json:"inReplyTo"`
normalized string normalized string
posthash []byte
receivedAt time.Time `json:"created_at"` receivedAt time.Time `json:"created_at"`
Content string `json:"content"` Content string `json:"content"`
@ -70,34 +69,34 @@ type PostJson struct {
instance string instance string
} }
func check_post(uri string) (PostJson, error) { func check_activity(uri string) (PostJson, error) {
var postjson PostJson var activityjson PostJson
// Ignore banned // Ignore banned
for _, banned := range settings.Banned { for _, banned := range settings.Banned {
if strings.Index(uri, "https://"+banned+"/") == 0 { if strings.Index(uri, "https://"+banned+"/") == 0 {
return postjson, errors.New("Banned instance") return activityjson, errors.New("Banned instance")
} }
} }
// Ignore invalid URIs // Ignore invalid URIs
endslash := strings.Index(uri[8:], "/") endslash := strings.Index(uri[8:], "/")
if endslash == -1 { if endslash == -1 {
return postjson, errors.New("Invalid URI " + uri) return activityjson, errors.New("Invalid URI " + uri)
} }
postjson.instance = uri[8 : endslash+8] activityjson.instance = uri[8 : endslash+8]
o, _ := GetRunner(postjson.instance) o, _ := GetRunner(activityjson.instance)
// Check if there were any recent requests on this // Check if there were any recent requests on this
if o.recenturis.Add(uri) != -1 { if o.recenturis.Add(uri) != -1 {
return postjson, errors.New("Recently requested within local cache") return activityjson, errors.New("Recently requested within local cache")
} }
selectRet := pool.QueryRow(context.Background(), "SELECT id, inReplyTo, published, summary, content, normalized, attributedto, posthash, received_at FROM posts WHERE id = $1", uri) selectRet := pool.QueryRow(context.Background(), "SELECT id, inReplyTo, published, summary, content, normalized, attributedto, received_at FROM activities WHERE id = $1", uri)
err := selectRet.Scan(&postjson.ID, &postjson.InReplyTo, &postjson.Published, &postjson.Summary, &postjson.Content, &postjson.normalized, &postjson.AttributedTo, &postjson.posthash, &postjson.receivedAt) err := selectRet.Scan(&activityjson.ID, &activityjson.InReplyTo, &activityjson.Published, &activityjson.Summary, &activityjson.Content, &activityjson.normalized, &activityjson.AttributedTo, &activityjson.receivedAt)
if err == nil { if err == nil {
return postjson, nil return activityjson, nil
} }
req, _ := http.NewRequest("GET", uri, nil) req, _ := http.NewRequest("GET", uri, nil)
@ -106,81 +105,82 @@ func check_post(uri string) (PostJson, error) {
resp, err := DoTries(&o, req) resp, err := DoTries(&o, req)
if err != nil { if err != nil {
return postjson, errors.New("Connection error to " + uri) return activityjson, errors.New("Connection error to " + uri)
} }
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return postjson, errors.New("Read error on " + uri) return activityjson, errors.New("Read error on " + uri)
} }
resp.Body.Close() resp.Body.Close()
err = json.Unmarshal(body, &postjson) err = json.Unmarshal(body, &activityjson)
if err != nil { if err != nil {
return postjson, err return activityjson, err
} }
if postjson.InReplyTo != "" && postjson.InReplyTo != uri { if activityjson.InReplyTo != "" && activityjson.InReplyTo != uri {
if postjson.InReplyTo != uri { if activityjson.InReplyTo != uri {
go check_post(postjson.InReplyTo) go check_actor(activityjson.InReplyTo)
} }
} }
// If AttributedTo is blank, this is likely an authentication failure // If AttributedTo is blank, this is likely an authentication failure
// For now, skip it... // For now, skip it...
if postjson.AttributedTo == "" { if activityjson.AttributedTo == "" {
return postjson, errors.New("Invalid AttributedTo value on " + uri) return activityjson, errors.New("Invalid AttributedTo value on " + uri)
} }
_, err = check_user(postjson.AttributedTo) // This must be done BEFORE the `INSERT INTO posts` below _, err = check_actor(activityjson.AttributedTo) // This must be done BEFORE the `INSERT INTO activities'` below
if err != nil { if err != nil {
return postjson, err return activityjson, err
} }
postjson.normalized = removeHTMLReg.ReplaceAllString(postjson.Content, " ") activityjson.normalized = removeHTMLReg.ReplaceAllString(activityjson.Content, " ")
postjson.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(postjson.normalized))) activityjson.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(activityjson.normalized)))
postjson.normalized = matchurl.ReplaceAllString(postjson.normalized, "") activityjson.normalized = matchurl.ReplaceAllString(activityjson.normalized, "")
postjson.normalized = spaceReg.ReplaceAllString(postjson.normalized, " ") activityjson.normalized = spaceReg.ReplaceAllString(activityjson.normalized, " ")
_, err = pool.Exec(context.Background(), "INSERT INTO posts (id, inreplyto, published, summary, content, normalized, attributedto, posthash, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)", postjson.ID, postjson.InReplyTo, postjson.Published, postjson.Summary, postjson.Content, postjson.normalized, postjson.AttributedTo, postjson.posthash, postjson.instance) _, err = pool.Exec(context.Background(), "INSERT INTO activities (id, inreplyto, published, summary, content, normalized, attributedto, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8)", activityjson.ID, activityjson.InReplyTo, activityjson.Published, activityjson.Summary, activityjson.Content, activityjson.normalized, activityjson.AttributedTo, activityjson.instance)
if err != nil { if err != nil {
return postjson, err logDebug.Print(err)
return activityjson, err
} }
for _, to := range postjson.To { for _, to := range activityjson.To {
if to != "https://www.w3.org/ns/activitystreams#Public" && to != "" { if to != "https://www.w3.org/ns/activitystreams#Public" && to != "" {
if strings.HasSuffix(to, "/followers") == true { if strings.HasSuffix(to, "/followers") == true {
// This check is very much a bad solution, may consider removing the entire for-loop // This check is very much a bad solution, may consider removing the entire for-loop
continue continue
} }
go check_user(to) go check_actor(to)
} }
} }
return postjson, nil return activityjson, nil
} }
func check_user(uri string) (UserJson, error) { func check_actor(uri string) (ActorJson, error) {
var userjson UserJson var actorjson ActorJson
for _, banned := range settings.Banned { for _, banned := range settings.Banned {
if strings.Index(uri, "https://"+banned+"/") == 0 { if strings.Index(uri, "https://"+banned+"/") == 0 {
return userjson, errors.New("Banned instance") return actorjson, errors.New("Banned instance")
} }
} }
selectRet := pool.QueryRow(context.Background(), "SELECT id, actor_type, inbox, outbox, followers, following, url, preferredUsername, name, summary, icon, image, publicKey, instance FROM accounts WHERE id = $1", uri) selectRet := pool.QueryRow(context.Background(), "SELECT id, actor_type, inbox, outbox, followers, following, url, preferredUsername, name, summary, icon, image, publicKey, instance FROM actors WHERE id = $1", uri)
err := selectRet.Scan(&userjson.ID, &userjson.Type, &userjson.Inbox, &userjson.Outbox, &userjson.Followers, &userjson.Following, &userjson.Url, &userjson.PreferredUsername, &userjson.Name, &userjson.Summary, &userjson.Icon.Url, &userjson.Image.Url, &userjson.PublicKey.PublicKeyPem, &userjson.instance) err := selectRet.Scan(&actorjson.ID, &actorjson.Type, &actorjson.Inbox, &actorjson.Outbox, &actorjson.Followers, &actorjson.Following, &actorjson.Url, &actorjson.PreferredUsername, &actorjson.Name, &actorjson.Summary, &actorjson.Icon.Url, &actorjson.Image.Url, &actorjson.PublicKey.PublicKeyPem, &actorjson.instance)
if err == nil { if err == nil {
return userjson, nil return actorjson, nil
} }
endslash := strings.Index(uri[8:], "/") endslash := strings.Index(uri[8:], "/")
if endslash == -1 { if endslash == -1 {
return userjson, errors.New("Invalid user: " + uri) return actorjson, errors.New("Invalid user: " + uri)
} }
userjson.instance = uri[8 : endslash+8] actorjson.instance = uri[8 : endslash+8]
o, _ := GetRunner(userjson.instance) o, _ := GetRunner(actorjson.instance)
req, _ := http.NewRequest("GET", uri, nil) req, _ := http.NewRequest("GET", uri, nil)
req.Header.Set("User-Agent", "Tusky") req.Header.Set("User-Agent", "Tusky")
req.Header.Add("Accept", "application/ld+json") req.Header.Add("Accept", "application/ld+json")
@ -192,7 +192,7 @@ func check_user(uri string) (UserJson, error) {
if err != nil { if err != nil {
if tries > 10 { if tries > 10 {
logErr.Print("Unable to connect to "+uri+" attempt 10/10, giving up.") logErr.Print("Unable to connect to "+uri+" attempt 10/10, giving up.")
return userjson, err return actorjson, err
} }
logWarn.Print("Unable to connect to "+uri+", attempt ",tries+1,"+/10 sleeping for 30 seconds.") logWarn.Print("Unable to connect to "+uri+", attempt ",tries+1,"+/10 sleeping for 30 seconds.")
time.Sleep(time.Second * 30) time.Sleep(time.Second * 30)
@ -202,21 +202,21 @@ func check_user(uri string) (UserJson, error) {
break break
} }
err = json.NewDecoder(resp.Body).Decode(&userjson) err = json.NewDecoder(resp.Body).Decode(&actorjson)
if err != nil { if err != nil {
// Going forward, this might need to be double-checked, but for now just die // Going forward, this might need to be double-checked, but for now just die
tries = tries + 1 tries = tries + 1
return userjson, err return actorjson, err
} }
io.Copy(ioutil.Discard, resp.Body) io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close() resp.Body.Close()
_, err = pool.Exec(context.Background(), "INSERT INTO accounts (id, actor_type, inbox, outbox, followers, following, url, preferredUsername, name, summary, icon, image, publicKey, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", userjson.ID, userjson.Type, userjson.Inbox, userjson.Outbox, userjson.Followers, userjson.Following, userjson.Url, userjson.PreferredUsername, userjson.Name, userjson.Summary, userjson.Icon.Url, userjson.Image.Url, userjson.PublicKey.PublicKeyPem, userjson.instance) _, err = pool.Exec(context.Background(), "INSERT INTO actors (id, actor_type, inbox, outbox, followers, following, url, preferredUsername, name, summary, icon, image, publicKey, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", actorjson.ID, actorjson.Type, actorjson.Inbox, actorjson.Outbox, actorjson.Followers, actorjson.Following, actorjson.Url, actorjson.PreferredUsername, actorjson.Name, actorjson.Summary, actorjson.Icon.Url, actorjson.Image.Url, actorjson.PublicKey.PublicKeyPem, actorjson.instance)
if err != nil { if err != nil {
return userjson, err return actorjson, err
} }
return userjson, nil return actorjson, nil
} }

8
stream.go
View File

@ -74,7 +74,7 @@ func StreamMastodon(endpoint string, o *RunningInstance) {
for s.Scan() { for s.Scan() {
line := s.Text() line := s.Text()
token := strings.SplitN(line, ":", 2) token := strings.SplitN(line, ":", 2)
var newpost ReportPost var newactivity ReportActivity
if len(token) != 2 { if len(token) != 2 {
continue continue
} }
@ -87,7 +87,7 @@ func StreamMastodon(endpoint string, o *RunningInstance) {
switch name { switch name {
case "update": case "update":
jsondata := token[1][1:] jsondata := token[1][1:]
err := json.Unmarshal([]byte(jsondata), &newpost) err := json.Unmarshal([]byte(jsondata), &newactivity)
if err != nil { if err != nil {
logDebug.Print("Unable to parse data from "+endpoint+", but still connected.") logDebug.Print("Unable to parse data from "+endpoint+", but still connected.")
continue continue
@ -100,9 +100,9 @@ func StreamMastodon(endpoint string, o *RunningInstance) {
continue continue
} }
go check_post(newpost.Uri) go check_activity(newactivity.Uri)
matchset := re.FindStringSubmatch(newpost.Uri) matchset := re.FindStringSubmatch(newactivity.Uri)
if matchset != nil { if matchset != nil {
newinstance := matchset[1] newinstance := matchset[1]
go CheckInstance(newinstance, endpoint) go CheckInstance(newinstance, endpoint)

17
tables.sql
View File

@ -1,8 +1,8 @@
DROP TABLE IF EXISTS instances; DROP TABLE IF EXISTS activities CASCADE;
DROP TABLE IF EXISTS posts; DROP TABLE IF EXISTS actors CASCADE;
DROP TABLE IF EXISTS accounts; DROP TABLE IF EXISTS instances CASCADE;
CREATE TABLE accounts ( CREATE TABLE actors (
actor_type VARCHAR(1000) NOT NULL, actor_type VARCHAR(1000) NOT NULL,
id VARCHAR(2083) NOT NULL PRIMARY KEY UNIQUE, id VARCHAR(2083) NOT NULL PRIMARY KEY UNIQUE,
inbox VARCHAR(2083) NOT NULL, inbox VARCHAR(2083) NOT NULL,
@ -20,15 +20,14 @@ CREATE TABLE accounts (
instance VARCHAR(1000) NOT NULL instance VARCHAR(1000) NOT NULL
); );
CREATE TABLE posts ( CREATE TABLE activities (
id VARCHAR(2083) NOT NULL PRIMARY KEY UNIQUE, id VARCHAR(2083) NOT NULL PRIMARY KEY UNIQUE,
inreplyto VARCHAR(2083), inreplyto VARCHAR(2083),
published TIMESTAMP with time zone NOT NULL, published TIMESTAMP with time zone NOT NULL,
summary TEXT, summary TEXT,
content TEXT, content TEXT,
normalized TEXT, normalized TEXT,
attributedto VARCHAR(2083) REFERENCES accounts, attributedto VARCHAR(2083) REFERENCES actors,
posthash bytea,
received_at TIMESTAMP with time zone DEFAULT now(), received_at TIMESTAMP with time zone DEFAULT now(),
instance VARCHAR(1000) NOT NULL instance VARCHAR(1000) NOT NULL
); );
@ -41,3 +40,7 @@ CREATE TABLE instances (
password VARCHAR(32), password VARCHAR(32),
software VARCHAR(50) software VARCHAR(50)
); );
ALTER TABLE activities ADD COLUMN normalized_idx tsvector;
UPDATE activities SET normalized_idx = to_tsvector('english', normalized);
CREATE INDEX ON activities USING gin(normalized_idx);

46
web.go
View File

@ -120,7 +120,7 @@ func inboxHandler() http.HandlerFunc {
return return
} }
go check_post(createobject.ID) go check_activity(createobject.ID)
slashend := strings.Index(createobject.ID[8:], "/") slashend := strings.Index(createobject.ID[8:], "/")
newinstance := createobject.ID[8 : 8+slashend] newinstance := createobject.ID[8 : 8+slashend]
log.Print("The at sign is: ", newinstance) log.Print("The at sign is: ", newinstance)
@ -184,35 +184,35 @@ func usersFedilogue(w http.ResponseWriter, r *http.Request) {
endpoints := map[string]string{"oauthAuthorizationEndpoint": "https://" + host + "/oauth/authorize", "oauthRegistrationEndpoint": "https://" + host + "/api/v1/apps", "oauthTokenEndpoint": "https://" + host + "/oauth/token", "sharedInbox": "https://" + host + "/inbox", "uploadMedia": "https://" + host + "/api/ap/upload_media"} endpoints := map[string]string{"oauthAuthorizationEndpoint": "https://" + host + "/oauth/authorize", "oauthRegistrationEndpoint": "https://" + host + "/api/v1/apps", "oauthTokenEndpoint": "https://" + host + "/oauth/token", "sharedInbox": "https://" + host + "/inbox", "uploadMedia": "https://" + host + "/api/ap/upload_media"}
context := []interface{}{"https://www.w3.org/ns/activitystreams", "https://" + host + "/schemas/litepub-0.1.jsonld", contextlist} context := []interface{}{"https://www.w3.org/ns/activitystreams", "https://" + host + "/schemas/litepub-0.1.jsonld", contextlist}
userjsonmap := make(map[string]interface{}) actorjsonmap := make(map[string]interface{})
userjsonmap["@context"] = context actorjsonmap["@context"] = context
userjsonmap["attachment"] = attachment actorjsonmap["attachment"] = attachment
userjsonmap["capabilities"] = capabilities actorjsonmap["capabilities"] = capabilities
userjsonmap["discoverable"] = false actorjsonmap["discoverable"] = false
userjsonmap["endpoints"] = endpoints actorjsonmap["endpoints"] = endpoints
userjsonmap["followers"] = "https://" + host + "/users/fedilogue/followers" actorjsonmap["followers"] = "https://" + host + "/users/fedilogue/followers"
userjsonmap["following"] = "https://" + host + "/users/fedilogue/following" actorjsonmap["following"] = "https://" + host + "/users/fedilogue/following"
userjsonmap["id"] = "https://" + host + "/users/fedilogue" actorjsonmap["id"] = "https://" + host + "/users/fedilogue"
userjsonmap["inbox"] = "https://" + host + "/users/fedilogue/inbox" actorjsonmap["inbox"] = "https://" + host + "/users/fedilogue/inbox"
userjsonmap["manuallyApprovesFollowers"] = false actorjsonmap["manuallyApprovesFollowers"] = false
userjsonmap["name"] = "Fedilogue Mass Follower" actorjsonmap["name"] = "Fedilogue Mass Follower"
userjsonmap["outbox"] = "https://" + host + "/users/fedilogue/outbox" actorjsonmap["outbox"] = "https://" + host + "/users/fedilogue/outbox"
userjsonmap["preferredUsername"] = "fedilogue" actorjsonmap["preferredUsername"] = "fedilogue"
userjsonmap["publicKey"] = publicKey actorjsonmap["publicKey"] = publicKey
userjsonmap["summary"] = "" actorjsonmap["summary"] = ""
userjsonmap["tag"] = tag actorjsonmap["tag"] = tag
userjsonmap["type"] = "Person" actorjsonmap["type"] = "Person"
userjsonmap["uri"] = "https://" + host + "/users/fedilogue" actorjsonmap["uri"] = "https://" + host + "/users/fedilogue"
userjsonbin, err := json.Marshal(userjsonmap) actorjsonbin, err := json.Marshal(actorjsonmap)
if err != nil { if err != nil {
fmt.Println(err.Error()) fmt.Println(err.Error())
return return
} }
userjsonstr := string(userjsonbin) actorjsonstr := string(actorjsonbin)
w.Header().Set("Content-Type", "application/activity+json; charset=utf-8") w.Header().Set("Content-Type", "application/activity+json; charset=utf-8")
fmt.Fprintf(w, userjsonstr) fmt.Fprintf(w, actorjsonstr)
} }
func errorHandler(w http.ResponseWriter, r *http.Request) { func errorHandler(w http.ResponseWriter, r *http.Request) {