farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added stream and poll

Browse Source 
fixed tables.sql
This commit is contained in:
farhan 2020-12-22 20:20:12 +00:00
parent 6b8d6a3bef
commit 39b53ed45c
4 changed files with 45 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
poll.go
View File

@ -3,7 +3,6 @@ package main
import (
//"crypto/sha1"
"encoding/json"
"fmt"
//"html"
"io/ioutil"
"log"
@ -244,11 +243,10 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost, http_c
ri_mutex.Unlock()
for _, newpost := range newposts {
fmt.Println("---------------> ", newpost.Uri)
go check_post(newpost.Uri)
at_sign := strings.Index(newpost.Account.Acct, "@")
newinstance := newpost.Account.Acct[at_sign+1:]
// Check min_id
if newpost.Id > min_id {
min_id = newpost.Id

16
retrieve.go
View File

@ -2,7 +2,6 @@ package main
import (
"context"
"fmt"
"strings"
"log"
"encoding/json"
@ -73,7 +72,6 @@ func requestConn() {
conn, _:= pool.Acquire(context.Background())
defer conn.Release()
for connRequest := range requestconnchan {
fmt.Println("Sending request")
connRequest.conn <-conn
_ = <-connRequest.b
}
@ -113,17 +111,13 @@ func check_post(uri string) (PostJson, error) {
close(connrequest.b)
if err == nil {
fmt.Println("First return!")
return postjson, nil
}
log.Print(uri)
endslash := strings.Index(uri[8:], "/")
postjson.instance = uri[8:endslash+8]
o := GetHTTPSession(postjson.instance)
// client := o.client
req, _ := http.NewRequest("GET", uri, nil)
req.Header.Add("Accept", "application/ld+json")
@ -135,7 +129,7 @@ func check_post(uri string) (PostJson, error) {
}
if postjson.InReplyTo != "" && postjson.InReplyTo != uri {
log.Print("GOING INTO NEW POST: ", postjson.InReplyTo)
log.Print("GOING INTO NEW POST: " + postjson.InReplyTo + " " + uri)
go check_post(postjson.InReplyTo)
}
@ -148,7 +142,7 @@ func check_post(uri string) (PostJson, error) {
myconn = <-connrequest.conn
_, err = myconn.Exec(context.Background(), "INSERT INTO posts (id, inReplyTo, published, summary, content, normalized, attributedto, posthash, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)", postjson.ID, postjson.InReplyTo, postjson.Published, postjson.Summary, postjson.Content, postjson.normalized, postjson.AttributedTo, postjson.posthash, postjson.instance)
_, err = myconn.Exec(context.Background(), "INSERT INTO posts (id, inreplyto, published, summary, content, normalized, attributedto, posthash, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)", postjson.ID, postjson.InReplyTo, postjson.Published, postjson.Summary, postjson.Content, postjson.normalized, postjson.AttributedTo, postjson.posthash, postjson.instance)
close(connrequest.b)
if err != nil {
log.Print("INSERT posts ", err)
@ -157,13 +151,10 @@ func check_post(uri string) (PostJson, error) {
for _, to := range postjson.To {
if to != "https://www.w3.org/ns/activitystreams#Public" {
log.Print("Going into: " + to)
go check_user(to)
}
}
fmt.Println("Second return")
return postjson, nil
}
@ -184,7 +175,6 @@ func check_user(uri string) (UserJson, error) {
close(connrequest.b)
if err == nil {
fmt.Println("First return!")
return userjson, nil
}
endslash := strings.Index(uri[8:], "/")
@ -220,8 +210,6 @@ func check_user(uri string) (UserJson, error) {
return userjson, err
}
fmt.Println("Second return")
return userjson, nil
}

1
stream.go
View File

@ -92,7 +92,6 @@ func StreamMastodon(endpoint string, reportPostChan chan ReportPost) {
continue
}
log.Print("----------> " + newpost.Uri)
go check_post(newpost.Uri)
at_sign := strings.Index(newpost.Account.Acct, "@")

69
tables.sql
View File

@ -1,34 +1,47 @@
DROP TABLE instances;
DROP TABLE accounts;
DROP TABLE posts;
CREATE TABLE accounts (
id serial NOT NULL PRIMARY KEY,
acct VARCHAR(1000) NOT NULL UNIQUE,
avatar VARCHAR(2083) NOT NULL,
bot boolean,
created_at timestamptz DEFAULT NOW(),
display_name VARCHAR(2000) NOT NULL,
uri VARCHAR(2083) NOT NULL
);
CREATE TABLE posts (
id serial NOT NULL PRIMARY KEY,
uri VARCHAR(2083) NOT NULL,
content text,
created_at timestamptz DEFAULT NOW(),
normalized text,
account_id int NOT NULL REFERENCES accounts (id),
posthash bytea UNIQUE,
received_at timestamptz DEFAULT NOW()
key serial NOT NULL,
actor_type character varying(1000) NOT NULL,
id character varying(2083) NOT NULL PRIMARY KEY,
inbox character varying(2083) NOT NULL,
outbox character varying(2083) NOT NULL,
followers character varying(2083) NOT NULL,
following character varying(2083) NOT NULL,
url character varying(2083) NOT NULL,
preferredusername character varying(1000) NOT NULL,
name character varying(1000) NOT NULL,
summary text,
icon character varying(2083),
image character varying(2083),
publickey text,
identifiedat timestamp with time zone DEFAULT now(),
instance character varying(1000) NOT NULL
);
CREATE TABLE instances (
id serial NOT NULL PRIMARY KEY,
endpoint VARCHAR(2083) NOT NULL,
id integer NOT NULL,
endpoint character varying(2083) NOT NULL,
autostart boolean,
state varchar(16),
username varchar(32),
password varchar(32),
software varchar(50)
state character varying(16),
username character varying(32),
password character varying(32),
software character varying(50)
);
CREATE TABLE posts (
key serial NOT NULL,
id character varying(2083) NOT NULL PRIMARY KEY,
inreplyto character varying(2083),
published timestamp with time zone NOT NULL,
summary text,
content text,
normalized text,
attributedto character varying(1000) NOT NULL,
posthash bytea,
received_at timestamp with time zone DEFAULT now(),
instance character varying(1000) NOT NULL
);