From 39b53ed45c499fa935b32fed57197873459b8aff Mon Sep 17 00:00:00 2001 From: Farhan Khan Date: Tue, 22 Dec 2020 20:20:12 +0000 Subject: [PATCH] added stream and poll fixed tables.sql --- poll.go | 4 +-- retrieve.go | 16 ++---------- stream.go | 1 - tables.sql | 71 +++++++++++++++++++++++++++++++---------------------- 4 files changed, 45 insertions(+), 47 deletions(-) diff --git a/poll.go b/poll.go index 99b7bfe..e13ac2c 100644 --- a/poll.go +++ b/poll.go @@ -3,7 +3,6 @@ package main import ( //"crypto/sha1" "encoding/json" - "fmt" //"html" "io/ioutil" "log" @@ -244,11 +243,10 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost, http_c ri_mutex.Unlock() for _, newpost := range newposts { - fmt.Println("---------------> ", newpost.Uri) + go check_post(newpost.Uri) at_sign := strings.Index(newpost.Account.Acct, "@") newinstance := newpost.Account.Acct[at_sign+1:] - // Check min_id if newpost.Id > min_id { min_id = newpost.Id diff --git a/retrieve.go b/retrieve.go index 71b6acc..c1a9aeb 100644 --- a/retrieve.go +++ b/retrieve.go @@ -2,7 +2,6 @@ package main import ( "context" - "fmt" "strings" "log" "encoding/json" @@ -73,7 +72,6 @@ func requestConn() { conn, _:= pool.Acquire(context.Background()) defer conn.Release() for connRequest := range requestconnchan { - fmt.Println("Sending request") connRequest.conn <-conn _ = <-connRequest.b } @@ -113,17 +111,13 @@ func check_post(uri string) (PostJson, error) { close(connrequest.b) if err == nil { - fmt.Println("First return!") return postjson, nil } - log.Print(uri) - endslash := strings.Index(uri[8:], "/") postjson.instance = uri[8:endslash+8] o := GetHTTPSession(postjson.instance) -// client := o.client req, _ := http.NewRequest("GET", uri, nil) req.Header.Add("Accept", "application/ld+json") @@ -135,7 +129,7 @@ func check_post(uri string) (PostJson, error) { } if postjson.InReplyTo != "" && postjson.InReplyTo != uri { - log.Print("GOING INTO NEW POST: ", postjson.InReplyTo) + log.Print("GOING INTO NEW POST: " + postjson.InReplyTo + " " + uri) go check_post(postjson.InReplyTo) } @@ -148,7 +142,7 @@ func check_post(uri string) (PostJson, error) { myconn = <-connrequest.conn - _, err = myconn.Exec(context.Background(), "INSERT INTO posts (id, inReplyTo, published, summary, content, normalized, attributedto, posthash, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)", postjson.ID, postjson.InReplyTo, postjson.Published, postjson.Summary, postjson.Content, postjson.normalized, postjson.AttributedTo, postjson.posthash, postjson.instance) + _, err = myconn.Exec(context.Background(), "INSERT INTO posts (id, inreplyto, published, summary, content, normalized, attributedto, posthash, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)", postjson.ID, postjson.InReplyTo, postjson.Published, postjson.Summary, postjson.Content, postjson.normalized, postjson.AttributedTo, postjson.posthash, postjson.instance) close(connrequest.b) if err != nil { log.Print("INSERT posts ", err) @@ -157,13 +151,10 @@ func check_post(uri string) (PostJson, error) { for _, to := range postjson.To { if to != "https://www.w3.org/ns/activitystreams#Public" { - log.Print("Going into: " + to) go check_user(to) } } - fmt.Println("Second return") - return postjson, nil } @@ -184,7 +175,6 @@ func check_user(uri string) (UserJson, error) { close(connrequest.b) if err == nil { - fmt.Println("First return!") return userjson, nil } endslash := strings.Index(uri[8:], "/") @@ -220,8 +210,6 @@ func check_user(uri string) (UserJson, error) { return userjson, err } - fmt.Println("Second return") - return userjson, nil } diff --git a/stream.go b/stream.go index 5f1bcfa..e3ca4c6 100644 --- a/stream.go +++ b/stream.go @@ -92,7 +92,6 @@ func StreamMastodon(endpoint string, reportPostChan chan ReportPost) { continue } - log.Print("----------> " + newpost.Uri) go check_post(newpost.Uri) at_sign := strings.Index(newpost.Account.Acct, "@") diff --git a/tables.sql b/tables.sql index a0b1085..c41cbff 100644 --- a/tables.sql +++ b/tables.sql @@ -1,34 +1,47 @@ +DROP TABLE instances; +DROP TABLE accounts; +DROP TABLE posts; + CREATE TABLE accounts ( - id serial NOT NULL PRIMARY KEY, - - acct VARCHAR(1000) NOT NULL UNIQUE, - avatar VARCHAR(2083) NOT NULL, - bot boolean, - created_at timestamptz DEFAULT NOW(), - display_name VARCHAR(2000) NOT NULL, - uri VARCHAR(2083) NOT NULL -); - -CREATE TABLE posts ( - id serial NOT NULL PRIMARY KEY, - - uri VARCHAR(2083) NOT NULL, - content text, - created_at timestamptz DEFAULT NOW(), - - normalized text, - account_id int NOT NULL REFERENCES accounts (id), - posthash bytea UNIQUE, - received_at timestamptz DEFAULT NOW() + key serial NOT NULL, + actor_type character varying(1000) NOT NULL, + id character varying(2083) NOT NULL PRIMARY KEY, + inbox character varying(2083) NOT NULL, + outbox character varying(2083) NOT NULL, + followers character varying(2083) NOT NULL, + following character varying(2083) NOT NULL, + url character varying(2083) NOT NULL, + preferredusername character varying(1000) NOT NULL, + name character varying(1000) NOT NULL, + summary text, + icon character varying(2083), + image character varying(2083), + publickey text, + identifiedat timestamp with time zone DEFAULT now(), + instance character varying(1000) NOT NULL ); CREATE TABLE instances ( - id serial NOT NULL PRIMARY KEY, - endpoint VARCHAR(2083) NOT NULL, - autostart boolean, - state varchar(16), - username varchar(32), - password varchar(32), - - software varchar(50) + id integer NOT NULL, + endpoint character varying(2083) NOT NULL, + autostart boolean, + state character varying(16), + username character varying(32), + password character varying(32), + software character varying(50) +); + + +CREATE TABLE posts ( + key serial NOT NULL, + id character varying(2083) NOT NULL PRIMARY KEY, + inreplyto character varying(2083), + published timestamp with time zone NOT NULL, + summary text, + content text, + normalized text, + attributedto character varying(1000) NOT NULL, + posthash bytea, + received_at timestamp with time zone DEFAULT now(), + instance character varying(1000) NOT NULL );