added stream and poll

fixed tables.sql

poll.go

@@ -3,7 +3,6 @@ package main
 import (
 	//"crypto/sha1"
 	"encoding/json"
-	"fmt"
 	//"html"
 	"io/ioutil"
 	"log"
@@ -244,11 +243,10 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost, http_c
 		ri_mutex.Unlock()
 
 		for _, newpost := range newposts {
-			fmt.Println("---------------> ", newpost.Uri)
+			go check_post(newpost.Uri)
 			at_sign := strings.Index(newpost.Account.Acct, "@")
 			newinstance := newpost.Account.Acct[at_sign+1:]
 
-
 			// Check min_id
 			if newpost.Id > min_id {
 				min_id = newpost.Id

retrieve.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"strings"
 	"log"
 	"encoding/json"
@@ -73,7 +72,6 @@ func requestConn() {
 	conn, _:= pool.Acquire(context.Background())
 	defer conn.Release()
 	for connRequest := range requestconnchan {
-		fmt.Println("Sending request")
 		connRequest.conn <-conn
 		_ = <-connRequest.b
 	}
@@ -113,17 +111,13 @@ func check_post(uri string) (PostJson, error) {
 	close(connrequest.b)
 
 	if err == nil {
-		fmt.Println("First return!")
 		return postjson, nil
 	}
-	log.Print(uri)
-
 	endslash := strings.Index(uri[8:], "/")
 	postjson.instance = uri[8:endslash+8]
 
 
 	o := GetHTTPSession(postjson.instance)
-//	client := o.client
 	req, _ := http.NewRequest("GET", uri, nil)
 	req.Header.Add("Accept", "application/ld+json")
 
@@ -135,7 +129,7 @@ func check_post(uri string) (PostJson, error) {
 	}
 
 	if postjson.InReplyTo != "" && postjson.InReplyTo != uri {
-		log.Print("GOING INTO NEW POST: ", postjson.InReplyTo)
+		log.Print("GOING INTO NEW POST: " + postjson.InReplyTo + " " + uri)
 		go check_post(postjson.InReplyTo)
 	}
 
@@ -148,7 +142,7 @@ func check_post(uri string) (PostJson, error) {
 
 	myconn = <-connrequest.conn
 
-	_, err = myconn.Exec(context.Background(), "INSERT INTO posts (id, inReplyTo, published, summary, content, normalized, attributedto, posthash, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)", postjson.ID, postjson.InReplyTo, postjson.Published, postjson.Summary, postjson.Content, postjson.normalized, postjson.AttributedTo, postjson.posthash, postjson.instance)
+	_, err = myconn.Exec(context.Background(), "INSERT INTO posts (id, inreplyto, published, summary, content, normalized, attributedto, posthash, instance) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)", postjson.ID, postjson.InReplyTo, postjson.Published, postjson.Summary, postjson.Content, postjson.normalized, postjson.AttributedTo, postjson.posthash, postjson.instance)
 	close(connrequest.b)
 	if err != nil {
 		log.Print("INSERT posts ", err)
@@ -157,13 +151,10 @@ func check_post(uri string) (PostJson, error) {
 
 	for _, to := range postjson.To {
 		if to != "https://www.w3.org/ns/activitystreams#Public" {
-			log.Print("Going into: " + to)
 			go check_user(to)
 		}
 	}
 
-	fmt.Println("Second return")
-
 	return postjson, nil
 }
 
@@ -184,7 +175,6 @@ func check_user(uri string) (UserJson, error) {
 	close(connrequest.b)
 
 	if err == nil {
-		fmt.Println("First return!")
 		return userjson, nil
 	}
 	endslash := strings.Index(uri[8:], "/")
@@ -220,8 +210,6 @@ func check_user(uri string) (UserJson, error) {
 		return userjson, err
 	}
 
-	fmt.Println("Second return")
-
 	return userjson, nil
 }
 

stream.go

@@ -92,7 +92,6 @@ func StreamMastodon(endpoint string, reportPostChan chan ReportPost) {
 			continue
 		}
 
-		log.Print("----------> " + newpost.Uri)
 		go check_post(newpost.Uri)
 
 		at_sign := strings.Index(newpost.Account.Acct, "@")

tables.sql

@@ -1,34 +1,47 @@
+DROP TABLE instances;
+DROP TABLE accounts;
+DROP TABLE posts;
+
 CREATE TABLE accounts (
-	id 		serial NOT NULL PRIMARY KEY,
+    key serial NOT NULL,
-
+    actor_type character varying(1000) NOT NULL,
-	acct		VARCHAR(1000) NOT NULL UNIQUE,
+    id character varying(2083) NOT NULL PRIMARY KEY,
-	avatar		VARCHAR(2083) NOT NULL,
+    inbox character varying(2083) NOT NULL,
-	bot		boolean,
+    outbox character varying(2083) NOT NULL,
-	created_at	timestamptz DEFAULT NOW(),
+    followers character varying(2083) NOT NULL,
-	display_name	VARCHAR(2000) NOT NULL,
+    following character varying(2083) NOT NULL,
-	uri		VARCHAR(2083) NOT NULL
+    url character varying(2083) NOT NULL,
-);
+    preferredusername character varying(1000) NOT NULL,
-
+    name character varying(1000) NOT NULL,
-CREATE TABLE posts (
+    summary text,
-	id		serial NOT NULL PRIMARY KEY,
+    icon character varying(2083),
-
+    image character varying(2083),
-	uri		VARCHAR(2083) NOT NULL,
+    publickey text,
-	content		text,
+    identifiedat timestamp with time zone DEFAULT now(),
-	created_at	timestamptz DEFAULT NOW(),
+    instance character varying(1000) NOT NULL
-
-	normalized	text,
-	account_id	int NOT NULL REFERENCES accounts (id),
-	posthash	bytea UNIQUE,
-	received_at	timestamptz DEFAULT NOW()
 );
 
 CREATE TABLE instances (
-	id		serial NOT NULL PRIMARY KEY,
+    id integer NOT NULL,
-	endpoint	VARCHAR(2083) NOT NULL,
+    endpoint character varying(2083) NOT NULL,
     autostart boolean,
-	state		varchar(16),
+    state character varying(16),
-	username	varchar(32),
+    username character varying(32),
-	password	varchar(32),
+    password character varying(32),
-
+    software character varying(50)
-	software	varchar(50)
+);
+
+
+CREATE TABLE posts (
+    key serial NOT NULL,
+    id character varying(2083) NOT NULL PRIMARY KEY,
+    inreplyto character varying(2083),
+    published timestamp with time zone NOT NULL,
+    summary text,
+    content text,
+    normalized text,
+    attributedto character varying(1000) NOT NULL,
+    posthash bytea,
+    received_at timestamp with time zone DEFAULT now(),
+    instance character varying(1000) NOT NULL
 );