farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

commits to the database, added accounts and created_time

Browse Source
This commit is contained in:
farhan 2020-10-26 02:48:52 +00:00
parent f460900f41
commit 84a07edf07
2 changed files with 23 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
poll/engine.go
View File

@ -32,6 +32,7 @@ type ReportPost struct {
Url string `json:"url"` Url string `json:"url"`
Account AccountType Account AccountType
Content string `json:"content"` Content string `json:"content"`
Created_at string `json:"created_at"`
// Derived values // Derived values
StrippedContent string StrippedContent string
@ -87,9 +88,10 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
p := bluemonday.NewPolicy() p := bluemonday.NewPolicy()
newposts := make([]ReportPost, 0) newposts := make([]ReportPost, 0)
api_timeline := "https://" + endpoint + "/api/v1/timelines/public?min_id=" + min_id api_timeline := "https://" + endpoint + "/api/v1/timelines/public?since_id=40&min_id=" + min_id
resp, err := http.Get(api_timeline) resp, err := http.Get(api_timeline)
if err != nil { if err != nil {
fmt.Println("Failure to retrieve HTTPS data...")
log.Fatal(err) log.Fatal(err)
pollMessageChan <- PollMessage{endpoint, resp.StatusCode, "" , 0} pollMessageChan <- PollMessage{endpoint, resp.StatusCode, "" , 0}
} }
@ -97,9 +99,11 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &newposts) err = json.Unmarshal(body, &newposts)
if err != nil { if err != nil {
pollMessageChan <- PollMessage{endpoint, resp.StatusCode, "", 0}
fmt.Println("Failure to unmarshal 1")
fmt.Println(string(body))
log.Fatal(err) log.Fatal(err)
panic(err) panic(err)
pollMessageChan <- PollMessage{endpoint, resp.StatusCode, "", 0}
} }
numposts := 0 numposts := 0
@ -113,7 +117,6 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
fmt.Fprint(posthash, newpost.Url) fmt.Fprint(posthash, newpost.Url)
fmt.Fprint(posthash, newpost.Content) fmt.Fprint(posthash, newpost.Content)
fmt.Fprint(posthash, newpost.Account.Acct) fmt.Fprint(posthash, newpost.Account.Acct)
fmt.Fprint(posthash, newpost.Account.Created_at)
fmt.Fprint(posthash, newpost.Account.Display_name) fmt.Fprint(posthash, newpost.Account.Display_name)
fmt.Fprint(posthash, newpost.Account.Url) fmt.Fprint(posthash, newpost.Account.Url)
newpost.Posthash = posthash.Sum(nil) newpost.Posthash = posthash.Sum(nil)
@ -195,13 +198,12 @@ func NewInstance(endpoint string, runninginstances *[]RunningInstance, reportIns
*runninginstances = append(*runninginstances, newinstance) *runninginstances = append(*runninginstances, newinstance)
go StartInstancePoll(endpoint, "", reportPostChan, pollMessageChan) go StartInstancePoll(endpoint, "", reportPostChan, pollMessageChan)
fmt.Println("Temporarily disabled Peer Hunting") // fmt.Println("Temporarily disabled Peer Hunting")
// go StartGetPeers(endpoint, reportInstanceChan) // go StartGetPeers(endpoint, reportInstanceChan)
} }
func writePost(pool *pgxpool.Pool, reportpost ReportPost) { func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
fmt.Println("Writing post", reportpost) // fmt.Println("Writing post", reportpost)
// sqlWritePost := `INSERT INTO post (url, content, strippedcontent, posthash) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (posthash) DO NOTHING`
conn, err := pool.Acquire(context.Background()) conn, err := pool.Acquire(context.Background())
if err != nil { if err != nil {
@ -210,12 +212,23 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
} }
defer conn.Release() defer conn.Release()
_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, strippedcontent, posthash) VALUES ($1, $2, $3, $4) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.StrippedContent, reportpost.Posthash) // Insert new post if new
_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, strippedcontent, posthash) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.StrippedContent, reportpost.Posthash)
if err != nil { if err != nil {
fmt.Println("Error on channel???") fmt.Println("Error on channel???")
fmt.Println(err) fmt.Println(err)
os.Exit(1) os.Exit(1)
} }
// Insert new account if new
_, err = conn.Exec(context.Background(), "INSERT INTO accounts (acct, avatar, bot, created_at, display_name, url) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (acct) DO NOTHING", reportpost.Account.Acct, reportpost.Account.Avatar, reportpost.Account.Bot, reportpost.Account.Created_at, reportpost.Account.Display_name, reportpost.Account.Url)
if err != nil {
fmt.Println("Error on channel???")
fmt.Println(err)
os.Exit(1)
}
// fmt.Println(reportpost.Account)
} }
func main() { func main() {

4
poll/tables.psql
View File

@ -1,6 +1,6 @@
CREATE TABLE accounts ( CREATE TABLE accounts (
id serial NOT NULL PRIMARY KEY, id serial NOT NULL PRIMARY KEY,
acct VARCHAR(100) NOT NULL, acct VARCHAR(100) NOT NULL UNIQUE,
avatar VARCHAR(2083) NOT NULL, avatar VARCHAR(2083) NOT NULL,
bot boolean, bot boolean,
created_at VARCHAR(100) NOT NULL, created_at VARCHAR(100) NOT NULL,
@ -13,6 +13,8 @@ CREATE TABLE posts (
url VARCHAR(2083) NOT NULL, url VARCHAR(2083) NOT NULL,
content text, content text,
strippedcontent text, strippedcontent text,
created_at timestamptz DEFAULT NOW(),
posthash bytea UNIQUE posthash bytea UNIQUE
); );