farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

renamed column name, removed html encoding in normalization field

Browse Source
This commit is contained in:
farhan 2020-10-29 23:49:38 +00:00
parent 97a8cbf578
commit 4222cb97e0
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
poll/engine.go
View File

@ -9,6 +9,7 @@ import (
"net/http"
"context"
"strings"
"html"
"time"
"fmt"
"net"
@ -165,7 +166,7 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
fmt.Fprint(posthash, newpost.Account.Display_name)
newpost.posthash = posthash.Sum(nil)
newpost.normalized = strings.ToLower(p.Sanitize(newpost.Content))
newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content)))
reportPostChan <- newpost
@ -298,17 +299,16 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
defer conn.Release()
// Insert new account if new
var acctid int
err = conn.QueryRow(context.Background(), "INSERT INTO accounts (acct, avatar, bot, created_at, display_name, url) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT(acct) DO UPDATE SET acct=EXCLUDED.acct RETURNING id", reportpost.Account.Acct, reportpost.Account.Avatar, reportpost.Account.Bot, reportpost.Account.Created_at, reportpost.Account.Display_name, reportpost.Account.Url).Scan(&acctid)
var accountid int
err = conn.QueryRow(context.Background(), "INSERT INTO accounts (acct, avatar, bot, created_at, display_name, url) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT(acct) DO UPDATE SET acct=EXCLUDED.acct RETURNING id", reportpost.Account.Acct, reportpost.Account.Avatar, reportpost.Account.Bot, reportpost.Account.Created_at, reportpost.Account.Display_name, reportpost.Account.Url).Scan(&accountid)
if err != nil {
fmt.Println("First ", err)
fmt.Println(reportpost.Account.Acct, reportpost.Account.Avatar, reportpost.Account.Bot, reportpost.Account.Created_at, reportpost.Account.Display_name, reportpost.Account.Url)
os.Exit(1) // For now I want this to die and learn why it failed
return
}
// Insert new post if new
_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, normalized, acct_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, acctid, reportpost.posthash)
_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, normalized, account_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, accountid, reportpost.posthash)
if err != nil { // For now I want to know why this failed.
fmt.Println("Second ", err)
os.Exit(1) // For now I want this to die and learn why it failed

2
poll/tables.sql
View File

@ -17,7 +17,7 @@ CREATE TABLE posts (
created_at timestamptz DEFAULT NOW(),
normalized text,
acct_id int NOT NULL REFERENCES accounts (id),
account_id int NOT NULL REFERENCES accounts (id),
posthash bytea UNIQUE
);